initial ? wtf

fpr-bench/tools/README.md

@@ -0,0 +1,55 @@
+# FPR Bench CLI
+
+Run JMH benchmarks via a single Python CLI and export JSON results.
+
+Requires Python 3.4+.
+
+## Run a benchmark
+From the repo root:
+
+```sh
+python fpr-bench/tools/fpr_bench.py run
+```
+
+This builds the shaded JMH jar and writes results to `fpr-bench/results/`.
+You will be prompted for benchmark types and common JMH settings.
+
+Available types: `throughput`, `latency`, `common` (runs the combined bench).
+
+## Pass JMH arguments
+Use `--` to pass flags directly to JMH (this skips the menu):
+
+```sh
+python fpr-bench/tools/fpr_bench.py run -- --wi 5 -i 5 -f 5 -tu us
+```
+
+## Results location
+JSON output is stored in:
+
+```
+fpr-bench/results/YYYYMMDD_HHMMSS__<tag>__<type>.json
+```
+
+If multiple types are selected, one JSON file is produced per type.
+
+## Examples
+Skip the build and use a custom tag:
+
+```sh
+python fpr-bench/tools/fpr_bench.py run --no-build --tag smoke
+```
+
+Run only latency and throughput types non-interactively:
+
+```sh
+python fpr-bench/tools/fpr_bench.py run --types latency,throughput -- --wi 3 -i 3 -f 1
+```
+
+Use custom `mvn` and `java` executables:
+
+```sh
+python fpr-bench/tools/fpr_bench.py run --mvn mvn.cmd --java java
+```
+
+If another JMH instance is detected, the tool will try to terminate it. If that
+fails, it runs with `-Djmh.ignoreLock=true`.

fpr-bench/tools/fpr_bench.py

@@ -0,0 +1,536 @@
+#!/usr/bin/env python3
+from __future__ import print_function
+
+import argparse
+import json
+from datetime import datetime
+import subprocess
+import sys
+import shutil
+import os
+import tempfile
+
+try:
+    from pathlib import Path
+except ImportError:
+    print("Error: Python 3.4+ is required to run this tool.", file=sys.stderr)
+    sys.exit(2)
+
+BENCH_TYPES = {
+    "throughput": "dev.relism.fpr.bench.RouterBenchThroughput",
+    "latency": "dev.relism.fpr.bench.RouterBenchLatency",
+    "common": "dev.relism.fpr.bench.RouterBench",
+}
+BENCH_ORDER = ["throughput", "latency", "common"]
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(
+        description="Run FPR JMH benchmarks and export JSON results."
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    run_parser = subparsers.add_parser("run", help="Build and run JMH benchmarks.")
+    run_parser.add_argument("--tag", default="default", help="Tag for the output run folder.")
+    run_parser.add_argument(
+        "--no-build",
+        action="store_true",
+        help="Skip the Maven build step.",
+    )
+    run_parser.add_argument("--mvn", default="mvn", help="Maven executable path.")
+    run_parser.add_argument("--java", default="java", help="Java executable path.")
+    run_parser.add_argument("--jar", help="Path to the shaded JMH jar.")
+    run_parser.add_argument(
+        "--types",
+        help="Comma-separated benchmark types: throughput, latency, common.",
+    )
+    run_parser.add_argument(
+        "--prof-gc",
+        action="store_true",
+        help="Enable JMH GC profiler (-prof gc).",
+    )
+    run_parser.add_argument(
+        "jmh_args",
+        nargs=argparse.REMAINDER,
+        help="Arguments passed to JMH (use -- to separate).",
+    )
+
+    args = parser.parse_args(argv)
+    if args.command is None:
+        parser.print_help()
+        return None
+    return args
+
+
+def repo_root():
+    # assumes this script lives under fpr-bench/tools (or similar)
+    return Path(__file__).resolve().parents[2]
+
+
+def validate_tag(tag):
+    tag = tag.strip() if tag else ""
+    if not tag:
+        return "default"
+    invalid_chars = set('<>:"/\\|?*')
+    if any(ch in invalid_chars for ch in tag):
+        raise ValueError('Tag contains invalid filename characters: <>:"/\\|?*')
+    return tag
+
+
+def resolve_executable(executable, label):
+    resolved = shutil.which(executable)
+    if resolved:
+        return resolved
+    if Path(executable).is_file():
+        return str(Path(executable))
+    raise RuntimeError("{} executable not found: {}".format(label, executable))
+
+
+def run_subprocess(cmd, cwd, failure_message):
+    try:
+        return_code = subprocess.call(cmd, cwd=cwd)
+    except OSError as exc:
+        print("{}: {}".format(failure_message, exc), file=sys.stderr)
+        return 2
+    if return_code != 0:
+        print(
+            "{} (exit code {})".format(failure_message, return_code),
+            file=sys.stderr,
+        )
+    return return_code
+
+
+def run_subprocess_capture(cmd, cwd, log_path, failure_message):
+    """
+    Runs a subprocess, captures combined stdout+stderr, and always writes it to log_path.
+    Returns the process exit code (or 2 on OS errors).
+    """
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+    except OSError as exc:
+        try:
+            log_path.write_text(str(exc) + "\n", encoding="utf-8")
+        except OSError:
+            pass
+        print("{}: {}".format(failure_message, exc), file=sys.stderr)
+        print("Log: {}".format(log_path), file=sys.stderr)
+        return 2
+
+    try:
+        with log_path.open("wb") as log_file:
+            stream = proc.stdout
+            if stream is None:
+                proc.wait()
+            else:
+                for chunk in iter(lambda: stream.read(4096), b""):
+                    log_file.write(chunk)
+                    try:
+                        sys.stdout.buffer.write(chunk)
+                        sys.stdout.buffer.flush()
+                    except AttributeError:
+                        sys.stdout.write(chunk.decode("utf-8", errors="replace"))
+                        sys.stdout.flush()
+        return_code = proc.wait()
+    except OSError as exc:
+        print("Warning: could not write log {}: {}".format(log_path, exc), file=sys.stderr)
+        return_code = proc.wait()
+
+    if return_code != 0:
+        print("{} (exit code {})".format(failure_message, return_code), file=sys.stderr)
+        print("Log: {}".format(log_path), file=sys.stderr)
+    return return_code
+
+
+def prompt_tag(default_tag):
+    raw = input("Tag [{}]: ".format(default_tag)).strip()
+    return raw if raw else default_tag
+
+
+def prompt_int(label, default_value):
+    while True:
+        raw = input("{} [{}]: ".format(label, default_value)).strip()
+        if not raw:
+            return default_value
+        try:
+            return int(raw)
+        except ValueError:
+            print("Please enter a whole number.")
+
+
+def prompt_optional(label):
+    raw = input("{} (blank to skip): ".format(label)).strip()
+    return raw if raw else None
+
+
+def prompt_bench_types():
+    while True:
+        print("Select benchmark types:")
+        for idx, name in enumerate(BENCH_ORDER, start=1):
+            print("  {}) {}".format(idx, name))
+        print("  a) all")
+        raw = input("Choice [a]: ").strip().lower()
+        if raw in ("", "a", "all"):
+            return list(BENCH_ORDER)
+
+        tokens = [token for token in raw.replace(",", " ").split() if token]
+        selected = []
+        invalid = None
+        for token in tokens:
+            if token.isdigit():
+                index = int(token)
+                if 1 <= index <= len(BENCH_ORDER):
+                    name = BENCH_ORDER[index - 1]
+                    if name not in selected:
+                        selected.append(name)
+                    continue
+            if token in BENCH_TYPES:
+                if token not in selected:
+                    selected.append(token)
+                continue
+            invalid = token
+            break
+
+        if selected and invalid is None:
+            return selected
+        if invalid:
+            print("Unknown selection: {}".format(invalid))
+        else:
+            print("No valid selections provided.")
+
+
+def prompt_jmh_args():
+    jmh_args = []
+    wi = prompt_int("Warmup iterations (-wi)", 5)
+    i = prompt_int("Measurement iterations (-i)", 5)
+    f = prompt_int("Forks (-f)", 1)
+    w = prompt_optional("Warmup time (-w, e.g. 1s)")
+    r = prompt_optional("Measurement time (-r, e.g. 1s)")
+    t = prompt_optional("Threads (-t)")
+    tu = prompt_optional("Time unit (-tu, e.g. us)")
+
+    if wi is not None:
+        jmh_args.extend(["-wi", str(wi)])
+    if i is not None:
+        jmh_args.extend(["-i", str(i)])
+    if f is not None:
+        jmh_args.extend(["-f", str(f)])
+    if w:
+        jmh_args.extend(["-w", w])
+    if r:
+        jmh_args.extend(["-r", r])
+    if t:
+        jmh_args.extend(["-t", t])
+    if tu:
+        jmh_args.extend(["-tu", tu])
+    return jmh_args
+
+
+def parse_types_arg(value):
+    if not value:
+        return None
+    tokens = [token.strip().lower() for token in value.split(",") if token.strip()]
+    if not tokens:
+        return None
+    if "all" in tokens:
+        return list(BENCH_ORDER)
+    selected = []
+    for token in tokens:
+        if token not in BENCH_TYPES:
+            raise ValueError("Unknown benchmark type: {}".format(token))
+        if token not in selected:
+            selected.append(token)
+    return selected
+
+
+def find_jmh_pids_windows():
+    command = [
+        "powershell",
+        "-NoProfile",
+        "-Command",
+        (
+            "Get-CimInstance Win32_Process | "
+            "Where-Object { $_.Name -match 'java' -and $_.CommandLine -and "
+            "($_.CommandLine -match 'org\\.openjdk\\.jmh' -or "
+            "$_.CommandLine -match 'jmh' -or $_.CommandLine -match 'fpr-bench') } | "
+            "Select-Object -ExpandProperty ProcessId"
+        ),
+    ]
+    try:
+        output = subprocess.check_output(command, universal_newlines=True)
+    except (OSError, subprocess.CalledProcessError):
+        return None
+    pids = []
+    for line in output.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            pids.append(int(line))
+        except ValueError:
+            continue
+    return pids
+
+
+def find_jmh_pids_unix():
+    try:
+        output = subprocess.check_output(
+            ["ps", "-ax", "-o", "pid=,command="], universal_newlines=True
+        )
+    except (OSError, subprocess.CalledProcessError):
+        return None
+    pids = []
+    for line in output.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parts = line.split(None, 1)
+        if len(parts) != 2:
+            continue
+        pid_str, cmdline = parts
+        cmd_lower = cmdline.lower()
+        if "java" not in cmd_lower:
+            continue
+        if "org.openjdk.jmh" in cmd_lower or "jmh" in cmd_lower or "fpr-bench" in cmd_lower:
+            try:
+                pids.append(int(pid_str))
+            except ValueError:
+                continue
+    return pids
+
+
+def kill_jmh_processes():
+    if os.name == "nt":
+        pids = find_jmh_pids_windows()
+    else:
+        pids = find_jmh_pids_unix()
+
+    if pids is None:
+        return False
+    if not pids:
+        return True
+
+    success = True
+    if os.name == "nt":
+        for pid in pids:
+            return_code = subprocess.call(
+                ["taskkill", "/PID", str(pid), "/T", "/F"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            if return_code != 0:
+                success = False
+    else:
+        for pid in pids:
+            return_code = subprocess.call(
+                ["kill", "-9", str(pid)],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            if return_code != 0:
+                success = False
+    return success
+
+
+def should_ignore_lock(kill_success):
+    if not kill_success:
+        return True
+    lock_path = Path(tempfile.gettempdir()) / "jmh.lock"
+    return lock_path.exists()
+
+
+def find_shaded_jar(bench_dir, override_path):
+    if override_path:
+        jar_path = Path(override_path).expanduser()
+        if not jar_path.is_absolute():
+            jar_path = (Path.cwd() / jar_path).resolve()
+        if not jar_path.is_file():
+            raise RuntimeError("Jar not found: {}".format(jar_path))
+        return jar_path
+
+    target_dir = bench_dir / "target"
+    if not target_dir.exists():
+        raise RuntimeError("Jar not found: {} does not exist".format(target_dir))
+    candidates = sorted(
+        target_dir.glob("*shaded*.jar"),
+        key=lambda path: path.stat().st_mtime,
+        reverse=True,
+    )
+    if not candidates:
+        raise RuntimeError("Jar not found: no shaded JMH jar under {}".format(target_dir))
+    return candidates[0].resolve()
+
+
+def ensure_results_dir(results_dir):
+    if results_dir.exists() and not results_dir.is_dir():
+        raise RuntimeError("Results path exists but is not a directory: {}".format(results_dir))
+    results_dir.mkdir(parents=True, exist_ok=True)
+
+
+def prepare_run_dir(results_dir, timestamp, tag):
+    safe_tag = validate_tag(tag)
+    folder_name = "{}__{}".format(timestamp, safe_tag)
+    run_dir = results_dir / folder_name
+    run_dir.mkdir(parents=True, exist_ok=True)
+    return run_dir.resolve()
+
+
+def output_paths_for(run_dir, bench_type, prof_gc):
+    suffix = "__gc" if prof_gc else ""
+    json_path = run_dir / "{}{}.json".format(bench_type, suffix)
+    log_path = run_dir / "{}{}.log".format(bench_type, suffix)
+    return json_path.resolve(), log_path.resolve()
+
+
+def run_benchmarks(args):
+    root = repo_root()
+    bench_dir = root / "fpr-bench"
+    results_dir = bench_dir / "results"
+    try:
+        ensure_results_dir(results_dir)
+    except RuntimeError as exc:
+        print("Error: {}".format(exc), file=sys.stderr)
+        return 2
+
+    jmh_args = list(args.jmh_args or [])
+    if jmh_args and jmh_args[0] == "--":
+        jmh_args = jmh_args[1:]
+
+    interactive = not jmh_args and sys.stdin.isatty()
+    if interactive:
+        tag = prompt_tag(args.tag or "default")
+    else:
+        tag = args.tag
+
+    try:
+        tag = validate_tag(tag)
+    except ValueError as exc:
+        print("Error: {}".format(exc), file=sys.stderr)
+        return 2
+
+    try:
+        selected_types = parse_types_arg(args.types)
+    except ValueError as exc:
+        print("Error: {}".format(exc), file=sys.stderr)
+        return 2
+    if selected_types is None:
+        if interactive:
+            selected_types = prompt_bench_types()
+        else:
+            selected_types = list(BENCH_ORDER)
+
+    if interactive:
+        jmh_args = prompt_jmh_args()
+
+    if not args.no_build:
+        try:
+            mvn_exec = resolve_executable(args.mvn, "Maven")
+        except RuntimeError as exc:
+            print("Error: {}".format(exc), file=sys.stderr)
+            return 2
+
+        mvn_cmd = [mvn_exec, "-q", "-pl", "fpr-bench", "-am", "package", "-Dfile.encoding=UTF-8"]
+        exit_code = run_subprocess(mvn_cmd, root, "Maven build failed")
+        if exit_code != 0:
+            return exit_code
+
+    try:
+        jar_path = find_shaded_jar(bench_dir, args.jar)
+    except RuntimeError as exc:
+        print("Error: {}".format(exc), file=sys.stderr)
+        return 2
+
+    ignore_lock = should_ignore_lock(kill_jmh_processes())
+    if ignore_lock:
+        print(
+            "Warning: Unable to terminate existing JMH instance or lock file exists; using -Djmh.ignoreLock=true",
+            file=sys.stderr,
+        )
+
+    try:
+        java_exec = resolve_executable(args.java, "Java")
+    except RuntimeError as exc:
+        print("Error: {}".format(exc), file=sys.stderr)
+        return 2
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    run_dir = prepare_run_dir(results_dir, timestamp, tag)
+    print("Run directory: {}".format(run_dir))
+
+    prof_gc = bool(getattr(args, "prof_gc", False))
+
+    for bench_type in selected_types:
+        bench_class = BENCH_TYPES[bench_type]
+        json_path, log_path = output_paths_for(run_dir, bench_type, prof_gc)
+
+        # overwrite if exists
+        try:
+            if json_path.exists():
+                json_path.unlink()
+        except OSError:
+            pass
+        try:
+            if log_path.exists():
+                log_path.unlink()
+        except OSError:
+            pass
+
+        jmh_cmd = [java_exec]
+        if ignore_lock:
+            jmh_cmd.append("-Djmh.ignoreLock=true")
+        jmh_cmd.extend(["-jar", str(jar_path)])
+
+        if prof_gc:
+            jmh_cmd.extend(["-prof", "gc"])
+
+        if jmh_args:
+            jmh_cmd.extend(jmh_args)
+
+        jmh_cmd.append("^{}\\.".format(bench_class))
+        jmh_cmd.extend(["-rf", "json", "-rff", str(json_path)])
+
+        exit_code = run_subprocess_capture(
+            jmh_cmd, root, log_path, "JMH run failed ({})".format(bench_type)
+        )
+        if exit_code != 0:
+            return exit_code
+
+        # validate JSON
+        try:
+            with json_path.open("r", encoding="utf-8") as handle:
+                data = json.load(handle)
+            if not isinstance(data, list):
+                raise ValueError("JSON root is not an array")
+        except (OSError, ValueError, json.JSONDecodeError) as exc:
+            print("Error: JSON could not be parsed: {}".format(exc), file=sys.stderr)
+            print("JSON: {}".format(json_path), file=sys.stderr)
+            print("Log : {}".format(log_path), file=sys.stderr)
+            return 2
+
+        size_bytes = json_path.stat().st_size
+        print("JMH run completed ({})".format(bench_type))
+        print("JSON: {}".format(json_path))
+        print("LOG : {}".format(log_path))
+        print("File size: {} bytes".format(size_bytes))
+        print("Benchmarks: {}".format(len(data)))
+
+    print("All selected benchmarks completed. Results folder: {}".format(run_dir))
+    return 0
+
+
+def main(argv):
+    args = parse_args(argv)
+    if args is None:
+        return 2
+    if args.command == "run":
+        return run_benchmarks(args)
+    print("Unknown command: {}".format(args.command), file=sys.stderr)
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))