537 lines
16 KiB
Python
537 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import json
|
|
from datetime import datetime
|
|
import subprocess
|
|
import sys
|
|
import shutil
|
|
import os
|
|
import tempfile
|
|
|
|
try:
|
|
from pathlib import Path
|
|
except ImportError:
|
|
print("Error: Python 3.4+ is required to run this tool.", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
BENCH_TYPES = {
|
|
"throughput": "dev.relism.fpr.bench.RouterBenchThroughput",
|
|
"latency": "dev.relism.fpr.bench.RouterBenchLatency",
|
|
"common": "dev.relism.fpr.bench.RouterBench",
|
|
}
|
|
BENCH_ORDER = ["throughput", "latency", "common"]
|
|
|
|
|
|
def parse_args(argv):
|
|
parser = argparse.ArgumentParser(
|
|
description="Run FPR JMH benchmarks and export JSON results."
|
|
)
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
run_parser = subparsers.add_parser("run", help="Build and run JMH benchmarks.")
|
|
run_parser.add_argument("--tag", default="default", help="Tag for the output run folder.")
|
|
run_parser.add_argument(
|
|
"--no-build",
|
|
action="store_true",
|
|
help="Skip the Maven build step.",
|
|
)
|
|
run_parser.add_argument("--mvn", default="mvn", help="Maven executable path.")
|
|
run_parser.add_argument("--java", default="java", help="Java executable path.")
|
|
run_parser.add_argument("--jar", help="Path to the shaded JMH jar.")
|
|
run_parser.add_argument(
|
|
"--types",
|
|
help="Comma-separated benchmark types: throughput, latency, common.",
|
|
)
|
|
run_parser.add_argument(
|
|
"--prof-gc",
|
|
action="store_true",
|
|
help="Enable JMH GC profiler (-prof gc).",
|
|
)
|
|
run_parser.add_argument(
|
|
"jmh_args",
|
|
nargs=argparse.REMAINDER,
|
|
help="Arguments passed to JMH (use -- to separate).",
|
|
)
|
|
|
|
args = parser.parse_args(argv)
|
|
if args.command is None:
|
|
parser.print_help()
|
|
return None
|
|
return args
|
|
|
|
|
|
def repo_root():
|
|
# assumes this script lives under fpr-bench/tools (or similar)
|
|
return Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def validate_tag(tag):
|
|
tag = tag.strip() if tag else ""
|
|
if not tag:
|
|
return "default"
|
|
invalid_chars = set('<>:"/\\|?*')
|
|
if any(ch in invalid_chars for ch in tag):
|
|
raise ValueError('Tag contains invalid filename characters: <>:"/\\|?*')
|
|
return tag
|
|
|
|
|
|
def resolve_executable(executable, label):
|
|
resolved = shutil.which(executable)
|
|
if resolved:
|
|
return resolved
|
|
if Path(executable).is_file():
|
|
return str(Path(executable))
|
|
raise RuntimeError("{} executable not found: {}".format(label, executable))
|
|
|
|
|
|
def run_subprocess(cmd, cwd, failure_message):
|
|
try:
|
|
return_code = subprocess.call(cmd, cwd=cwd)
|
|
except OSError as exc:
|
|
print("{}: {}".format(failure_message, exc), file=sys.stderr)
|
|
return 2
|
|
if return_code != 0:
|
|
print(
|
|
"{} (exit code {})".format(failure_message, return_code),
|
|
file=sys.stderr,
|
|
)
|
|
return return_code
|
|
|
|
|
|
def run_subprocess_capture(cmd, cwd, log_path, failure_message):
|
|
"""
|
|
Runs a subprocess, captures combined stdout+stderr, and always writes it to log_path.
|
|
Returns the process exit code (or 2 on OS errors).
|
|
"""
|
|
try:
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
cwd=cwd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
except OSError as exc:
|
|
try:
|
|
log_path.write_text(str(exc) + "\n", encoding="utf-8")
|
|
except OSError:
|
|
pass
|
|
print("{}: {}".format(failure_message, exc), file=sys.stderr)
|
|
print("Log: {}".format(log_path), file=sys.stderr)
|
|
return 2
|
|
|
|
try:
|
|
with log_path.open("wb") as log_file:
|
|
stream = proc.stdout
|
|
if stream is None:
|
|
proc.wait()
|
|
else:
|
|
for chunk in iter(lambda: stream.read(4096), b""):
|
|
log_file.write(chunk)
|
|
try:
|
|
sys.stdout.buffer.write(chunk)
|
|
sys.stdout.buffer.flush()
|
|
except AttributeError:
|
|
sys.stdout.write(chunk.decode("utf-8", errors="replace"))
|
|
sys.stdout.flush()
|
|
return_code = proc.wait()
|
|
except OSError as exc:
|
|
print("Warning: could not write log {}: {}".format(log_path, exc), file=sys.stderr)
|
|
return_code = proc.wait()
|
|
|
|
if return_code != 0:
|
|
print("{} (exit code {})".format(failure_message, return_code), file=sys.stderr)
|
|
print("Log: {}".format(log_path), file=sys.stderr)
|
|
return return_code
|
|
|
|
|
|
def prompt_tag(default_tag):
|
|
raw = input("Tag [{}]: ".format(default_tag)).strip()
|
|
return raw if raw else default_tag
|
|
|
|
|
|
def prompt_int(label, default_value):
|
|
while True:
|
|
raw = input("{} [{}]: ".format(label, default_value)).strip()
|
|
if not raw:
|
|
return default_value
|
|
try:
|
|
return int(raw)
|
|
except ValueError:
|
|
print("Please enter a whole number.")
|
|
|
|
|
|
def prompt_optional(label):
|
|
raw = input("{} (blank to skip): ".format(label)).strip()
|
|
return raw if raw else None
|
|
|
|
|
|
def prompt_bench_types():
|
|
while True:
|
|
print("Select benchmark types:")
|
|
for idx, name in enumerate(BENCH_ORDER, start=1):
|
|
print(" {}) {}".format(idx, name))
|
|
print(" a) all")
|
|
raw = input("Choice [a]: ").strip().lower()
|
|
if raw in ("", "a", "all"):
|
|
return list(BENCH_ORDER)
|
|
|
|
tokens = [token for token in raw.replace(",", " ").split() if token]
|
|
selected = []
|
|
invalid = None
|
|
for token in tokens:
|
|
if token.isdigit():
|
|
index = int(token)
|
|
if 1 <= index <= len(BENCH_ORDER):
|
|
name = BENCH_ORDER[index - 1]
|
|
if name not in selected:
|
|
selected.append(name)
|
|
continue
|
|
if token in BENCH_TYPES:
|
|
if token not in selected:
|
|
selected.append(token)
|
|
continue
|
|
invalid = token
|
|
break
|
|
|
|
if selected and invalid is None:
|
|
return selected
|
|
if invalid:
|
|
print("Unknown selection: {}".format(invalid))
|
|
else:
|
|
print("No valid selections provided.")
|
|
|
|
|
|
def prompt_jmh_args():
|
|
jmh_args = []
|
|
wi = prompt_int("Warmup iterations (-wi)", 5)
|
|
i = prompt_int("Measurement iterations (-i)", 5)
|
|
f = prompt_int("Forks (-f)", 1)
|
|
w = prompt_optional("Warmup time (-w, e.g. 1s)")
|
|
r = prompt_optional("Measurement time (-r, e.g. 1s)")
|
|
t = prompt_optional("Threads (-t)")
|
|
tu = prompt_optional("Time unit (-tu, e.g. us)")
|
|
|
|
if wi is not None:
|
|
jmh_args.extend(["-wi", str(wi)])
|
|
if i is not None:
|
|
jmh_args.extend(["-i", str(i)])
|
|
if f is not None:
|
|
jmh_args.extend(["-f", str(f)])
|
|
if w:
|
|
jmh_args.extend(["-w", w])
|
|
if r:
|
|
jmh_args.extend(["-r", r])
|
|
if t:
|
|
jmh_args.extend(["-t", t])
|
|
if tu:
|
|
jmh_args.extend(["-tu", tu])
|
|
return jmh_args
|
|
|
|
|
|
def parse_types_arg(value):
|
|
if not value:
|
|
return None
|
|
tokens = [token.strip().lower() for token in value.split(",") if token.strip()]
|
|
if not tokens:
|
|
return None
|
|
if "all" in tokens:
|
|
return list(BENCH_ORDER)
|
|
selected = []
|
|
for token in tokens:
|
|
if token not in BENCH_TYPES:
|
|
raise ValueError("Unknown benchmark type: {}".format(token))
|
|
if token not in selected:
|
|
selected.append(token)
|
|
return selected
|
|
|
|
|
|
def find_jmh_pids_windows():
|
|
command = [
|
|
"powershell",
|
|
"-NoProfile",
|
|
"-Command",
|
|
(
|
|
"Get-CimInstance Win32_Process | "
|
|
"Where-Object { $_.Name -match 'java' -and $_.CommandLine -and "
|
|
"($_.CommandLine -match 'org\\.openjdk\\.jmh' -or "
|
|
"$_.CommandLine -match 'jmh' -or $_.CommandLine -match 'fpr-bench') } | "
|
|
"Select-Object -ExpandProperty ProcessId"
|
|
),
|
|
]
|
|
try:
|
|
output = subprocess.check_output(command, universal_newlines=True)
|
|
except (OSError, subprocess.CalledProcessError):
|
|
return None
|
|
pids = []
|
|
for line in output.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
pids.append(int(line))
|
|
except ValueError:
|
|
continue
|
|
return pids
|
|
|
|
|
|
def find_jmh_pids_unix():
|
|
try:
|
|
output = subprocess.check_output(
|
|
["ps", "-ax", "-o", "pid=,command="], universal_newlines=True
|
|
)
|
|
except (OSError, subprocess.CalledProcessError):
|
|
return None
|
|
pids = []
|
|
for line in output.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
parts = line.split(None, 1)
|
|
if len(parts) != 2:
|
|
continue
|
|
pid_str, cmdline = parts
|
|
cmd_lower = cmdline.lower()
|
|
if "java" not in cmd_lower:
|
|
continue
|
|
if "org.openjdk.jmh" in cmd_lower or "jmh" in cmd_lower or "fpr-bench" in cmd_lower:
|
|
try:
|
|
pids.append(int(pid_str))
|
|
except ValueError:
|
|
continue
|
|
return pids
|
|
|
|
|
|
def kill_jmh_processes():
|
|
if os.name == "nt":
|
|
pids = find_jmh_pids_windows()
|
|
else:
|
|
pids = find_jmh_pids_unix()
|
|
|
|
if pids is None:
|
|
return False
|
|
if not pids:
|
|
return True
|
|
|
|
success = True
|
|
if os.name == "nt":
|
|
for pid in pids:
|
|
return_code = subprocess.call(
|
|
["taskkill", "/PID", str(pid), "/T", "/F"],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
if return_code != 0:
|
|
success = False
|
|
else:
|
|
for pid in pids:
|
|
return_code = subprocess.call(
|
|
["kill", "-9", str(pid)],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
if return_code != 0:
|
|
success = False
|
|
return success
|
|
|
|
|
|
def should_ignore_lock(kill_success):
|
|
if not kill_success:
|
|
return True
|
|
lock_path = Path(tempfile.gettempdir()) / "jmh.lock"
|
|
return lock_path.exists()
|
|
|
|
|
|
def find_shaded_jar(bench_dir, override_path):
|
|
if override_path:
|
|
jar_path = Path(override_path).expanduser()
|
|
if not jar_path.is_absolute():
|
|
jar_path = (Path.cwd() / jar_path).resolve()
|
|
if not jar_path.is_file():
|
|
raise RuntimeError("Jar not found: {}".format(jar_path))
|
|
return jar_path
|
|
|
|
target_dir = bench_dir / "target"
|
|
if not target_dir.exists():
|
|
raise RuntimeError("Jar not found: {} does not exist".format(target_dir))
|
|
candidates = sorted(
|
|
target_dir.glob("*shaded*.jar"),
|
|
key=lambda path: path.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
if not candidates:
|
|
raise RuntimeError("Jar not found: no shaded JMH jar under {}".format(target_dir))
|
|
return candidates[0].resolve()
|
|
|
|
|
|
def ensure_results_dir(results_dir):
|
|
if results_dir.exists() and not results_dir.is_dir():
|
|
raise RuntimeError("Results path exists but is not a directory: {}".format(results_dir))
|
|
results_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def prepare_run_dir(results_dir, timestamp, tag):
|
|
safe_tag = validate_tag(tag)
|
|
folder_name = "{}__{}".format(timestamp, safe_tag)
|
|
run_dir = results_dir / folder_name
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
return run_dir.resolve()
|
|
|
|
|
|
def output_paths_for(run_dir, bench_type, prof_gc):
|
|
suffix = "__gc" if prof_gc else ""
|
|
json_path = run_dir / "{}{}.json".format(bench_type, suffix)
|
|
log_path = run_dir / "{}{}.log".format(bench_type, suffix)
|
|
return json_path.resolve(), log_path.resolve()
|
|
|
|
|
|
def run_benchmarks(args):
|
|
root = repo_root()
|
|
bench_dir = root / "fpr-bench"
|
|
results_dir = bench_dir / "results"
|
|
try:
|
|
ensure_results_dir(results_dir)
|
|
except RuntimeError as exc:
|
|
print("Error: {}".format(exc), file=sys.stderr)
|
|
return 2
|
|
|
|
jmh_args = list(args.jmh_args or [])
|
|
if jmh_args and jmh_args[0] == "--":
|
|
jmh_args = jmh_args[1:]
|
|
|
|
interactive = not jmh_args and sys.stdin.isatty()
|
|
if interactive:
|
|
tag = prompt_tag(args.tag or "default")
|
|
else:
|
|
tag = args.tag
|
|
|
|
try:
|
|
tag = validate_tag(tag)
|
|
except ValueError as exc:
|
|
print("Error: {}".format(exc), file=sys.stderr)
|
|
return 2
|
|
|
|
try:
|
|
selected_types = parse_types_arg(args.types)
|
|
except ValueError as exc:
|
|
print("Error: {}".format(exc), file=sys.stderr)
|
|
return 2
|
|
if selected_types is None:
|
|
if interactive:
|
|
selected_types = prompt_bench_types()
|
|
else:
|
|
selected_types = list(BENCH_ORDER)
|
|
|
|
if interactive:
|
|
jmh_args = prompt_jmh_args()
|
|
|
|
if not args.no_build:
|
|
try:
|
|
mvn_exec = resolve_executable(args.mvn, "Maven")
|
|
except RuntimeError as exc:
|
|
print("Error: {}".format(exc), file=sys.stderr)
|
|
return 2
|
|
|
|
mvn_cmd = [mvn_exec, "-q", "-pl", "fpr-bench", "-am", "package", "-Dfile.encoding=UTF-8"]
|
|
exit_code = run_subprocess(mvn_cmd, root, "Maven build failed")
|
|
if exit_code != 0:
|
|
return exit_code
|
|
|
|
try:
|
|
jar_path = find_shaded_jar(bench_dir, args.jar)
|
|
except RuntimeError as exc:
|
|
print("Error: {}".format(exc), file=sys.stderr)
|
|
return 2
|
|
|
|
ignore_lock = should_ignore_lock(kill_jmh_processes())
|
|
if ignore_lock:
|
|
print(
|
|
"Warning: Unable to terminate existing JMH instance or lock file exists; using -Djmh.ignoreLock=true",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
try:
|
|
java_exec = resolve_executable(args.java, "Java")
|
|
except RuntimeError as exc:
|
|
print("Error: {}".format(exc), file=sys.stderr)
|
|
return 2
|
|
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
run_dir = prepare_run_dir(results_dir, timestamp, tag)
|
|
print("Run directory: {}".format(run_dir))
|
|
|
|
prof_gc = bool(getattr(args, "prof_gc", False))
|
|
|
|
for bench_type in selected_types:
|
|
bench_class = BENCH_TYPES[bench_type]
|
|
json_path, log_path = output_paths_for(run_dir, bench_type, prof_gc)
|
|
|
|
# overwrite if exists
|
|
try:
|
|
if json_path.exists():
|
|
json_path.unlink()
|
|
except OSError:
|
|
pass
|
|
try:
|
|
if log_path.exists():
|
|
log_path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
jmh_cmd = [java_exec]
|
|
if ignore_lock:
|
|
jmh_cmd.append("-Djmh.ignoreLock=true")
|
|
jmh_cmd.extend(["-jar", str(jar_path)])
|
|
|
|
if prof_gc:
|
|
jmh_cmd.extend(["-prof", "gc"])
|
|
|
|
if jmh_args:
|
|
jmh_cmd.extend(jmh_args)
|
|
|
|
jmh_cmd.append("^{}\\.".format(bench_class))
|
|
jmh_cmd.extend(["-rf", "json", "-rff", str(json_path)])
|
|
|
|
exit_code = run_subprocess_capture(
|
|
jmh_cmd, root, log_path, "JMH run failed ({})".format(bench_type)
|
|
)
|
|
if exit_code != 0:
|
|
return exit_code
|
|
|
|
# validate JSON
|
|
try:
|
|
with json_path.open("r", encoding="utf-8") as handle:
|
|
data = json.load(handle)
|
|
if not isinstance(data, list):
|
|
raise ValueError("JSON root is not an array")
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
print("Error: JSON could not be parsed: {}".format(exc), file=sys.stderr)
|
|
print("JSON: {}".format(json_path), file=sys.stderr)
|
|
print("Log : {}".format(log_path), file=sys.stderr)
|
|
return 2
|
|
|
|
size_bytes = json_path.stat().st_size
|
|
print("JMH run completed ({})".format(bench_type))
|
|
print("JSON: {}".format(json_path))
|
|
print("LOG : {}".format(log_path))
|
|
print("File size: {} bytes".format(size_bytes))
|
|
print("Benchmarks: {}".format(len(data)))
|
|
|
|
print("All selected benchmarks completed. Results folder: {}".format(run_dir))
|
|
return 0
|
|
|
|
|
|
def main(argv):
|
|
args = parse_args(argv)
|
|
if args is None:
|
|
return 2
|
|
if args.command == "run":
|
|
return run_benchmarks(args)
|
|
print("Unknown command: {}".format(args.command), file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|