f86d1109f5df
Review benchmark validity and add upstream-inspi
11 days ago
| 1 | #!/usr/bin/env python3
|
| 2 | """Run local VCS correctness diagnostics; stdout is diagnostic JSONL, not timing rows."""
|
| 3 | import argparse
|
| 4 | import json
|
| 5 | from pathlib import Path
|
| 6 |
|
| 7 | from oakbench.conformance import SCENARIOS, run_scenario
|
| 8 |
|
| 9 |
|
| 10 | def main():
|
| 11 | parser = argparse.ArgumentParser(description=__doc__)
|
| 12 | parser.add_argument("--workdir", type=Path, required=True)
|
| 13 | parser.add_argument("--subjects", default="git,oak")
|
| 14 | parser.add_argument("--git-bin", default="git")
|
| 15 | parser.add_argument("--oak-bin", default="oak")
|
| 16 | parser.add_argument("--scenarios", default=",".join(SCENARIOS))
|
| 17 | args = parser.parse_args()
|
| 18 | subjects = args.subjects.split(",")
|
| 19 | scenarios = args.scenarios.split(",")
|
| 20 | if not set(subjects) <= {"git", "oak"} or not set(scenarios) <= set(SCENARIOS):
|
| 21 | parser.error("unknown subject or scenario")
|
| 22 | failed = False
|
| 23 | for subject in subjects:
|
| 24 | for scenario in scenarios:
|
| 25 | row = run_scenario(args.workdir.resolve(), subject, getattr(args, f"{subject}_bin"), scenario)
|
| 26 | print(json.dumps(row, sort_keys=True), flush=True)
|
| 27 | failed |= row["returncode"] == 1
|
| 28 | return int(failed)
|
| 29 |
|
| 30 |
|
| 31 | if __name__ == "__main__":
|
| 32 | raise SystemExit(main())
|