Add reproducible evaluation and provenance research

scripts/verify.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def run(*arguments: str) -> None:
+    command = [sys.executable, *arguments]
+    print("+", " ".join(command), flush=True)
+    subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, check=True)
+
+
+ENVIRONMENT = os.environ.copy()
+ENVIRONMENT["PYTHONPATH"] = str(ROOT / "src")
+
+
+def main() -> int:
+    with tempfile.TemporaryDirectory(prefix="aaisa-verify-") as directory:
+        output = Path(directory)
+        run("-m", "compileall", "-q", "src", "tests")
+        run("-m", "unittest", "discover", "-s", "tests", "-v")
+        run("-m", "aaisa", "rules", "--format", "json")
+        run(
+            "-m", "aaisa", "evaluate", "research/corpus-v1/manifest.json",
+            "--output", str(output / "evaluation.json"),
+        )
+        run(
+            "-m", "aaisa", "scan", "examples", "--format", "json",
+            "--output", str(output / "examples.json"),
+        )
+    print("AAISA verification passed", flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())