Add reproducible evaluation and provenance research

tests/test_evaluation.py

@@ -0,0 +1,64 @@
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+from aaisa.evaluation import evaluate_manifest
+
+
+class EvaluationTests(unittest.TestCase):
+    def test_research_corpus_is_reproducible_and_passing(self) -> None:
+        manifest = Path(__file__).parents[1] / "research" / "corpus-v1" / "manifest.json"
+        first = evaluate_manifest(manifest)
+        second = evaluate_manifest(manifest)
+        self.assertEqual(first, second)
+        self.assertEqual(first["summary"]["failed"], 0)
+        self.assertEqual(first["corpus"]["case_count"], 12)
+        self.assertTrue(first["corpus"]["digest"].startswith("sha256:"))
+
+    def test_manifest_cannot_escape_corpus_root(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            outside = root.parent / "aaisa-outside-fixture.txt"
+            outside.write_text("safe", encoding="utf-8")
+            manifest = root / "manifest.json"
+            manifest.write_text(json.dumps({
+                "schema_version": "1.0.0",
+                "cases": [{"id": "escape", "artifact": "../aaisa-outside-fixture.txt", "expected_findings": []}],
+            }), encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "escapes the corpus"):
+                evaluate_manifest(manifest)
+
+    def test_manifest_rejects_symlinked_fixture(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            target = root / "target.txt"
+            target.write_text("safe", encoding="utf-8")
+            link = root / "linked.txt"
+            try:
+                link.symlink_to(target)
+            except OSError:
+                self.skipTest("symbolic links are not available")
+            manifest = root / "manifest.json"
+            manifest.write_text(json.dumps({
+                "schema_version": "1.0.0",
+                "cases": [{"id": "linked", "artifact": "linked.txt", "expected_findings": []}],
+            }), encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "escapes the corpus"):
+                evaluate_manifest(manifest)
+
+    def test_unknown_expected_rule_is_rejected(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "safe.txt").write_text("safe", encoding="utf-8")
+            manifest = root / "manifest.json"
+            manifest.write_text(json.dumps({
+                "schema_version": "1.0.0",
+                "cases": [{"id": "unknown", "artifact": "safe.txt", "expected_findings": [{"rule_id": "UNKNOWN"}]}],
+            }), encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "unknown rule"):
+                evaluate_manifest(manifest)
+
+
+if __name__ == "__main__":
+    unittest.main()