Add reproducible evaluation and provenance research

2026-09-17 10:40:29 -05:00
parent d9fc3ca18b
commit 68f02d035f
53 changed files with 1432 additions and 14 deletions
@@ -0,0 +1,67 @@
+{
+  "$schema": "../../schemas/aaisa-corpus.schema.json",
+  "schema_version": "1.0.0",
+  "name": "AAISA deterministic foundation corpus v1",
+  "cases": [
+    {
+      "id": "prompt-injection-direct",
+      "artifact": "fixtures/injection.prompt",
+      "expected_findings": [{"rule_id": "AAISA-PROMPT-001", "count": 1}]
+    },
+    {
+      "id": "prompt-benign",
+      "artifact": "fixtures/benign.prompt",
+      "expected_findings": []
+    },
+    {
+      "id": "dependency-unpinned",
+      "artifact": "fixtures/unpinned/requirements.txt",
+      "expected_findings": [{"rule_id": "AAISA-DEPENDENCY-001", "count": 1}]
+    },
+    {
+      "id": "dependency-pinned",
+      "artifact": "fixtures/pinned/requirements.txt",
+      "expected_findings": []
+    },
+    {
+      "id": "container-mutable",
+      "artifact": "fixtures/mutable/Dockerfile",
+      "expected_findings": [{"rule_id": "AAISA-CONTAINER-001", "count": 1}]
+    },
+    {
+      "id": "container-immutable",
+      "artifact": "fixtures/immutable/Dockerfile",
+      "expected_findings": []
+    },
+    {
+      "id": "agent-overprivileged",
+      "artifact": "fixtures/unsafe-agent.json",
+      "expected_findings": [{"rule_id": "AAISA-AGENT-001", "count": 3}]
+    },
+    {
+      "id": "agent-constrained",
+      "artifact": "fixtures/safe-agent.json",
+      "expected_findings": []
+    },
+    {
+      "id": "slsa-incomplete",
+      "artifact": "fixtures/incomplete-provenance.json",
+      "expected_findings": [{"rule_id": "AAISA-PROVENANCE-002", "count": 1}]
+    },
+    {
+      "id": "slsa-complete",
+      "artifact": "fixtures/complete-provenance.json",
+      "expected_findings": []
+    },
+    {
+      "id": "mlbom-incomplete",
+      "artifact": "fixtures/incomplete-ml-bom.json",
+      "expected_findings": [{"rule_id": "AAISA-MLBOM-001", "count": 1}]
+    },
+    {
+      "id": "mlbom-complete",
+      "artifact": "fixtures/complete-ml-bom.json",
+      "expected_findings": []
+    }
+  ]
+}