Add reproducible evaluation and provenance research

2026-09-17 10:40:29 -05:00
parent d9fc3ca18b
commit 68f02d035f
53 changed files with 1432 additions and 14 deletions
@@ -0,0 +1,37 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://dtechtechresearch.com/schemas/aaisa-corpus-1.0.0.json",
+  "title": "AAISA Evaluation Corpus Manifest",
+  "type": "object",
+  "required": ["schema_version", "name", "cases"],
+  "properties": {
+    "schema_version": {"const": "1.0.0"},
+    "name": {"type": "string", "minLength": 1},
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": ["id", "artifact", "expected_findings"],
+        "properties": {
+          "id": {"type": "string", "minLength": 1},
+          "artifact": {"type": "string", "minLength": 1},
+          "expected_findings": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "required": ["rule_id"],
+              "properties": {
+                "rule_id": {"type": "string", "pattern": "^AAISA-[A-Z]+-[0-9]{3}$"},
+                "count": {"type": "integer", "minimum": 1}
+              },
+              "additionalProperties": false
+            }
+          }
+        },
+        "additionalProperties": false
+      }
+    }
+  },
+  "additionalProperties": true
+}