Add reproducible evaluation and provenance research

2026-09-17 10:40:29 -05:00
parent d9fc3ca18b
commit 68f02d035f
53 changed files with 1432 additions and 14 deletions
@@ -0,0 +1,68 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://dtechtechresearch.com/schemas/aaisa-evaluation-1.0.0.json",
+  "title": "AAISA Evaluation Result",
+  "type": "object",
+  "required": ["schema_version", "tool", "corpus", "summary", "metrics", "cases"],
+  "properties": {
+    "schema_version": {"const": "1.0.0"},
+    "tool": {
+      "type": "object",
+      "required": ["name", "version"],
+      "properties": {"name": {"const": "AAISA"}, "version": {"type": "string"}},
+      "additionalProperties": false
+    },
+    "corpus": {
+      "type": "object",
+      "required": ["name", "digest", "case_count"],
+      "properties": {
+        "name": {"type": "string"},
+        "digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
+        "case_count": {"type": "integer", "minimum": 1}
+      },
+      "additionalProperties": false
+    },
+    "summary": {
+      "type": "object",
+      "required": ["passed", "failed"],
+      "properties": {
+        "passed": {"type": "integer", "minimum": 0},
+        "failed": {"type": "integer", "minimum": 0}
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": {"$ref": "#/$defs/metric"}
+    },
+    "cases": {"type": "array", "items": {"$ref": "#/$defs/case"}}
+  },
+  "$defs": {
+    "metric": {
+      "type": "object",
+      "required": ["true_positive", "false_positive", "false_negative", "precision", "recall"],
+      "properties": {
+        "true_positive": {"type": "integer", "minimum": 0},
+        "false_positive": {"type": "integer", "minimum": 0},
+        "false_negative": {"type": "integer", "minimum": 0},
+        "precision": {"type": ["number", "null"], "minimum": 0, "maximum": 1},
+        "recall": {"type": ["number", "null"], "minimum": 0, "maximum": 1}
+      },
+      "additionalProperties": false
+    },
+    "case": {
+      "type": "object",
+      "required": ["id", "artifact", "artifact_sha256", "expected", "actual", "errors", "passed"],
+      "properties": {
+        "id": {"type": "string"},
+        "artifact": {"type": "string"},
+        "artifact_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"},
+        "expected": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 1}},
+        "actual": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 1}},
+        "errors": {"type": "array"},
+        "passed": {"type": "boolean"}
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}