Add reproducible evaluation and provenance research

This commit is contained in:
2026-09-17 10:40:29 -05:00
parent d9fc3ca18b
commit 68f02d035f
53 changed files with 1432 additions and 14 deletions
+16
View File
@@ -28,6 +28,22 @@ class CliTests(unittest.TestCase):
with redirect_stderr(StringIO()):
self.assertEqual(main(["scan", "/definitely/missing/aaisa-path"]), 2)
def test_rule_catalog_and_explanation_are_json(self) -> None:
output = StringIO()
with redirect_stdout(output):
self.assertEqual(main(["rules", "--format", "json"]), 0)
catalog = json.loads(output.getvalue())
self.assertGreaterEqual(len(catalog["rules"]), 8)
output = StringIO()
with redirect_stdout(output):
self.assertEqual(main(["explain", "AAISA-AGENT-001", "--format", "json"]), 0)
self.assertTrue(any("OWASP Agentic AI" in item for item in json.loads(output.getvalue())["standards"]))
def test_evaluation_cli_returns_zero_for_passing_corpus(self) -> None:
manifest = Path(__file__).parents[1] / "research" / "corpus-v1" / "manifest.json"
with redirect_stdout(StringIO()):
self.assertEqual(main(["evaluate", str(manifest)]), 0)
if __name__ == "__main__":
unittest.main()