Add reproducible evaluation and provenance research
This commit is contained in:
@@ -28,6 +28,22 @@ class CliTests(unittest.TestCase):
|
||||
with redirect_stderr(StringIO()):
|
||||
self.assertEqual(main(["scan", "/definitely/missing/aaisa-path"]), 2)
|
||||
|
||||
def test_rule_catalog_and_explanation_are_json(self) -> None:
|
||||
output = StringIO()
|
||||
with redirect_stdout(output):
|
||||
self.assertEqual(main(["rules", "--format", "json"]), 0)
|
||||
catalog = json.loads(output.getvalue())
|
||||
self.assertGreaterEqual(len(catalog["rules"]), 8)
|
||||
output = StringIO()
|
||||
with redirect_stdout(output):
|
||||
self.assertEqual(main(["explain", "AAISA-AGENT-001", "--format", "json"]), 0)
|
||||
self.assertTrue(any("OWASP Agentic AI" in item for item in json.loads(output.getvalue())["standards"]))
|
||||
|
||||
def test_evaluation_cli_returns_zero_for_passing_corpus(self) -> None:
|
||||
manifest = Path(__file__).parents[1] / "research" / "corpus-v1" / "manifest.json"
|
||||
with redirect_stdout(StringIO()):
|
||||
self.assertEqual(main(["evaluate", str(manifest)]), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user