dTechMediaCreations/advanced-artifactual-intelligence-security-agent
2
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add reproducible evaluation and provenance research

Browse Source
This commit is contained in:
duaneAI
2026-09-17 10:40:29 -05:00
parent d9fc3ca18b
commit 68f02d035f
53 changed files with 1432 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+16
View File
@@ -28,6 +28,22 @@ class CliTests(unittest.TestCase):
with redirect_stderr(StringIO()):
self.assertEqual(main(["scan", "/definitely/missing/aaisa-path"]), 2)
def test_rule_catalog_and_explanation_are_json(self) -> None:
output = StringIO()
with redirect_stdout(output):
self.assertEqual(main(["rules", "--format", "json"]), 0)
catalog = json.loads(output.getvalue())
self.assertGreaterEqual(len(catalog["rules"]), 8)
output = StringIO()
with redirect_stdout(output):
self.assertEqual(main(["explain", "AAISA-AGENT-001", "--format", "json"]), 0)
self.assertTrue(any("OWASP Agentic AI" in item for item in json.loads(output.getvalue())["standards"]))
def test_evaluation_cli_returns_zero_for_passing_corpus(self) -> None:
manifest = Path(__file__).parents[1] / "research" / "corpus-v1" / "manifest.json"
with redirect_stdout(StringIO()):
self.assertEqual(main(["evaluate", str(manifest)]), 0)
if __name__ == "__main__":
unittest.main()
tests/test_evaluation.py
+64
View File
@@ -0,0 +1,64 @@
import json
from pathlib import Path
import tempfile
import unittest
from aaisa.evaluation import evaluate_manifest
class EvaluationTests(unittest.TestCase):
def test_research_corpus_is_reproducible_and_passing(self) -> None:
manifest = Path(__file__).parents[1] / "research" / "corpus-v1" / "manifest.json"
first = evaluate_manifest(manifest)
second = evaluate_manifest(manifest)
self.assertEqual(first, second)
self.assertEqual(first["summary"]["failed"], 0)
self.assertEqual(first["corpus"]["case_count"], 12)
self.assertTrue(first["corpus"]["digest"].startswith("sha256:"))
def test_manifest_cannot_escape_corpus_root(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
outside = root.parent / "aaisa-outside-fixture.txt"
outside.write_text("safe", encoding="utf-8")
manifest = root / "manifest.json"
manifest.write_text(json.dumps({
"schema_version": "1.0.0",
"cases": [{"id": "escape", "artifact": "../aaisa-outside-fixture.txt", "expected_findings": []}],
}), encoding="utf-8")
with self.assertRaisesRegex(ValueError, "escapes the corpus"):
evaluate_manifest(manifest)
def test_manifest_rejects_symlinked_fixture(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
target = root / "target.txt"
target.write_text("safe", encoding="utf-8")
link = root / "linked.txt"
try:
link.symlink_to(target)
except OSError:
self.skipTest("symbolic links are not available")
manifest = root / "manifest.json"
manifest.write_text(json.dumps({
"schema_version": "1.0.0",
"cases": [{"id": "linked", "artifact": "linked.txt", "expected_findings": []}],
}), encoding="utf-8")
with self.assertRaisesRegex(ValueError, "escapes the corpus"):
evaluate_manifest(manifest)
def test_unknown_expected_rule_is_rejected(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "safe.txt").write_text("safe", encoding="utf-8")
manifest = root / "manifest.json"
manifest.write_text(json.dumps({
"schema_version": "1.0.0",
"cases": [{"id": "unknown", "artifact": "safe.txt", "expected_findings": [{"rule_id": "UNKNOWN"}]}],
}), encoding="utf-8")
with self.assertRaisesRegex(ValueError, "unknown rule"):
evaluate_manifest(manifest)
if __name__ == "__main__":
unittest.main()
tests/test_scanner.py
+2
View File
@@ -43,8 +43,10 @@ class ScannerTests(unittest.TestCase):
json_payload = json.loads(json_report(report))
sarif_payload = json.loads(sarif_report(report))
self.assertEqual(json_payload["schema_version"], "1.0.0")
self.assertEqual(len(json_payload["findings"][0]["fingerprint"]), 64)
self.assertEqual(sarif_payload["version"], "2.1.0")
self.assertEqual(sarif_payload["runs"][0]["results"][0]["ruleId"], "AAISA-DEPENDENCY-001")
self.assertEqual(len(sarif_payload["runs"][0]["results"][0]["partialFingerprints"]["aaisa/v1"]), 64)
if __name__ == "__main__":
tests/test_structured_rules.py
+80
View File
@@ -0,0 +1,80 @@
from pathlib import Path
import json
import unittest
from aaisa.models import Artifact
from aaisa.structured_rules import AgentToolPermissionRule, CycloneDxMLBomRule, SlsaProvenanceRule
def json_artifact(name: str, payload: object, kind: str) -> Artifact:
text = json.dumps(payload)
return Artifact(Path(name), name, kind, "application/json", len(text), "0" * 64, text)
class StructuredRuleTests(unittest.TestCase):
def test_unsafe_agent_tool_reports_independent_controls(self) -> None:
payload = {"tools": [{
"name": "shell-exec",
"permissions": ["*"],
"allow_network": True,
"allowed_hosts": [],
"requires_approval": False,
}]}
findings = list(AgentToolPermissionRule().evaluate(json_artifact("agent.json", payload, "agent-manifest")))
self.assertEqual(len(findings), 3)
self.assertEqual({finding.severity.label() for finding in findings}, {"critical", "high"})
def test_constrained_agent_tool_is_accepted(self) -> None:
payload = {"tools": [{
"name": "shell-exec",
"permissions": ["workspace:read"],
"allow_network": True,
"allowed_hosts": ["packages.example.invalid"],
"requires_approval": True,
}]}
self.assertFalse(list(AgentToolPermissionRule().evaluate(json_artifact("agent.json", payload, "agent-manifest"))))
def test_incomplete_slsa_provenance_is_reported(self) -> None:
payload = {"_type": "https://in-toto.io/Statement/v1", "predicateType": "https://slsa.dev/provenance/v1", "predicate": {}}
finding = list(SlsaProvenanceRule().evaluate(json_artifact("provenance.json", payload, "provenance")))[0]
self.assertIn("subject SHA-256 digest", finding.evidence or "")
self.assertIn("runDetails.builder.id", finding.evidence or "")
def test_complete_slsa_provenance_is_accepted(self) -> None:
payload = {
"_type": "https://in-toto.io/Statement/v1",
"subject": [{"name": "model.bin", "digest": {"sha256": "a" * 64}}],
"predicateType": "https://slsa.dev/provenance/v1",
"predicate": {
"buildDefinition": {
"buildType": "https://example.invalid/build/model/v1",
"externalParameters": {},
"resolvedDependencies": [{"uri": "git+https://example.invalid/repo@v1"}],
},
"runDetails": {"builder": {"id": "https://example.invalid/builders/research"}},
},
}
self.assertFalse(list(SlsaProvenanceRule().evaluate(json_artifact("provenance.json", payload, "provenance"))))
def test_non_hex_digest_is_not_accepted(self) -> None:
payload = {
"_type": "https://in-toto.io/Statement/v1",
"subject": [{"name": "model.bin", "digest": {"sha256": "z" * 64}}],
"predicateType": "https://slsa.dev/provenance/v1",
"predicate": {
"buildDefinition": {"buildType": "example", "externalParameters": {}, "resolvedDependencies": [{}]},
"runDetails": {"builder": {"id": "example"}},
},
}
finding = list(SlsaProvenanceRule().evaluate(json_artifact("provenance.json", payload, "provenance")))[0]
self.assertIn("subject SHA-256 digest", finding.evidence or "")
def test_ml_bom_requires_transparency_and_integrity(self) -> None:
payload = {"bomFormat": "CycloneDX", "specVersion": "1.7", "components": [{"type": "machine-learning-model", "name": "example"}]}
finding = list(CycloneDxMLBomRule().evaluate(json_artifact("ml-bom.json", payload, "ml-bom")))[0]
self.assertIn("modelCard", finding.evidence or "")
self.assertIn("SHA-256", finding.evidence or "")
if __name__ == "__main__":
unittest.main()