65 lines
2.8 KiB
Python
65 lines
2.8 KiB
Python
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
from aaisa.evaluation import evaluate_manifest
|
|
|
|
|
|
class EvaluationTests(unittest.TestCase):
|
|
def test_research_corpus_is_reproducible_and_passing(self) -> None:
|
|
manifest = Path(__file__).parents[1] / "research" / "corpus-v1" / "manifest.json"
|
|
first = evaluate_manifest(manifest)
|
|
second = evaluate_manifest(manifest)
|
|
self.assertEqual(first, second)
|
|
self.assertEqual(first["summary"]["failed"], 0)
|
|
self.assertEqual(first["corpus"]["case_count"], 12)
|
|
self.assertTrue(first["corpus"]["digest"].startswith("sha256:"))
|
|
|
|
def test_manifest_cannot_escape_corpus_root(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
outside = root.parent / "aaisa-outside-fixture.txt"
|
|
outside.write_text("safe", encoding="utf-8")
|
|
manifest = root / "manifest.json"
|
|
manifest.write_text(json.dumps({
|
|
"schema_version": "1.0.0",
|
|
"cases": [{"id": "escape", "artifact": "../aaisa-outside-fixture.txt", "expected_findings": []}],
|
|
}), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "escapes the corpus"):
|
|
evaluate_manifest(manifest)
|
|
|
|
def test_manifest_rejects_symlinked_fixture(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
target = root / "target.txt"
|
|
target.write_text("safe", encoding="utf-8")
|
|
link = root / "linked.txt"
|
|
try:
|
|
link.symlink_to(target)
|
|
except OSError:
|
|
self.skipTest("symbolic links are not available")
|
|
manifest = root / "manifest.json"
|
|
manifest.write_text(json.dumps({
|
|
"schema_version": "1.0.0",
|
|
"cases": [{"id": "linked", "artifact": "linked.txt", "expected_findings": []}],
|
|
}), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "escapes the corpus"):
|
|
evaluate_manifest(manifest)
|
|
|
|
def test_unknown_expected_rule_is_rejected(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
(root / "safe.txt").write_text("safe", encoding="utf-8")
|
|
manifest = root / "manifest.json"
|
|
manifest.write_text(json.dumps({
|
|
"schema_version": "1.0.0",
|
|
"cases": [{"id": "unknown", "artifact": "safe.txt", "expected_findings": [{"rule_id": "UNKNOWN"}]}],
|
|
}), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "unknown rule"):
|
|
evaluate_manifest(manifest)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|