1import importlib.util
2import unittest
3from pathlib import Path
4 
5 
6MODULE_PATH = (
7    Path(__file__).resolve().parents[1] / "scripts" / "compression_evaluator.py"
8)
9MODULE_SPEC = importlib.util.spec_from_file_location(
10    "compression_evaluator", MODULE_PATH
11)
12if MODULE_SPEC is None or MODULE_SPEC.loader is None:
13    raise RuntimeError(f"Unable to load compression_evaluator.py from {MODULE_PATH}")
14COMPRESSION_EVALUATOR = importlib.util.module_from_spec(MODULE_SPEC)
15MODULE_SPEC.loader.exec_module(COMPRESSION_EVALUATOR)
16 
17 
18class CompressionEvaluatorTests(unittest.TestCase):
19    def test_json_ground_truth_terms_score_when_response_mentions_artifacts(
20        self,
21    ) -> None:
22        evaluator = COMPRESSION_EVALUATOR.CompressionEvaluator()
23 
24        rich_score = evaluator._heuristic_score(
25            {"id": "artifact_files_modified"},
26            "We modified src/app.py and updated README.md during the session.",
27            '[{"path": "src/app.py", "operation": "modified"}, {"path": "README.md", "operation": "updated"}]',
28        )
29        poor_score = evaluator._heuristic_score(
30            {"id": "artifact_files_modified"},
31            "We changed some files but I do not remember which ones.",
32            '[{"path": "src/app.py", "operation": "modified"}, {"path": "README.md", "operation": "updated"}]',
33        )
34 
35        self.assertGreater(rich_score, poor_score)
36        self.assertGreaterEqual(rich_score, 4.0)
37 
38    def test_plain_text_ground_truth_still_uses_substring_match(self) -> None:
39        evaluator = COMPRESSION_EVALUATOR.CompressionEvaluator()
40 
41        exact_score = evaluator._heuristic_score(
42            {"id": "continuity_work_state"},
43            "Next: fix the websocket timeout before rerunning tests.",
44            "fix the websocket timeout",
45        )
46        missing_score = evaluator._heuristic_score(
47            {"id": "continuity_work_state"},
48            "Next: inspect logs again.",
49            "fix the websocket timeout",
50        )
51 
52        self.assertGreater(exact_score, missing_score)
53 
54 
55if __name__ == "__main__":
56    unittest.main()
57