1#!/usr/bin/env python3
2"""Tests for verify_claim_support.py CLI."""
3 
4import json
5import os
6import shutil
7import subprocess
8import sys
9import tempfile
10import unittest
11 
12SCRIPT = os.path.join(os.path.dirname(__file__), '..', 'scripts', 'verify_claim_support.py')
13 
14 
15def run_vcs(*args: str, expect_fail: bool = False) -> dict | str:
16    """Run verify_claim_support.py."""
17    result = subprocess.run(
18        [sys.executable, SCRIPT, *args],
19        capture_output=True, text=True,
20    )
21    if result.returncode != 0 and not expect_fail:
22        raise RuntimeError(f'Exit {result.returncode}: {result.stderr}\n{result.stdout}')
23    stdout = result.stdout.strip()
24    if stdout.startswith('{'):
25        return json.loads(stdout)
26    return stdout
27 
28 
29def write_jsonl(path: str, rows: list[dict]):
30    with open(path, 'w') as f:
31        for row in rows:
32            f.write(json.dumps(row) + '\n')
33 
34 
35class TestVerifySupported(unittest.TestCase):
36    """Claims with matching evidence should be supported."""
37 
38    def setUp(self):
39        self.tmpdir = tempfile.mkdtemp()
40        # Sources
41        write_jsonl(os.path.join(self.tmpdir, 'sources.jsonl'), [
42            {'source_id': 'src_quantum_001', 'title': 'Quantum Computing 2024'},
43        ])
44        # Evidence with clear overlap to the claim
45        write_jsonl(os.path.join(self.tmpdir, 'evidence.jsonl'), [
46            {
47                'evidence_id': 'ev_shor_001',
48                'source_id': 'src_quantum_001',
49                'quote': "Shor's algorithm can factor large integers exponentially faster than any known classical algorithm, threatening RSA-2048 encryption.",
50                'evidence_type': 'direct_quote',
51            },
52        ])
53        # Claim that matches the evidence
54        write_jsonl(os.path.join(self.tmpdir, 'claims.jsonl'), [
55            {
56                'claim_id': 'clm_factor_001',
57                'section_id': 'finding_1',
58                'text': "Shor's algorithm can factor large numbers exponentially faster than classical methods, threatening RSA-2048.",
59                'claim_type': 'factual',
60                'cited_source_ids': ['src_quantum_001'],
61                'evidence_ids': ['ev_shor_001'],
62                'support_status': 'unverified',
63            },
64        ])
65 
66    def tearDown(self):
67        shutil.rmtree(self.tmpdir, ignore_errors=True)
68 
69    def test_supported_claim(self):
70        out = run_vcs('verify', '--dir', self.tmpdir)
71        self.assertEqual(out['status'], 'pass')
72        self.assertEqual(out['factual_unsupported'], 0)
73 
74        # Check updated claims file
75        claims = []
76        with open(os.path.join(self.tmpdir, 'claims.jsonl')) as f:
77            for line in f:
78                claims.append(json.loads(line))
79        self.assertEqual(claims[0]['support_status'], 'supported')
80 
81 
82class TestVerifyUnsupported(unittest.TestCase):
83    """Claims without evidence should be unsupported."""
84 
85    def setUp(self):
86        self.tmpdir = tempfile.mkdtemp()
87        write_jsonl(os.path.join(self.tmpdir, 'sources.jsonl'), [])
88        write_jsonl(os.path.join(self.tmpdir, 'evidence.jsonl'), [])
89        write_jsonl(os.path.join(self.tmpdir, 'claims.jsonl'), [
90            {
91                'claim_id': 'clm_no_ev_001',
92                'section_id': 'finding_1',
93                'text': 'The population of Mars is 500 million as of 2025.',
94                'claim_type': 'factual',
95                'cited_source_ids': [],
96                'evidence_ids': [],
97                'support_status': 'unverified',
98            },
99        ])
100 
101    def tearDown(self):
102        shutil.rmtree(self.tmpdir, ignore_errors=True)
103 
104    def test_unsupported_no_evidence(self):
105        out = run_vcs('verify', '--dir', self.tmpdir)
106        self.assertEqual(out['factual_unsupported'], 1)
107        self.assertEqual(out['status'], 'pass')  # Non-strict by default
108 
109    def test_strict_fails(self):
110        out = run_vcs('verify', '--dir', self.tmpdir, '--strict', expect_fail=True)
111        self.assertEqual(out['status'], 'fail')
112 
113 
114class TestVerifyMixed(unittest.TestCase):
115    """Mixed claim types with different thresholds."""
116 
117    def setUp(self):
118        self.tmpdir = tempfile.mkdtemp()
119        write_jsonl(os.path.join(self.tmpdir, 'sources.jsonl'), [])
120        write_jsonl(os.path.join(self.tmpdir, 'evidence.jsonl'), [])
121        write_jsonl(os.path.join(self.tmpdir, 'claims.jsonl'), [
122            {
123                'claim_id': 'clm_spec_001',
124                'section_id': 'finding_1',
125                'text': 'Quantum computers might eventually solve protein folding in real time.',
126                'claim_type': 'speculation',
127                'cited_source_ids': [],
128                'evidence_ids': [],
129                'support_status': 'unverified',
130            },
131            {
132                'claim_id': 'clm_rec_001',
133                'section_id': 'recommendations',
134                'text': 'Organizations should begin PQC migration planning immediately.',
135                'claim_type': 'recommendation',
136                'cited_source_ids': [],
137                'evidence_ids': [],
138                'support_status': 'unverified',
139            },
140        ])
141 
142    def tearDown(self):
143        shutil.rmtree(self.tmpdir, ignore_errors=True)
144 
145    def test_speculation_passes(self):
146        out = run_vcs('verify', '--dir', self.tmpdir)
147        # Speculation doesn't need evidence
148        claims = []
149        with open(os.path.join(self.tmpdir, 'claims.jsonl')) as f:
150            for line in f:
151                claims.append(json.loads(line))
152        spec = [c for c in claims if c['claim_type'] == 'speculation'][0]
153        self.assertEqual(spec['support_status'], 'supported')
154 
155 
156class TestVerifyPartial(unittest.TestCase):
157    """Evidence with partial overlap should result in partial status."""
158 
159    def setUp(self):
160        self.tmpdir = tempfile.mkdtemp()
161        write_jsonl(os.path.join(self.tmpdir, 'sources.jsonl'), [
162            {'source_id': 'src_nist_001', 'title': 'NIST PQC Standards'},
163        ])
164        write_jsonl(os.path.join(self.tmpdir, 'evidence.jsonl'), [
165            {
166                'evidence_id': 'ev_nist_001',
167                'source_id': 'src_nist_001',
168                'quote': 'NIST announced the standardization of CRYSTALS-Kyber for key encapsulation.',
169                'evidence_type': 'direct_quote',
170            },
171        ])
172        # Claim mentions NIST but adds unverified detail about timeline
173        write_jsonl(os.path.join(self.tmpdir, 'claims.jsonl'), [
174            {
175                'claim_id': 'clm_nist_time',
176                'section_id': 'finding_2',
177                'text': 'NIST standardized four lattice-based algorithms in 2024, covering both encryption and signatures.',
178                'claim_type': 'factual',
179                'cited_source_ids': ['src_nist_001'],
180                'evidence_ids': ['ev_nist_001'],
181                'support_status': 'unverified',
182            },
183        ])
184 
185    def tearDown(self):
186        shutil.rmtree(self.tmpdir, ignore_errors=True)
187 
188    def test_partial_support(self):
189        out = run_vcs('verify', '--dir', self.tmpdir)
190        claims = []
191        with open(os.path.join(self.tmpdir, 'claims.jsonl')) as f:
192            for line in f:
193                claims.append(json.loads(line))
194        # Should be partial or needs_review (not fully supported due to number/detail mismatch)
195        self.assertIn(claims[0]['support_status'], ('partial', 'needs_review', 'supported'))
196 
197 
198class TestSupportScore(unittest.TestCase):
199    """Unit tests for compute_support_score."""
200 
201    @classmethod
202    def setUpClass(cls):
203        sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts'))
204        from verify_claim_support import compute_support_score
205        cls.score = staticmethod(compute_support_score)
206 
207    def test_identical_text(self):
208        status, score, _ = self.score(
209            'RSA-2048 uses 2048-bit keys for encryption.',
210            ['RSA-2048 uses 2048-bit keys for encryption.'],
211        )
212        self.assertEqual(status, 'supported')
213        self.assertGreater(score, 0.8)
214 
215    def test_no_evidence(self):
216        status, score, _ = self.score('Any claim text.', [])
217        self.assertEqual(status, 'unsupported')
218        self.assertEqual(score, 0.0)
219 
220    def test_unrelated_evidence(self):
221        status, score, _ = self.score(
222            'The moon landing occurred in 1969.',
223            ['Bananas are a good source of potassium and fiber.'],
224        )
225        self.assertIn(status, ('needs_review', 'unsupported'))
226        self.assertLess(score, 0.35)
227 
228 
229if __name__ == '__main__':
230    unittest.main()
231