test_match_runner.py

1"""Offline fixtures only. No checkpoint loads, models, engines or actual matches."""
2import copy,io,json,os,signal,subprocess,sys,tempfile,time,unittest,uuid
3from pathlib import Path
4from types import SimpleNamespace
5from unittest.mock import patch
6import match_runner as r
7import chess,chess.pgn
8
9class Clock:
10    def __init__(self,stop=False):self.stop=stop
11    def soft_stop(self):return self.stop
12    def remaining(self):return 7000
13
14class FixtureHarness:
15    def __init__(self,fail=False):self.calls=[];self.fail=fail
16    def game(self,model,engine,color,out,**kw):
17        self.calls.append((color,kw))
18        if self.fail:raise RuntimeError('synthetic failure')
19        out.mkdir();game=chess.pgn.Game();node=game;board=chess.Board()
20        for uci in kw['opening']:
21            move=chess.Move.from_uci(uci);board.push(move);node=node.add_variation(move)
22        game.headers['Result']='*';(out/'game.pgn').write_text(str(game)+'\n')
23        result={'result':'*','termination':'offline_prefix_fixture','error':None,'cleanup_error':None,
24                'plies':board.ply(),'final_fen':board.fen(),'moves':[],'seconds':0.0}
25        (out/'game.json').write_text(json.dumps(result));return result
26
27class Tests(unittest.TestCase):
28    @classmethod
29    def setUpClass(cls):cls.suite,_=r.inputs()
30    def test_all100_variable_prefixes_schedule_and_cleanup(self):
31        closed=[];seeds=[];h=FixtureHarness()
32        def factory(adapter,checkpoint,seed,timeout):
33            seeds.append(seed);return SimpleNamespace(close=lambda:closed.append(seed))
34        with tempfile.TemporaryDirectory() as t:
35            rows=r.play(self.suite,Path(t),Path('unused'),h,Clock(),factory)
36            summary=json.loads((Path(t)/'summary.json').read_text())
37            self.assertEqual(len(rows),100);self.assertEqual(len(closed),100)
38            self.assertEqual(seeds,[g['seed'] for g in self.suite['schedule']])
39            self.assertEqual({len(call[1]['opening']) for call in h.calls},{2,4,6,20})
40            self.assertEqual(summary['unfinished'],100);self.assertEqual(summary['model_queries'],0)
41            self.assertIsNone(summary['score_on_scored_games'])
42            self.assertEqual(summary['unattempted_or_unaudited'],0)
43    def test_exception_and_softstop_cleanup(self):
44        closed=[]
45        def factory(*args):return SimpleNamespace(close=lambda:closed.append(True))
46        with tempfile.TemporaryDirectory() as t:
47            with self.assertRaisesRegex(RuntimeError,'synthetic failure'):r.play(self.suite,Path(t),Path('unused'),FixtureHarness(True),Clock(),factory)
48            self.assertEqual(closed,[True])
49            self.assertEqual(json.loads((Path(t)/'summary.json').read_text())['audited_games'],0)
50        with tempfile.TemporaryDirectory() as t:
51            rows=r.play(self.suite,Path(t),Path('unused'),FixtureHarness(),Clock(True),lambda *a:self.fail('model started after soft stop'))
52            self.assertEqual(rows,[])
53    def test_audit_rejects_book_metrics_and_fake_scores(self):
54        opening=self.suite['openings'][0];h=FixtureHarness()
55        with tempfile.TemporaryDirectory() as t:
56            folder=Path(t)/'game';report=h.game(None,None,True,folder,opening=opening['uci'])
57            bad=copy.deepcopy(report);bad['moves']=[{'ply':1,'uci':opening['uci'][0]}]
58            with self.assertRaisesRegex(ValueError,'book moves'):r.audit(h,folder,bad,opening,True)
59            game=chess.pgn.read_game(io.StringIO((folder/'game.pgn').read_text()));game.headers['Result']='1-0'
60            (folder/'game.pgn').write_text(str(game));bad=copy.deepcopy(report);bad['result']='1-0'
61            with self.assertRaisesRegex(ValueError,'fabricated'):r.audit(h,folder,bad,opening,True)
62    def test_incomplete_never_scores_and_pair_uncertainty(self):
63        rows=[{'result':'1-0','model_color':'white','pair_id':'a','model_queries':2,'helper_overrides':1},
64              {'result':'0-1','model_color':'black','pair_id':'a','model_queries':2,'helper_overrides':0},
65              {'result':'*','model_color':'white','pair_id':'b','model_queries':0,'helper_overrides':0}]
66        summary=r.summarize(rows)
67        self.assertEqual((summary['wins'],summary['losses'],summary['draws'],summary['unfinished']),(2,0,0,1))
68        self.assertEqual(summary['fully_scored_pairs'],1);self.assertIsNone(summary['pair_standard_error'])
69        self.assertEqual(summary['score_on_scored_games'],1.)
70    def test_plan_only_never_launches(self):
71        with patch.object(sys,'argv',['runner']),patch.object(r,'launch',side_effect=AssertionError('launched')):
72            with patch('sys.stdout',new=io.StringIO()) as output:r.main()
73            self.assertEqual(json.loads(output.getvalue())['status'],'plan_only')
74    def comparison_fixture(self):
75        _,contract=r.inputs()
76        base={'request':{'role':'baseline','suite_sha256':contract['suite_sha256'],
77            'checkpoint_sha256':self.suite['protocol']['baseline_checkpoint_sha256'],
78            **{k:'same' for k in ('execution_contract_sha256','runner_sha256','guardian_sha256','device','budget_seconds')}},
79            **{k:'same' for k in ('protocol','model_config','python','torch','transformers','python_chess')}}
80        candidate=copy.deepcopy(base);candidate['request']['role']='candidate'
81        rows=[dict(g,result='1-0',error=None,cleanup_error=None,pgn_and_proofs_verified=True,model_queries=0,helper_overrides=0) for g in self.suite['schedule']]
82        better=copy.deepcopy(rows)
83        for row in better:row['result']='1-0' if row['model_color']=='white' else '0-1'
84        return base,candidate,rows,better
85    def test_matched_comparison_and_unfinished_gate(self):
86        base,candidate,rows,better=self.comparison_fixture()
87        result=r.compare(base,candidate,rows,better,self.suite)
88        self.assertEqual(result['candidate_minus_baseline_score'],.5)
89        self.assertTrue(result['evidence_for_improvement']);self.assertFalse(result['promotion_authorized'])
90        better[0]['result']='*';result=r.compare(base,candidate,rows,better,self.suite)
91        self.assertFalse(result['evidence_for_improvement']);self.assertFalse(result['all_games_scored_without_errors'])
92    def test_comparison_rejects_protocol_and_schedule_changes(self):
93        base,candidate,rows,better=self.comparison_fixture()
94        candidate['protocol']='different'
95        with self.assertRaisesRegex(ValueError,'protocol'):r.compare(base,candidate,rows,better,self.suite)
96        candidate['protocol']=base['protocol'];better[0]['seed']+=1
97        with self.assertRaisesRegex(ValueError,'schedule'):r.compare(base,candidate,rows,better,self.suite)
98    def test_comparison_replays_artifacts_and_rejects_summary_tampering(self):
99        base,candidate,_,_=self.comparison_fixture()
100        _,contract=r.inputs()
101        expected={'suite_sha256':contract['suite_sha256'],'runner_sha256':r.sha(r.__file__),
102                  'execution_contract_sha256':r.sha(r.ROOT/'execution-contract.json'),
103                  'guardian_sha256':r.sha(r.ROOT.parent/'runtime/training_deadline.py')}
104        with tempfile.TemporaryDirectory() as t:
105            paths=[Path(t)/'base',Path(t)/'candidate']
106            for path,manifest in zip(paths,(base,candidate)):
107                path.mkdir();manifest['request'].update(expected);manifest['protocol']=self.suite['protocol']
108                r.play(self.suite,path,Path('unused'),FixtureHarness(),Clock(),lambda *a:SimpleNamespace(close=lambda:None))
109                r.write(path/'manifest.json',manifest)
110            with patch.object(r,'verify_sources'):
111                result=r.compare_directories(*paths)
112                self.assertFalse(result['all_games_scored_without_errors'])
113                rows=json.loads((paths[1]/'results.json').read_text());rows[0]['result']='1-0'
114                r.write(paths[1]/'results.json',rows)
115                with self.assertRaisesRegex(ValueError,'stored result differs'):r.compare_directories(*paths)
116    def test_engine_handshakes_share_remaining_game_budget(self):
117        limits=[]
118        class H(FixtureHarness):
119            def UCI(self,command,timeout):limits.append(timeout)
120            def engine_command(self):return ['/unused-fixture']
121            def game(self,model,engine,color,out,**kw):
122                engine();return super().game(model,engine,color,out,**kw)
123        clock=Clock();clock.remaining=lambda:6
124        suite=copy.deepcopy(self.suite);suite['schedule']=suite['schedule'][:1]
125        with tempfile.TemporaryDirectory() as t:
126            r.play(suite,Path(t),Path('unused'),H(),clock,lambda *a:SimpleNamespace(close=lambda:None))
127        self.assertEqual(len(limits),1);self.assertGreater(limits[0],0);self.assertLessEqual(limits[0],.5)
128    def test_private_worker_cannot_bypass_watchdog(self):
129        with patch.dict(os.environ,{},clear=True):
130            with self.assertRaisesRegex(RuntimeError,'guardian-owned'):r.worker('/unused/request.json')
131    def test_run_uuid_cannot_reset_budget_or_change_directory(self):
132        d=r.load_guardian()
133        with tempfile.TemporaryDirectory() as t:
134            base=Path(t);root=base/'suite';root.mkdir();runtime=base/'runtime';runtime.mkdir()
135            (runtime/'training_deadline.py').write_text('fixture');(root/'execution-contract.json').write_text('{}')
136            checkpoint=base/'checkpoint';checkpoint.write_bytes(b'fixture');manifest=base/'manifest';manifest.write_text('{}')
137            run_id=str(uuid.uuid4());out=root/'runs'/run_id
138            approval={'approved':True,'run_id':run_id,'run_directory':str(out),'suite_sha256':'fixed',
139                'execution_contract_sha256':r.sha(root/'execution-contract.json'),'runner_sha256':r.sha(r.__file__),
140                'guardian_sha256':r.sha(runtime/'training_deadline.py'),'checkpoint':str(checkpoint),'checkpoint_sha256':r.sha(checkpoint),
141                'dataset_manifest':str(manifest),'dataset_manifest_sha256':r.sha(manifest),'device':'cuda:0','role':'candidate','budget_seconds':7200}
142            path=base/'approval';path.write_text(json.dumps(approval))
143            with patch.object(r,'ROOT',root),patch.object(r,'load_guardian',return_value=d),patch.object(r,'inputs',return_value=(self.suite,{'suite_sha256':'fixed'})),patch.object(d,'supervise',return_value={'returncode':0,'deadline_expired':False}) as supervise:
144                r.launch(path);original=(out/'deadline.json').read_bytes()
145                with self.assertRaises(FileExistsError):r.launch(path)
146                self.assertEqual((out/'deadline.json').read_bytes(),original);self.assertEqual(supervise.call_count,1)
147                (out/'deadline.json').unlink()
148                with self.assertRaises(FileExistsError):r.launch(path)
149                self.assertFalse((out/'deadline.json').exists())
150                approval['run_directory']=str(base/'alternate');path.write_text(json.dumps(approval))
151                with self.assertRaisesRegex(ValueError,'alternate'):r.launch(path)
152    def test_detached_engine_fixture_dies_with_parent_sigkill(self):
153        # A Python sleeper stands in for UCI: never launch Sunfish or a model.
154        with tempfile.TemporaryDirectory() as t:
155            root=Path(t);pidfile=root/'child.pid';ready=root/'ready';parent=root/'parent.py'
156            payload=f'import pathlib,time; pathlib.Path({str(ready)!r}).write_text("ready"); time.sleep(60)'
157            parent.write_text('import subprocess,sys,os,time,pathlib\n'
158                +f'p=subprocess.Popen([sys.executable,{str(Path(r.__file__).resolve())!r},"--engine-child",str(os.getpid()),sys.executable,"-c",{payload!r}],start_new_session=True)\n'
159                +f'pathlib.Path({str(pidfile)!r}).write_text(str(p.pid))\n'
160                +'time.sleep(60)\n')
161            proc=subprocess.Popen([sys.executable,str(parent)]);child=None
162            try:
163                until=time.monotonic()+8
164                while not ready.exists() and time.monotonic()<until:time.sleep(.02)
165                self.assertTrue(ready.exists(),'fixture child did not start');child=int(pidfile.read_text())
166                proc.kill();proc.wait(timeout=2)
167                until=time.monotonic()+3
168                while time.monotonic()<until:
169                    stat=Path(f'/proc/{child}/stat')
170                    if not stat.exists() or stat.read_text().split()[2]=='Z':break
171                    time.sleep(.02)
172                else:self.fail('detached fixture survived parent SIGKILL')
173            finally:
174                if proc.poll() is None:proc.kill();proc.wait()
175                if child:
176                    try:os.kill(child,signal.SIGKILL)
177                    except ProcessLookupError:pass
178
179if __name__=='__main__':unittest.main(verbosity=2)
180