matched_nll.py

1"""Fixed September validation comparison; CPU preparation, terminal-only GPU run."""
2import argparse,fcntl,hashlib,json,os,struct,sys,time
3from pathlib import Path
4ROOT=Path(__file__).resolve().parent
5SCALEUP=ROOT.parent;PROJECT=SCALEUP.parent;RUNTIME=SCALEUP/'training-runtime-v2'
6OLD=PROJECT/'lichess-month-202609/dataset'
7OUTPUT=Path('/PUBLIC_PLACEHOLDER/install-python-chess')
8TRAIN=OUTPUT.parent/'training-four-hour-v1'
9sys.path.insert(0,str(RUNTIME))
10import training_deadline as d
11def load(p):return json.loads(Path(p).read_text())
12def sha(p):
13    h=hashlib.sha256()
14    with Path(p).open('rb') as f:
15        while b:=f.read(8*1024*1024):h.update(b)
16    return h.hexdigest()
17def write_new(p,value):
18    with Path(p).open('x') as f:json.dump(value,f,indent=2);f.flush();os.fsync(f.fileno())
19def sample():
20    import importlib.util
21    if importlib.util.find_spec('chess') is None:
22        chess_root=Path('/PUBLIC_PLACEHOLDER/install-python-chess')
23        spec=importlib.util.spec_from_file_location('chess',chess_root/'__init__.py',submodule_search_locations=[str(chess_root)])
24        module=importlib.util.module_from_spec(spec);sys.modules['chess']=module;spec.loader.exec_module(module)
25    from chessdata.reader import _read_shard
26    m=load(OLD/'manifest.json');data=[];h=hashlib.sha256(b'chess-matched-september-validation-v1\0')
27    for s in m['shards']:
28        if s['split']!='validation':continue
29        tokens,offsets,metadata=_read_shard(OLD,s)
30        for a,b,meta in zip(offsets,offsets[1:],metadata):
31            value=bytes(tokens[a:b]);sequence=meta['sequence_sha256'].encode('ascii')
32            h.update(sequence);h.update(struct.pack('<I',len(value)));h.update(value);data.append(value)
33    counts={'games':len(data),'stored_tokens':sum(map(len,data)),'prediction_targets':sum(len(v)-1 for v in data)}
34    if counts!={'games':7000,'stored_tokens':1602704,'prediction_targets':1595704}:raise ValueError('fixed sample counts changed')
35    audit=SCALEUP/'native-union-complete.json';pointer=load(audit);proof=OUTPUT.parent/'union/previous-heldout-audit.json'
36    if sha(proof)!=pointer['heldout_audit_sha256'] or load(proof)['status']!='passed':raise ValueError('historical heldout proof changed')
37    binding={'selection':'all original September validation games, immutable shard order','manifest_sha256':sha(OLD/'manifest.json'),
38        'dataset_fingerprint':m['fingerprint'],'sample_sha256':h.hexdigest(),'sample_hash_format':'domain tag, then each ASCII sequence SHA256, uint32LE stored length, raw uint8 tokens',
39        **counts,'heldout_audit_sha256':sha(proof),'tokenizer_sha256':sha(RUNTIME/'chessdata/tokenizer.py'),
40        'evaluation_code_sha256':{n:sha(RUNTIME/n) for n in ('train_chess.py','training_core.py','training_deadline.py')},
41        'protocol':'FP32 weights, BF16 autocast SDPA, micro8, raw128-class token NLL; labels shifted once; PAD ignored; EOS retained; no legal logit masking',
42        'legality_sample_games':128,'test_evaluated':False}
43    return data,binding
44def specification():
45    saved=load(ROOT/'slice.json');_,actual=sample()
46    if saved!=actual:raise ValueError('fixed sample changed')
47    terminal=load(SCALEUP/'four-hour-terminal-verification.json')
48    if terminal['status']!='terminal_verified_pending_independent_review_and_matched_evaluation':raise ValueError('terminal training verification required')
49    if Path('/proc/370996').exists():raise ValueError('training owner still active')
50    baseline=PROJECT/'training-v1/month-2h/best.pt';candidate=Path(terminal['checkpoints']['best']['path'])
51    cfg=load(RUNTIME/'four-hour-config.json');new_manifest=Path(cfg['dataset'])/'manifest.json'
52    if sha(new_manifest)!=cfg['dataset_manifest_sha256']:raise ValueError('candidate dataset changed')
53    checkpoints={'baseline':{'path':str(baseline.resolve()),'sha256':sha(baseline),'dataset_fingerprint':saved['dataset_fingerprint']},
54        'candidate':{'path':str(candidate.resolve()),'sha256':sha(candidate),'dataset_fingerprint':load(new_manifest)['fingerprint']}}
55    if checkpoints['baseline']['sha256']!='6133648b2926b38542ee3ee1251a1ba111d61da9a8ea69d1c5f0559b09933643':raise ValueError('baseline changed')
56    if checkpoints['candidate']['sha256']!=terminal['checkpoints']['best']['sha256']:raise ValueError('candidate changed')
57    return {'owner_sha256':sha(__file__),'slice_sha256':sha(ROOT/'slice.json'),'terminal_verification_sha256':sha(SCALEUP/'four-hour-terminal-verification.json'),
58        'checkpoints':checkpoints,'output':str(OUTPUT.resolve()),'budget_seconds':600,'device':'cuda:0','micro':8,'legality_games':128}
59def worker():
60    parent=Path('/proc',str(os.getppid()),'cmdline').read_bytes().split(b'\0')
61    if str(RUNTIME/'training_deadline.py').encode() not in parent or b'--guardian' not in parent or os.getpgrp()!=os.getpid():raise ValueError('owned guardian required')
62    request=load(OUTPUT/'request.json');spec=specification()
63    if request!={'approved':True,'specification':spec}:raise ValueError('worker binding changed')
64    deadline=d.read_deadline(OUTPUT/'deadline.json')
65    if deadline['identity']!=spec or deadline['budget_seconds']!=600:raise ValueError('deadline mismatch')
66    write_new(OUTPUT/'worker-started.json',{'pid':os.getpid(),'unix':time.time()})
67    import gc,torch,training_core as c
68    from train_chess import evaluate
69    torch.set_num_threads(4);torch.cuda.set_device(0);data,binding=sample();results={}
70    for label,entry in spec['checkpoints'].items():
71        if d.Deadline(deadline).soft_stop():raise TimeoutError('comparison deadline reached')
72        if sha(entry['path'])!=entry['sha256']:raise ValueError('checkpoint changed')
73        saved=c.load_checkpoint(entry['path'])
74        if saved['identity']['tokenizer']['sha256']!=binding['tokenizer_sha256'] or saved['identity']['parameters']!=85940736:raise ValueError('tokenizer/model mismatch')
75        if saved['identity']['dataset_fingerprint']!=entry['dataset_fingerprint']:raise ValueError('checkpoint dataset mismatch')
76        if any(saved['model_config'].get(k)!=v for k,v in c.BASELINE.items() if k!='attn_implementation'):raise ValueError('model configuration mismatch')
77        net=c.model(saved['model_config'],'cuda:0');net.load_state_dict(saved['model_state_dict']);del saved
78        value=evaluate(net,data,'cuda:0',micro=8,deadline=d.Deadline(deadline),legality_games=128)
79        if value is None or value['games']!=7000 or value['valid_targets']!=1595704:raise ValueError('incomplete paired evaluation')
80        results[label]=value;del net;gc.collect();torch.cuda.empty_cache()
81    write_new(OUTPUT/'results.json',{'status':'paired_complete','sample':binding,'checkpoints':spec['checkpoints'],'metrics':results,
82        'candidate_minus_baseline_nll':results['candidate']['nll']-results['baseline']['nll'],'scope':'matched validation, not test or playing strength','utc_unix':time.time()})
83def launch(approval):
84    spec=specification()
85    if load(approval)!={'approved':True,'specification':spec}:raise ValueError('exact reviewed comparison approval required')
86    OUTPUT.mkdir(exist_ok=False)
87    with (OUTPUT/'owner.lock').open('x') as lock:
88        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
89        deadline=d.create_or_resume(OUTPUT/'deadline.json',spec,budget=600,margin=15)
90        write_new(OUTPUT/'request.json',{'approved':True,'specification':spec})
91        env=dict(os.environ,CHESS_JOB_LOCK_FD=str(lock.fileno()),CUDA_VISIBLE_DEVICES='0,1')
92        result=d.supervise([sys.executable,str(Path(__file__).resolve()),'--worker'],deadline,env)
93        write_new(OUTPUT/'supervision.json',result)
94        if result['returncode'] or result['deadline_expired']:raise RuntimeError(result)
95        if load(OUTPUT/'results.json')['status']!='paired_complete':raise ValueError('missing complete paired result')
96def main():
97    p=argparse.ArgumentParser();p.add_argument('--prepare',action='store_true');p.add_argument('--worker',action='store_true');p.add_argument('--approval');a=p.parse_args()
98    if a.prepare:
99        _,binding=sample();write_new(ROOT/'slice.json',binding);print(json.dumps(binding));return
100    if a.worker:return worker()
101    if a.approval:return launch(a.approval)
102    print(json.dumps({'approved':False,'specification':specification()},indent=2))
103if __name__=='__main__':main()
104