amend_book_holdouts.py

1"""Subset-only correction for independently found delayed heldout transpositions."""
2import json
3from pathlib import Path
4import numpy as np
5from build_curriculum import ROOT,DTYPE,guard,sha,atomic_json,digest
6OLD=ROOT/'opening-book-v2';OUT=ROOT/'opening-book-v3'
7def main():
8    if OUT.exists():raise ValueError('refuse overwrite')
9    guard(300_000_000)
10    report=json.loads(Path(__file__).with_name('book-late-closure-review.json').read_text())
11    assert report['approved'] and report['parent_manifest_sha256']==sha(OLD/'manifest.json')
12    extra_path=Path(report['keys_path']);assert sha(extra_path)==report['keys_sha256']
13    old=json.loads((OLD/'manifest.json').read_text());assert sha(OLD/'metadata.jsonl')==old['metadata_sha256']
14    closure={k.tobytes() for k in np.fromfile(OLD/'opening-protected-keys.bin',dtype='V32')};closure.update(k.tobytes() for k in np.fromfile(extra_path,dtype='V32'))
15    data=np.empty(old['rows'],dtype=DTYPE)
16    for f in old['files']:
17        assert sha(OLD/f['name'])==f['sha256'];a=np.fromfile(OLD/f['name'],dtype=DTYPE);data[a['game']]=a
18    OUT.mkdir();kept={0:[],1:[]};pairs=[];n=0;removed=[];nonpad={0:0,1:0};pair_counts={0:0,1:0}
19    with (OUT/'metadata.jsonl').open('w') as out:
20        for oldid,line in enumerate((OLD/'metadata.jsonl').open()):
21            md=json.loads(line);split=md['split'];key=bytes.fromhex(md['position_key'])
22            if split==0 and key in closure:removed.append(md['position_key']);continue
23            row=data[oldid].copy();row['game']=n;md['metadata_index']=n;n+=1;out.write(json.dumps(md,separators=(',',':'))+'\n');local=len(kept[split]);kept[split].append(row);pair_counts[split]+=len(md['book_moves']);nonpad[split]+=int(row['attention'].sum())
24            if split==0:pairs.extend((local,j) for j in range(len(md['book_moves'])))
25    files=[];ordinal=0
26    for split in (0,1):
27        for start in range(0,len(kept[split]),65536):
28            a=np.asarray(kept[split][start:start+65536],dtype=DTYPE);p=OUT/f'records-{ordinal:06d}.bin';a.tofile(p);files.append(dict(name=p.name,source=2,split=split,rows=len(a),sha256=sha(p)));ordinal+=1
29    np.asarray(pairs,dtype=[('row','<u4'),('choice','<u2')]).tofile(OUT/'train-pairs.bin');np.asarray(sorted(closure),dtype='V32').tofile(OUT/'opening-protected-keys.bin');atomic_json(OUT/'shared.json',json.loads((OLD/'shared.json').read_text()))
30    identity={**old['identity'],'parent_manifest_sha256':sha(OLD/'manifest.json'),'amendment_code_sha256':sha(Path(__file__)),'late_closure_receipt_sha256':sha(Path(__file__).with_name('book-late-closure-review.json')),'opening_protected_keys_sha256':sha(OUT/'opening-protected-keys.bin'),'heldout_group_scope':'any occurrence in the full source line, including after ply8','removed_delayed_group_training_keys':removed}
31    identity['late_group_review_receipt']=report
32    counts={**old['counts'],'opening_holdout_related_states_withheld':old['counts']['opening_holdout_related_states_withheld']+len(removed),'train_states':len(kept[0]),'train_state_move_pairs':pair_counts[0],'train_canonical_serialized_slots':123*len(kept[0]),'train_pair_pass_serialized_slots':123*pair_counts[0],'train_pair_pass_target_tokens':3*pair_counts[0],'train_nonpadding_input_slots':nonpad[0]}
33    m={**old,'identity':identity,'identity_sha256':digest(identity),'files':files,'rows':n,'counts':counts,'metadata_sha256':sha(OUT/'metadata.jsonl'),'pair_map_sha256':sha(OUT/'train-pairs.bin'),'opening_protection_keys':len(closure),'label_readiness':'pending_independent_review'}
34    atomic_json(OUT/'manifest.json',m);print(json.dumps(dict(removed=removed,counts=counts,protected_keys=len(closure),manifest_sha256=sha(OUT/'manifest.json'))),flush=True)
35if __name__=='__main__':main()
36