prepare_theory_replay.py

1"""Read-only replay views for a proposed40/40/20 follow-up; never trains."""
2import collections,json
3from pathlib import Path
4import numpy as np
5from build_curriculum import ROOT,STATE,DTYPE,sha,atomic_json,guard
6OUT=ROOT/'opening-followup-replay-v1';BOOK=ROOT/'opening-book-v3';CURRENT=ROOT/'curriculum-v3';EXACT=ROOT/'theory-v4'
7
8def main():
9    if OUT.exists():raise ValueError('refuse overwrite')
10    guard(150_000_000);bm=json.loads((BOOK/'manifest.json').read_text());assert bm['label_readiness']=='independently_reviewed_ready';OUT.mkdir();cm=json.loads((CURRENT/'manifest.json').read_text());em=json.loads((EXACT/'manifest.json').read_text())
11    excluded={k.tobytes() for k in np.fromfile(BOOK/'opening-protected-keys.bin',dtype='V32')}
12    bookkeys={bytes.fromhex(json.loads(x)['position_key']) for x in (BOOK/'metadata.jsonl').open()};excluded.update(bookkeys)
13    keys=np.asarray(sorted(excluded),dtype='V32');base=np.fromfile(CURRENT/'ordinary-source1-indices.bin',dtype='<u4');base.sort();m=json.loads((STATE/'manifest.json').read_text());offset=0;kept=[];counts=collections.Counter()
14    for f in m['files']:
15        if f['source']!=1 or f['split']!=0:continue
16        rows=np.memmap(STATE/f['name'],dtype=DTYPE,mode='r');a=np.searchsorted(base,offset);b=np.searchsorted(base,offset+len(rows));ids=base[a:b];selected=rows[ids-offset];mask=~np.isin(np.ascontiguousarray(selected['key']).view('V32').ravel(),keys);kept.append(ids[mask]);counts['opening_owned_or_protected_elite_excluded']+=int((~mask).sum());offset+=len(rows)
17    elite=np.concatenate(kept);elite.tofile(OUT/'elite-indices.bin');exact_pair_dtype=np.dtype([('source','u1'),('row','<u4'),('choice','<u2')]);pairs=[];exact_rows=[];metadata=[json.loads(x) for x in (EXACT/'metadata.jsonl').read_text().splitlines()]
18    for f in em['files']:
19        rows=np.memmap(EXACT/f['name'],dtype=DTYPE,mode='r')
20        for local,r in enumerate(rows):
21            if r['key'].tobytes() in excluded:counts['opening_protected_exact_excluded']+=1;continue
22            exact_rows.append((f['source'],local));pairs.extend((f['source'],local,j) for j in range(len(metadata[int(r['game'])]['tied_target_triples'])))
23    np.asarray(pairs,dtype=exact_pair_dtype).tofile(OUT/'exact-pairs.bin');np.asarray(exact_rows,dtype=[('source','u1'),('row','<u4')]).tofile(OUT/'exact-rows.bin')
24    bp=bm['counts']['train_state_move_pairs'];opening=3*bp;ending=opening;elite_exposure=(opening+1)//2;total=opening+ending+elite_exposure
25    proposal=dict(status='proposal only; extra training not approved',opening_pair_passes=3,opening_exposures=opening,endgame_exposures=ending,elite_exposures=elite_exposure,total_exposures=total,weights='40% opening /40% exact endgame /20% Elite, rounded by at most one decision',opening_unique_states=bm['counts']['train_states'],opening_unique_pairs=bp,endgame_unique_states=len(exact_rows),endgame_unique_pairs=len(pairs),endgame_pair_equivalent_passes=ending/len(pairs),endgame_average_state_exposures=ending/len(exact_rows),elite_unique_available=len(elite),elite_sampling='without replacement from this heldout-clean view',all_exact_pairs_fit=ending>=len(pairs),all_elite_exposures_fit=elite_exposure<=len(elite),hours_at_992=total/992/3600,serialized_exposure_slots=123*total,supervised_target_exposures=3*total,sampling='interleave sources; opening3separatedcompletepairpasses; exactfirstcovereverypreservingpair, then state-balanced cycling of verified alternatives; Elite withoutreplacement',forgetting='compare source/phase ordinary validation, exact2000holdout and frozenopening128holdout before/after; lowerLR/stopping owned by trainer; currentCM7unchanged')
26    refs=[CURRENT/'manifest.json',CURRENT/'ordinary-source1-indices.bin',EXACT/'manifest.json',EXACT/'metadata.jsonl',BOOK/'manifest.json',BOOK/'opening-protected-keys.bin',BOOK/'metadata.jsonl',ROOT/'exact-validation-v1'/'manifest.json',STATE/'manifest.json']
27    manifest=dict(readiness='pending_independent_review',dependencies=[dict(path=str(p),sha256=sha(p)) for p in refs],elite_index_sha256=sha(OUT/'elite-indices.bin'),exact_pairs_sha256=sha(OUT/'exact-pairs.bin'),exact_rows_sha256=sha(OUT/'exact-rows.bin'),exact_pair_dtype=exact_pair_dtype.descr,counts=dict(counts),proposal=proposal,code_sha256=sha(Path(__file__)),training_launched=False)
28    proposal['exact_complete_pair_passes']=1
29    proposal['exact_remainder_state_balanced_exposures']=ending-len(pairs)
30    proposal['exact_additional_exposures_per_state_floor']=(ending-len(pairs))//len(exact_rows)
31    proposal['exact_additional_exposures_per_state_ceil']=(ending-len(pairs)+len(exact_rows)-1)//len(exact_rows)
32    proposal['exact_pair_pass_equivalent_is_not_uniform_repetition']=True
33    proposal['sampling']='interleave40/40/20; opening3separatedcompletepairpasses; exactonecompletepairpass then state-balanced remainder, cycling alternatives; Elite withoutreplacement'
34    manifest['elite_index_semantics']='uint32 original sealed state-v2 source1/split0 row index, not composite state ID'
35    manifest['exact_index_semantics']='source-local row in theory-v4; choice indexes complete metadata tied_target_triples via record.game'
36    proposal['alternative_budgets']=[dict(book_pair_passes=k,opening=k*bp,endgame=k*bp,elite=(k*bp+1)//2,total=2*k*bp+(k*bp+1)//2,minutes_at_992=(2*k*bp+(k*bp+1)//2)/992/60) for k in (1,3,5)]
37    assert proposal['all_exact_pairs_fit'] and proposal['all_elite_exposures_fit']
38    atomic_json(OUT/'manifest.json',manifest);print(json.dumps(manifest),flush=True)
39if __name__=='__main__':main()
40