holdout_amendment.py

1"""Freeze stratified exact validation; exclude full source-game transpositions."""
2import collections,hashlib,json,time
3from pathlib import Path
4import numpy as np
5from inventory_inputs import ROOT,STATE,PROJECT,c,protection,aliases,_read_shard,sha
6from records import DTYPE,atomic_json,digest
7from build_curriculum import guard
8from prepare import replay
9import chess.syzygy
10OUT=ROOT/'exact-holdout-v1';OLD=ROOT/'theory-v3'
11SEED='exact-validation-family-outcome-20261029-v1'
12
13def stratified_indices(metadata,n=2000):
14    groups=collections.defaultdict(list)
15    for i,m in enumerate(metadata):groups[(chess.syzygy.normalize_tablename(m['material_family']),m['outcome'])].append(i)
16    if not 0<n<len(metadata):raise ValueError('invalid holdout size')
17    # Largest-remainder proportional allocation; ties and rows are deterministic.
18    quota={k:len(v)*n//len(metadata) for k,v in groups.items()}
19    order=sorted(groups,key=lambda k:(-(len(groups[k])*n%len(metadata)),k))
20    for k in order[:n-sum(quota.values())]:quota[k]+=1
21    chosen=[]
22    for k in sorted(groups):
23        chosen.extend(sorted(groups[k],key=lambda i:hashlib.sha256((SEED+metadata[i]['position_key']).encode()).digest())[:quota[k]])
24    return sorted(chosen),{str(k):dict(available=len(groups[k]),selected=quota[k]) for k in sorted(groups)}
25
26def export(root,indices,metadata,rows,split,protection_sha):
27    root.mkdir();items=[];selected=[]
28    for i in indices:
29        r=rows[i].copy();r['game']=len(items);r['split']=split;selected.append(r);items.append({**metadata[i],'parent_theory_metadata_index':i})
30    (root/'metadata.jsonl').write_text(''.join(json.dumps(m)+'\n' for m in items));files=[]
31    for src in (0,1):
32        a=np.asarray([r for r in selected if int(r['source'])==src],dtype=DTYPE);p=root/f'records-source{src}.bin';a.tofile(p);files.append(dict(name=p.name,rows=len(a),source=src,split=split,sha256=sha(p)))
33    atomic_json(root/'shared.json',json.loads((OLD/'shared.json').read_text()))
34    ident=dict(parent_manifest_sha256=sha(OLD/'manifest.json'),amendment_code_sha256=sha(Path(__file__)),selection_sha256=sha(OUT/'selection.json'),protection_sha256=protection_sha,split=split)
35    atomic_json(root/'manifest.json',dict(format='chess-state-records-v2',identity=ident,identity_sha256=digest(ident),record_bytes=416,dtype=DTYPE.descr,contract=c.CONTRACT,files=files,rows=len(items),metadata_sha256=sha(root/'metadata.jsonl'),shared_sha256=sha(root/'shared.json'),label_readiness='pending_independent_review',label_semantics='unchanged exact zero-counter FEN-only labels; full ties retained'))
36
37def main():
38    if OUT.exists():raise ValueError('refuse overwrite')
39    guard(300_000_000);OUT.mkdir();db,config=protection()
40    metadata=[json.loads(x) for x in (OLD/'metadata.jsonl').read_text().splitlines()];old=json.loads((OLD/'manifest.json').read_text());assert old['label_readiness']=='independently_reviewed_ready'
41    rows={}
42    for f in old['files']:
43        assert sha(OLD/f['name'])==f['sha256']
44        for r in np.fromfile(OLD/f['name'],dtype=DTYPE):rows[int(r['game'])]=r
45    selected,strata=stratified_indices(metadata);selected_set=set(selected)
46    frozen=dict(seed=SEED,selection='proportional largest remainder by normalized family/outcome; SHA256 within stratum',parent_manifest_sha256=sha(OLD/'manifest.json'),selected_metadata_indices=selected,selected_keys=[metadata[i]['position_key'] for i in selected],strata=strata,model_results_used=False,limitation='heldout from next continuation only; inherited CM6 may have seen these states')
47    atomic_json(OUT/'selection.json',frozen)
48    wanted={metadata[i]['source_sequence_sha256'] for i in selected};heldout_aliases=set(a for i in selected for a in metadata[i]['game_aliases'])
49    gameids=[]
50    for gid,raw in db.execute('SELECT id,metadata FROM games'):
51        gm=json.loads(raw)
52        if heldout_aliases.intersection(aliases(gm)):gameids.append(gid)
53    keys=set();found=set();receipt=[];start=time.monotonic()
54    for src,ref in enumerate(config['sources']):
55        root=Path(ref['root']);m=json.loads((root/'manifest.json').read_text());assert sha(root/'manifest.json')==ref['manifest_sha256']
56        shards=sorted((s for s in m['shards'] if s['split']=='train'),key=lambda s:digest([config['seed'],sorted(s['files'])]))
57        sourcewanted={metadata[i]['source_sequence_sha256'] for i in selected if int(rows[i]['source'])==src};scanned=0
58        for shard in shards:
59            tokens,offsets,metas=_read_shard(root,shard)
60            for a,b,meta in zip(offsets,offsets[1:],metas):
61                scanned+=1;seq=meta['sequence_sha256']
62                if seq not in sourcewanted or seq in found:continue
63                _,gamekeys,_=replay((tokens[a:b],meta,'protect',config['seed'],src));keys.update(gamekeys);found.add(seq)
64                receipt.append(dict(source=src,sequence_sha256=seq,game_aliases=aliases(meta),positions_including_terminal=len(gamekeys),position_keys_sha256=hashlib.sha256(b''.join(gamekeys)).hexdigest(),source_shard=shard))
65            if sourcewanted<=found:break
66        print(json.dumps(dict(source=src,games_scanned=scanned,found=len(found),wanted=len(wanted),seconds=time.monotonic()-start)),flush=True)
67    if found!=wanted:raise ValueError('missing full heldout games')
68    if not {bytes.fromhex(metadata[i]['position_key']) for i in selected}<=keys:raise ValueError('selected keys missing from games')
69    np.asarray(sorted(keys),dtype='V32').tofile(OUT/'protected-keys.bin');np.asarray(sorted(gameids),dtype='<u4').tofile(OUT/'protected-game-ids.bin')
70    atomic_json(OUT/'game-replay-receipt.json',receipt)
71    protected=dict(selection_sha256=sha(OUT/'selection.json'),keys_sha256=sha(OUT/'protected-keys.bin'),key_count=len(keys),game_ids_sha256=sha(OUT/'protected-game-ids.bin'),game_id_count=len(gameids),aliases=sorted(heldout_aliases),source_sequences=sorted(wanted),full_game_replay_sha256=sha(OUT/'game-replay-receipt.json'),source_manifests=config['sources'],readiness='pending_independent_review')
72    atomic_json(OUT/'protection.json',protected)
73    kept=[i for i,m in enumerate(metadata) if i not in selected_set and bytes.fromhex(m['position_key']) not in keys and not heldout_aliases.intersection(m['game_aliases'])]
74    export(ROOT/'theory-v4',kept,metadata,rows,0,sha(OUT/'protection.json'));export(ROOT/'exact-validation-v1',selected,metadata,rows,1,sha(OUT/'protection.json'))
75    atomic_json(OUT/'summary.json',dict(validation_rows=len(selected),training_theory_rows=len(kept),other_theory_rows_removed=len(metadata)-len(selected)-len(kept),selected_source_games=len(wanted),protected_keys=len(keys),protected_game_ids=len(gameids)))
76    print((OUT/'summary.json').read_text(),flush=True)
77if __name__=='__main__':main()
78