build_pilot.py

1"""Bounded state-v2 pilot export; remains unready until independent review."""
2import collections,hashlib,json,time
3from pathlib import Path
4from inventory_inputs import ROOT,PROJECT,c,protection,aliases,replay,_read_shard,sha,guard,save
5from root_policy import root_policy,validate_history
6import chess.syzygy
7from records import DTYPE,CONTRACT,canonical,digest
8import numpy as np
9
10OUT=ROOT/'pilot-v1-staging'
11
12def window_encodable(board):
13    # Keep the preceding zeroing move inside the 16-ply encoded suffix, so the
14    # boundary's legal en-passant identity is reconstructible too.
15    if board.halfmove_clock>15:return False
16    b=board.copy(stack=True)
17    for n in range(board.halfmove_clock+1):
18        if b.castling_rights:return False
19        if n<board.halfmove_clock:b.pop()
20    return True
21
22def encode_record(board,move,source,game,ply):
23    e=c.example(board,move,16)
24    row=np.zeros((),dtype=DTYPE)
25    row['x']=e['input_ids'];row['y']=e['labels'];row['attention']=e['attention_mask']
26    row['source']=source;row['split']=0;row['game']=game;row['ply']=ply;row['fullmove']=board.fullmove_number
27    row['key']=np.frombuffer(c.position_key(board),dtype=np.uint8)
28    return row
29
30def family_cap_indices(metadata):
31    keep=list(range(len(metadata)));counts=collections.Counter(m['material_family'] for m in metadata)
32    while keep and max(counts.values())*5>len(keep):
33        family=max(counts,key=lambda f:(counts[f],f))
34        index=next(i for i in reversed(keep) if metadata[i]['material_family']==family)
35        keep.remove(index);counts[family]-=1
36    return keep
37
38def main():
39    protected,config=protection()
40    candidate_path=ROOT/'endgame-candidates.jsonl';inv=json.loads((ROOT/'endgame-inventory.json').read_text())
41    if sha(candidate_path)!=inv['candidate_sha256']:raise ValueError('candidate integrity')
42    candidates=[json.loads(x) for x in candidate_path.read_text().splitlines()]
43    targets={};selected=collections.Counter()
44    for r in sorted(candidates,key=lambda r:hashlib.sha256(('pilot-v1:'+r['position_key']).encode()).digest()):
45        src=0 if r['source_kind']=='human_main' else 1
46        if r['halfmove_clock']>15 or selected[src]>=256:continue
47        targets[(r['source_sequence_sha256'],r['ply'])]=r;selected[src]+=1
48    if OUT.exists():raise ValueError('pilot staging exists; refuse overwrite')
49    guard(20_000_000);OUT.mkdir()
50    tb=chess.syzygy.open_tablebase(str(ROOT),max_fds=32)
51    target_sequences={k[0] for k in targets}
52    rows=[];metadata=[];audit=[];stats=collections.Counter();families=collections.Counter();accepted=collections.Counter();seen=set();processed_sequences=set();start=time.monotonic()
53    for source,ref in enumerate(config['sources']):
54        root=Path(ref['root']);m=json.loads((root/'manifest.json').read_text());n=0
55        shards=[s for s in m['shards'] if s['split']=='train'];shards.sort(key=lambda s:hashlib.sha256(('curriculum-v1:'+','.join(sorted(s['files']))).encode()).digest())
56        for shard in shards:
57            tokens,offsets,metas=_read_shard(root,shard)
58            for a,b,meta in zip(offsets,offsets[1:],metas):
59                if n>=2000:break
60                n+=1
61                if meta['sequence_sha256'] not in target_sequences:continue
62                if meta['sequence_sha256'] in processed_sequences:continue
63                processed_sequences.add(meta['sequence_sha256'])
64                if any(protected.execute('SELECT 1 FROM protected_games WHERE k=?',(x,)).fetchone() for x in aliases(meta)):raise ValueError('target heldout game')
65                for board,ply,_ in replay(tokens[a:b],meta):
66                    target=targets.get((meta['sequence_sha256'],ply))
67                    if target is None:continue
68                    if board.fen(en_passant='fen')!=target['fen'] or c.position_key(board).hex()!=target['position_key']:raise ValueError('candidate reconstruction mismatch')
69                    if protected.execute('SELECT 1 FROM protected WHERE k=?',(c.position_key(board),)).fetchone():raise ValueError('heldout target')
70                    if not window_encodable(board):stats['unencodable_window']+=1;continue
71                    validate_history(board,True)
72                    result=root_policy(tb,board,history_known=True,depth=1,max_nodes=1500)
73                    stats['evaluated']+=1;reason='ready_candidate' if result['move_only_ready'] else 'unresolved_or_claim'
74                    audit.append(dict(position_key=target['position_key'],source=source,result=result,status=reason))
75                    if not result['move_only_ready']:stats[reason]+=1;continue
76                    stats['semantically_ready_candidates']+=1
77                    if result['outcome_interval'][0]<0:stats['loss_roots_inventory_only']+=1;continue
78                    fam=target['material_family']
79                    if accepted[source]>=128 or families[fam]>=51:stats['selection_cap']+=1;continue
80                    if target['position_key'] in seen:raise ValueError('duplicate pilot key')
81                    seen.add(target['position_key']);accepted[source]+=1;families[fam]+=1
82                    tied=result['preserving_moves'];ix=int(hashlib.sha256(('representative:'+target['position_key']).encode()).hexdigest(),16)%len(tied)
83                    move=c.chess.Move.from_uci(tied[ix]);game=len(metadata)
84                    rows.append(encode_record(board,move,source,game,ply))
85                    prior=board.copy(stack=True)
86                    for _ in range(min(16,len(prior.move_stack))):prior.pop()
87                    metadata.append(dict(position_key=target['position_key'],fen=target['fen'],material_family=fam,
88                        source_kind='tablebase_exact',human_origin=target['source_kind'],source_sequence_sha256=meta['sequence_sha256'],
89                        game_aliases=aliases(meta),source_ply=ply,certification_scope='authoritative source game; complete reversible history encoded',
90                        encoded_history_covers_repetition_window=True,history_dropout=False,
91                        history_start_fen=prior.fen(en_passant='fen'),history_uci=[m.uci() for m in c.genuine_history(board)],
92                        outcome=result['outcome_interval'][0],raw_wdl=result['raw_wdl'],raw_dtz=result['raw_dtz'],
93                        preserving_moves=tied,tied_target_triples=[c.triple(c.chess.Move.from_uci(u)) for u in tied],
94                        selected_move=move.uci(),complete_preserving_move_set=True,
95                        target_selection='one stable representative per state; all ties retained in sidecar; repeated exposures are not new unique states'))
96            print(json.dumps(dict(source=source,games=n,evaluated=stats['evaluated'],rows=len(rows),elapsed=time.monotonic()-start)),flush=True)
97            guard(20_000_000)
98            if n>=2000:break
99    tb.close()
100    if stats['evaluated']+stats['unencodable_window']!=len(targets):raise ValueError('not all selected targets reconstructed')
101    keep=family_cap_indices(metadata);stats['final_family_cap_pruned']=len(metadata)-len(keep)
102    rows=[rows[i] for i in keep];metadata=[metadata[i] for i in keep]
103    for i,row in enumerate(rows):row['game']=i
104    if not rows:raise ValueError('no accepted pilot rows')
105    families=collections.Counter(m['material_family'] for m in metadata)
106    accepted=collections.Counter(int(r['source']) for r in rows)
107    records=np.asarray(rows,dtype=DTYPE)
108    # One shard per source is required by Records' source filtering.
109    files=[]
110    for source in (0,1):
111        subset=records[records['source']==source];name=f'records-source{source}.bin';(OUT/name).write_bytes(subset.tobytes())
112        files.append(dict(name=name,sha256=sha(OUT/name),rows=len(subset),source=source,split=0))
113    (OUT/'metadata.jsonl').write_text(''.join(json.dumps(r,separators=(',',':'))+'\n' for r in metadata))
114    (OUT/'label-audit.jsonl').write_text(''.join(json.dumps(r,separators=(',',':'))+'\n' for r in audit))
115    shared=dict(position_ids=list(range(122)),loss_mask=[int(i in (119,120,121)) for i in range(122)],target_positions=[119,120,121])
116    (OUT/'shared.json').write_bytes(canonical(shared))
117    identity=dict(kind='bounded-tablebase-pilot-v1',codec=CONTRACT,source_candidate_sha256=inv['candidate_sha256'],
118        code={name:sha(Path(__file__).with_name(name)) for name in ('root_policy.py','build_pilot.py')},
119        source_references=config['references'],history_dropout=False,unique_state_count=len(rows))
120    manifest=dict(format='chess-state-records-v2',identity=identity,identity_sha256=digest(identity),contract=CONTRACT,
121        record_bytes=DTYPE.itemsize,dtype=DTYPE.descr,files=files,shared_sha256=sha(OUT/'shared.json'),rows=len(rows),
122        metadata_sha256=sha(OUT/'metadata.jsonl'),label_audit_sha256=sha(OUT/'label-audit.jsonl'),
123        label_readiness='awaiting_independent_review',training_launched=False,counts=dict(stats),source_rows=dict(accepted),material_families=dict(families))
124    manifest['source_id_semantics']={'0':'human main origin; label kind tablebase_exact','1':'human Elite origin; label kind tablebase_exact'}
125    manifest['objective']='actual-rule WDL preservation for winning/drawing roots; not DTZ-optimal defense'
126    manifest['maximum_actual_family_fraction']=max(families.values())/len(rows)
127    (OUT/'manifest.json').write_bytes(canonical(manifest));save('pilot-build-summary.json',manifest)
128    print(json.dumps(dict(rows=len(rows),counts=dict(stats),source_rows=dict(accepted),elapsed=time.monotonic()-start)),flush=True)
129
130if __name__=='__main__':main()
131