expand_theory.py

1"""FEN-only exact zero-counter endgames from immutable sealed records."""
2import json,collections,hashlib,time
3from pathlib import Path
4import numpy as np
5from build_curriculum import ROOT,STATE,DTYPE,c,guard,sha,atomic_json,digest
6from inventory_inputs import protection,aliases
7from root_policy import root_policy
8from build_pilot import family_cap_indices
9import chess.syzygy
10OUT=ROOT/'theory-v3'
11
12def board_from_row(row):
13    b=c.chess.Board(None)
14    for sq,t in enumerate(row['x'][51:115]):
15        p=int(t)-128
16        if p:b.set_piece_at(sq,c.chess.Piece((p-1)%6+1,p<=6))
17    b.turn=int(row['x'][115])==141
18    if int(row['x'][116])!=143:raise ValueError('castling unsupported')
19    ep=int(row['x'][117]);b.ep_square=None if ep==159 else ep-160
20    b.halfmove_clock=int(row['x'][118])-224;b.fullmove_number=int(row['fullmove'])
21    c.check_board(b)
22    if c.position_key(b)!=row['key'].tobytes():raise ValueError('decode key')
23    return b
24
25def main():
26    if OUT.exists():raise ValueError('refuse overwrite')
27    guard(20_000_000);OUT.mkdir();db,_=protection();tb=chess.syzygy.open_tablebase(str(ROOT),max_fds=32)
28    m=json.loads((STATE/'manifest.json').read_text());records=[];metadata=[];stats=collections.Counter();seen=set();start=time.monotonic()
29    # Bound work by 25000 eligible roots/source. All ordinary sources already sealed/protected.
30    files=sorted((f for f in m['files'] if f['split']==0),key=lambda f:hashlib.sha256(('theory-v2:'+f['name']).encode()).digest())
31    for f in files:
32        src=f['source']
33        if stats[f'evaluated:{src}']>=25000:continue
34        rows=np.memmap(STATE/f['name'],dtype=DTYPE,mode='r')
35        mask=(np.count_nonzero(rows['x'][:,51:115]!=128,axis=1)<=5)&(rows['x'][:,118]==224)&(rows['x'][:,116]==143)
36        for row in rows[mask]:
37            if stats[f'evaluated:{src}']>=25000:break
38            b=board_from_row(row);key=c.position_key(b)
39            if key in seen:continue
40            seen.add(key)
41            if db.execute('SELECT 1 FROM protected WHERE k=?',(key,)).fetchone():raise ValueError('protected root')
42            meta=json.loads(db.execute('SELECT metadata FROM games WHERE id=?',(int(row['game']),)).fetchone()[0])
43            if any(db.execute('SELECT 1 FROM protected_games WHERE k=?',(a,)).fetchone() for a in aliases(meta)):raise ValueError('protected game')
44            stats[f'evaluated:{src}']+=1
45            result=root_policy(tb,b,history_known=True,depth=1,max_nodes=1500)
46            if not result['move_only_ready'] or result['outcome_interval'][0]<0:stats['unresolved_claim_or_loss']+=1;continue
47            ties=result['preserving_moves'];move=c.chess.Move.from_uci(ties[int.from_bytes(key[:4],'little')%len(ties)])
48            e=c.example(b,move,0);r=row.copy();r['x']=e['input_ids'];r['y']=e['labels'];r['attention']=e['attention_mask'];r['game']=len(metadata)
49            fam=''.join(sorted((p.symbol().upper() for p in b.piece_map().values() if p.color),key=lambda x:'KQRBNP'.index(x)))+'v'+''.join(sorted((p.symbol().upper() for p in b.piece_map().values() if not p.color),key=lambda x:'KQRBNP'.index(x)))
50            metadata.append(dict(position_key=key.hex(),fen=b.fen(en_passant='fen'),material_family=fam,source_kind='tablebase_exact',human_origin=('human_main','human_elite')[src],game_aliases=aliases(meta),source_sequence_sha256=meta['sequence_sha256'],source_ply=int(row['ply']),original_record_file=f['name'],original_game_id=int(row['game']),history_basis='actual halfmove zero: reversible repetition window known empty; earlier history unknown and irrelevant',history_uci=[],history_start_fen=b.fen(en_passant='fen'),history_dropout=False,outcome=result['outcome_interval'][0],raw_wdl=result['raw_wdl'],raw_dtz=result['raw_dtz'],preserving_moves=ties,tied_target_triples=[c.triple(c.chess.Move.from_uci(u)) for u in ties],selected_move=move.uci(),complete_preserving_move_set=True))
51            records.append(r)
52        if sum(stats[k] for k in ('evaluated:0','evaluated:1'))%500<100:print(json.dumps(dict(stats=stats,ready=len(records),seconds=time.monotonic()-start)),flush=True)
53    keep=family_cap_indices(metadata);records=[records[i] for i in keep];metadata=[metadata[i] for i in keep]
54    for i,r in enumerate(records):r['game']=i
55    (OUT/'metadata.jsonl').write_text(''.join(json.dumps(x)+'\n' for x in metadata))
56    files=[]
57    for src in (0,1):
58        a=np.asarray([r for r in records if int(r['source'])==src],dtype=DTYPE);p=OUT/f'records-source{src}.bin';a.tofile(p);files.append(dict(name=p.name,rows=len(a),source=src,split=0,sha256=sha(p)))
59    shared=json.loads((STATE/'shared.json').read_text());atomic_json(OUT/'shared.json',shared)
60    identity=dict(kind='zero-counter-FEN-only-exact-v2',source_manifest_sha256=sha(STATE/'manifest.json'),code_sha256=sha(Path(__file__)),policy_sha256=sha(Path(__file__).with_name('root_policy.py')))
61    manifest=dict(format='chess-state-records-v2',identity=identity,identity_sha256=digest(identity),record_bytes=416,dtype=DTYPE.descr,contract=c.CONTRACT,files=files,shared_sha256=sha(OUT/'shared.json'),metadata_sha256=sha(OUT/'metadata.jsonl'),rows=len(records),stats=dict(stats),label_readiness='pending_independent_review',exposures=3,training_launched=False)
62    atomic_json(OUT/'manifest.json',manifest);print(json.dumps(manifest),flush=True)
63if __name__=='__main__':main()
64