build_book_targets.py

1"""All legal book transitions, no frequency or engine admission criterion."""
2import collections,hashlib,io,json,sqlite3,zipfile
3from pathlib import Path
4import numpy as np
5from inventory_inputs import ROOT,STATE,c,protection,sha
6from records import DTYPE,atomic_json,digest
7from build_curriculum import guard
8import chess.pgn
9OUT=ROOT/'opening-book-v2'
10
11def main():
12    if OUT.exists():raise ValueError('refuse overwrite')
13    guard(750_000_000);OUT.mkdir();protected,_=protection()
14    extra=json.loads((ROOT/'exact-holdout-v1'/'protection.json').read_text());assert sha(ROOT/'exact-holdout-v1'/'protected-keys.bin')==extra['keys_sha256']
15    exactkeys={k.tobytes() for k in np.fromfile(ROOT/'exact-holdout-v1'/'protected-keys.bin',dtype='V32')}
16    frozen=json.loads((ROOT/'opening-engine-v1'/'frozen-selection.json').read_text());groups=set(frozen['validation_prefix_groups'])
17    frozen_candidates=[json.loads(x) for x in (ROOT/'opening-engine-v1'/'candidates.jsonl').read_text().splitlines()];valroots={m['position_key'] for m in frozen_candidates if m['split']==1}
18    db=sqlite3.connect('file:'+str(ROOT/'opening-inventory.sqlite')+'?mode=ro',uri=True)
19    edges=collections.defaultdict(dict);raw_pairs=set();historical_pairs=0;exact_pairs=0
20    for key,move,book,n,heldout in db.execute('SELECT e.k,e.move,e.book,e.n,p.protected FROM edges e JOIN positions p USING(k)'):
21        raw_pairs.add((key,move))
22        if heldout:historical_pairs+=1;continue
23        if key in exactkeys:exact_pairs+=1;continue
24        edges[key].setdefault(move,{})[book]=n
25    seen=set();closure=set();occurrence_pairs=set();bookstats={};tmpmeta=OUT/'contexts.jsonl';tmprecords=OUT/'contexts.bin'
26    with tmpmeta.open('w') as mf,tmprecords.open('wb') as rf:
27        for book in ('8moves_v3.pgn','bjbraams_chessdb_198350_lines.pgn'):
28            archive=ROOT/(book+'.zip');assert sha(archive)==frozen['book_hashes'][archive.name];line=0;heldout_lines=0
29            with zipfile.ZipFile(archive) as z,io.TextIOWrapper(z.open(book),encoding='utf-8') as handle:
30                while (game:=chess.pgn.read_game(handle)) is not None:
31                    line+=1
32                    if game.errors:raise ValueError('PGN integrity')
33                    board=game.board();startfen=board.fen(en_passant='fen');moves=list(game.mainline_moves());states=[]
34                    for ply,move in enumerate(moves):
35                        if move not in board.legal_moves or board.is_game_over(claim_draw=False):raise ValueError('illegal/postautomatic source transition')
36                        key=c.position_key(board);states.append(key)
37                        if key in edges:
38                            occurrence_pairs.add((key,move.uci()))
39                            if key not in seen:
40                                if protected.execute('SELECT 1 FROM protected WHERE k=?',(key,)).fetchone():raise ValueError('historical heldout')
41                                choices=sorted(edges[key]);assert all(c.chess.Move.from_uci(u) in board.legal_moves for u in choices)
42                                e=c.example(board,c.chess.Move.from_uci(choices[0]),16);row=np.zeros((),dtype=DTYPE);row['x']=e['input_ids'];row['y']=e['labels'];row['attention']=e['attention_mask'];row['source']=2;row['game']=len(seen);row['ply']=ply;row['fullmove']=board.fullmove_number;row['key']=np.frombuffer(key,dtype=np.uint8);rf.write(row.tobytes())
43                                prefix=[m.uci() for m in c.genuine_history(board)];assert len(prefix)==ply<=16
44                                md=dict(position_key=key.hex(),fen=board.fen(en_passant='fen'),history_start_fen=startfen,history_uci=prefix,source_ply=ply,book=book,book_line=line,book_moves=choices,tied_target_triples=[c.triple(c.chess.Move.from_uci(u)) for u in choices],move_provenance=edges[key],label_kind='book_supported',exact_optimality_claim=False,complete_source_book_move_set=True,complete_optimal_move_set=False,history_dropout=False)
45                                mf.write(json.dumps(md,separators=(',',':'))+'\n');seen.add(key)
46                        board.push(move)
47                    states.append(c.position_key(board))
48                    # Any genuine occurrence through a frozen prefix group protects
49                    # every canonical state of that full source line.
50                    if any(k.hex() in groups for k in states[:9]):closure.update(states);heldout_lines+=1
51                    if line%25000==0:print(json.dumps(dict(book=book,lines=line,unique_contexts=len(seen))),flush=True)
52            bookstats[book]=dict(lines=line,heldout_prefix_group_lines=heldout_lines,sha256=sha(archive))
53    expected={(k,u) for k,v in edges.items() for u in v}
54    if occurrence_pairs!=expected or seen!=set(edges):raise ValueError('full transition coverage mismatch')
55    if not {bytes.fromhex(k) for k in valroots}<=closure:raise ValueError('validation root missing from closure')
56    np.asarray(sorted(closure),dtype='V32').tofile(OUT/'opening-protected-keys.bin')
57    data=np.memmap(tmprecords,dtype=DTYPE,mode='r');kept={0:[],1:[]};pairmap=[];pair_counts=collections.Counter();meta_count=0;nonpad=collections.Counter();excluded=0;metadata_path=OUT/'metadata.jsonl'
58    with metadata_path.open('w') as out:
59        for oldid,line in enumerate(tmpmeta.open()):
60            md=json.loads(line);key=bytes.fromhex(md['position_key']);split=1 if md['position_key'] in valroots else 0
61            if split==0 and key in closure:excluded+=1;continue
62            row=data[oldid].copy();row['game']=meta_count;row['split']=split;md['split']=split;md['metadata_index']=meta_count;out.write(json.dumps(md,separators=(',',':'))+'\n');meta_count+=1
63            local=len(kept[split]);kept[split].append(row);pair_counts[split]+=len(md['book_moves']);nonpad[split]+=int(row['attention'].sum())
64            if split==0:pairmap.extend((local,j) for j in range(len(md['book_moves'])))
65    files=[];ordinal=0
66    for split in (0,1):
67        for start in range(0,len(kept[split]),65536):
68            a=np.asarray(kept[split][start:start+65536],dtype=DTYPE);p=OUT/f'records-{ordinal:06d}.bin';a.tofile(p);files.append(dict(name=p.name,rows=len(a),source=2,split=split,sha256=sha(p)));ordinal+=1
69    pairdtype=np.dtype([('row','<u4'),('choice','<u2')]);np.asarray(pairmap,dtype=pairdtype).tofile(OUT/'train-pairs.bin')
70    shared=dict(position_ids=list(range(122)),loss_mask=[int(i in (119,120,121)) for i in range(122)],target_positions=[119,120,121]);atomic_json(OUT/'shared.json',shared)
71    identity=dict(kind='all_legal_book_transitions',code_sha256=sha(Path(__file__)),books=bookstats,historical_protection_sha256=sha(STATE/'manifest.json'),exact_protection_sha256=sha(ROOT/'exact-holdout-v1'/'protection.json'),opening_selection_sha256=sha(ROOT/'opening-engine-v1'/'frozen-selection.json'),opening_protected_keys_sha256=sha(OUT/'opening-protected-keys.bin'),engine_checks_used=False,frequency_gate=False)
72    counts=dict(raw_unique_states=len({k for k,u in raw_pairs}),raw_distinct_state_move_pairs=len(raw_pairs),after_historical_and_exact_states=len(edges),after_historical_and_exact_pairs=len(expected),opening_holdout_related_states_withheld=excluded,train_states=len(kept[0]),validation_states=len(kept[1]),train_state_move_pairs=pair_counts[0],validation_state_move_pairs=pair_counts[1],train_canonical_serialized_slots=123*len(kept[0]),train_pair_pass_serialized_slots=123*pair_counts[0],train_pair_pass_target_tokens=3*pair_counts[0],train_nonpadding_input_slots=nonpad[0],validation_nonpadding_input_slots=nonpad[1])
73    manifest=dict(format='chess-state-records-v2',identity=identity,identity_sha256=digest(identity),contract=c.CONTRACT,dtype=DTYPE.descr,record_bytes=416,files=files,rows=meta_count,metadata_sha256=sha(metadata_path),shared_sha256=sha(OUT/'shared.json'),pair_map_dtype=pairdtype.descr,pair_map_sha256=sha(OUT/'train-pairs.bin'),counts=counts,label_readiness='pending_independent_review',source_id_semantics={'2':'opening-book origin'},label_policy='every legal observed book transition; no frequency/engine gate; multiple alternatives preserved',opening_protection_keys=len(closure),training_launched=False,current_run_modified=False)
74    atomic_json(OUT/'manifest.json',manifest)
75    # Remove only this script's own redundant scratch after publication.
76    del data;tmprecords.unlink();tmpmeta.unlink();print(json.dumps(manifest),flush=True)
77if __name__=='__main__':main()
78