prepare_openings_v1.py

1"""Bounded genuine-prefix book candidates; frozen split before engine scores."""
2import collections,hashlib,io,json,sqlite3,zipfile
3from pathlib import Path
4import numpy as np
5from inventory_inputs import ROOT,STATE,c,protection,sha
6from records import atomic_json
7from build_curriculum import guard
8import chess.pgn
9OUT=ROOT/'opening-engine-v1'
10SEED='opening-engine-v1-20261029'
11
12def choose_pool(rows,priority,cap=1024):
13    bykey={r['key']:r for r in rows};chosen=[bykey[k] for k in sorted(priority) if k in bykey]
14    used={r['key'] for r in chosen};groups=collections.defaultdict(list)
15    for r in rows:
16        if r['key'] not in used:groups[(r['ply']//4,r['fen'].split()[1],r['book_count']>1)].append(r)
17    for group in groups.values():group.sort(key=lambda r:hashlib.sha256((SEED+r['key']).encode()).digest())
18    cursors=collections.Counter()
19    while len(chosen)<cap:
20        added=False
21        for k in sorted(groups):
22            if cursors[k]<len(groups[k]):chosen.append(groups[k][cursors[k]]);cursors[k]+=1;added=True
23            if len(chosen)>=cap:break
24        if not added:break
25    return chosen
26
27def main():
28    if OUT.exists():raise ValueError('refuse overwrite')
29    guard(150_000_000);OUT.mkdir();protected,_=protection()
30    extra=json.loads((ROOT/'exact-holdout-v1'/'protection.json').read_text());excludedkeys={x.tobytes() for x in np.fromfile(ROOT/'exact-holdout-v1'/'protected-keys.bin',dtype='V32')};excludedseq=set(extra['source_sequences'])
31    db=sqlite3.connect('file:'+str(ROOT/'opening-inventory.sqlite')+'?mode=ro',uri=True)
32    original=json.loads((ROOT/'opening-supported-candidates.json').read_text())['candidates'];priority={m['position_key'] for m in original};human={m['position_key']:sorted(set(m['source_sequence_sha256'])-excludedseq) for m in original}
33    rows=[]
34    for key,fen,ply,nbooks in db.execute('SELECT p.k,p.fen,MIN(v.first_ply),COUNT(DISTINCT v.book) FROM positions p JOIN provenance v USING(k) WHERE p.protected=0 AND EXISTS(SELECT 1 FROM edges e WHERE e.k=p.k) GROUP BY p.k'):
35        if key in excludedkeys:continue
36        rows.append(dict(key=key.hex(),fen=fen,ply=ply,book_count=nbooks))
37    pool=choose_pool(rows,priority);targets={r['key']:r for r in pool};found={};bookhashes={}
38    for name in ('8moves_v3.pgn','bjbraams_chessdb_198350_lines.pgn'):
39        archive=ROOT/(name+'.zip');bookhashes[archive.name]=sha(archive)
40        with zipfile.ZipFile(archive) as z,io.TextIOWrapper(z.open(name),encoding='utf-8') as handle:
41            lineno=0
42            while (game:=chess.pgn.read_game(handle)) is not None:
43                lineno+=1
44                if game.errors:raise ValueError('invalid upstream line')
45                b=game.board();startfen=b.fen(en_passant='fen');moves=list(game.mainline_moves());states=[];boards=[]
46                for move in moves:
47                    if move not in b.legal_moves:raise ValueError('illegal upstream move')
48                    states.append(c.position_key(b));boards.append(b.copy(stack=True));b.push(move)
49                states.append(c.position_key(b))
50                for ply,board in enumerate(boards):
51                    key=states[ply].hex()
52                    if key not in targets or key in found:continue
53                    if board.is_game_over(claim_draw=True) or board.is_repetition(2):continue
54                    if protected.execute('SELECT 1 FROM protected WHERE k=?',(states[ply],)).fetchone():raise ValueError('historical heldout root')
55                    bookmoves=sorted({r[0] for r in db.execute('SELECT move FROM edges WHERE k=?',(states[ply],))})
56                    prior=board.copy(stack=True)
57                    for _ in range(min(16,len(prior.move_stack))):prior.pop()
58                    # Genuine root at ply8 (or current state for a shorter prefix).
59                    group=states[min(8,ply)].hex()
60                    found[key]=dict(position_key=key,fen=board.fen(en_passant='fen'),book=name,book_line=lineno,source_book_start_fen=startfen,source_book_line_uci=[m.uci() for m in moves],source_book_line_keys=[k.hex() for k in states],source_ply=ply,prefix_group=group,history_start_fen=prior.fen(en_passant='fen'),history_uci=[m.uci() for m in c.genuine_history(board)],book_moves=bookmoves,book_provenance=[dict(book=r[0],first_line=r[1],first_ply=r[2],occurrences=r[3],endpoints=r[4]) for r in db.execute('SELECT book,first_line,first_ply,n,endpoints FROM provenance WHERE k=?',(states[ply],))],human_support_sequences=human.get(key,[]),original_priority=key in priority,label_kind='approximate_engine_screened_opening',engine_status='not_yet_evaluated')
61        print(json.dumps(dict(book=name,lines=lineno,found=len(found))),flush=True)
62    candidates=[found[k] for k in sorted(found)];groups=collections.defaultdict(list)
63    for r in candidates:groups[r['prefix_group']].append(r)
64    validation_groups=set();n=0
65    for g in sorted(groups,key=lambda k:hashlib.sha256((SEED+':validation:'+k).encode()).digest()):
66        if n>=128:break
67        validation_groups.add(g);n+=len(groups[g])
68    valkeys={k for r in candidates if r['prefix_group'] in validation_groups for k in r['source_book_line_keys']}
69    kept=[];removed=0
70    for r in candidates:
71        if r['prefix_group'] in validation_groups:r['split']=1
72        elif r['position_key'] in valkeys:removed+=1;continue
73        else:r['split']=0
74        kept.append(r)
75    (OUT/'candidates.jsonl').write_text(''.join(json.dumps(r)+'\n' for r in kept))
76    atomic_json(OUT/'frozen-selection.json',dict(seed=SEED,cap=1024,priority_requested=33,priority_surviving_new_holdouts=sum(r['original_priority'] for r in candidates),pool_size=len(pool),recovered=len(candidates),frozen_rows=len(kept),split_counts=dict(collections.Counter(r['split'] for r in kept)),validation_prefix_groups=sorted(validation_groups),validation_line_keys=sorted(valkeys),train_removed_validation_line_transpositions=removed,book_hashes=bookhashes,candidates_sha256=sha(OUT/'candidates.jsonl'),historical_protection_manifest_sha256=sha(STATE/'manifest.json'),new_protection_sha256=sha(ROOT/'exact-holdout-v1'/'protection.json'),code_sha256=sha(Path(__file__)),model_results_used=False))
77    print((OUT/'frozen-selection.json').read_text()[:1000],flush=True)
78if __name__=='__main__':main()
79