inventory_inputs.py

1"""CPU-only inventory. No training output is published by this program."""
2import argparse, collections, hashlib, io, json, sqlite3, sys, time, zipfile
3from pathlib import Path
4from acquire_inputs import ROOT, BOOKS, guard, save, sha
5
6PROJECT=Path('/work/source/2026-10-27/task-14/causal-chess-v1')
7sys.path.insert(0,str(PROJECT/'state-v2'))
8sys.path.insert(0,str(PROJECT/'elite-continuation-v1'))
9import codec as c
10import chess.pgn, chess.syzygy
11from chessdata.reader import _read_shard
12STATE=Path('/work/causal-chess-state-v2-20261029')
13
14def protection():
15    m=json.loads((STATE/'manifest.json').read_text());done=json.loads((STATE/'completion.json').read_text())
16    if sha(STATE/'manifest.json')!=done['manifest_sha256']:raise ValueError('seal mismatch')
17    identity=json.dumps(m['identity'],sort_keys=True,separators=(',',':')).encode()
18    if hashlib.sha256(identity).hexdigest()!=m['identity_sha256']:raise ValueError('identity mismatch')
19    if sha(PROJECT/'state-v2'/'codec.py')!=m['identity']['code']['codec.py']:raise ValueError('codec changed')
20    config=m['identity']['config']
21    for r in config['references']:
22        if sha(Path(r['root'])/'manifest.json')!=r['manifest_sha256']:raise ValueError('historical source changed')
23    db=sqlite3.connect('file:'+str(STATE/'journal.sqlite')+'?mode=ro',uri=True)
24    db.execute('PRAGMA query_only=ON')
25    if db.execute("SELECT v FROM settings WHERE k='identity'").fetchone()[0]!=m['identity_sha256']:raise ValueError('protection identity')
26    if db.execute("SELECT v FROM progress WHERE k='protection_complete'").fetchone()!=('true',):raise ValueError('protection unfinished')
27    for table,field in [('protected','protected_positions'),('protected_games','protected_game_aliases')]:
28        if db.execute('SELECT COUNT(*) FROM '+table).fetchone()[0]!=done[field]:raise ValueError('protection count')
29    save('protection-receipt.json',dict(manifest_sha256=done['manifest_sha256'],identity_sha256=m['identity_sha256'],
30        protected_positions=done['protected_positions'],protected_game_aliases=done['protected_game_aliases'],
31        journal_open_mode='read-only; no copy',references_verified=config['references']))
32    return db,config
33
34def is_protected(db,key):return db.execute('SELECT 1 FROM protected WHERE k=?',(key,)).fetchone() is not None
35
36def aliases(meta):return ['seq:'+meta['sequence_sha256']]+['id:'+x for x in set(meta.get('game_ids',[])+([meta['game_id']] if meta.get('game_id') else []))]
37
38def replay(tokens,meta):
39    if tokens[0]!=c.BOS or tokens[-1]!=c.EOS or (len(tokens)-2)%3:raise ValueError('token contract')
40    board=c.chess.Board();boards=[];moves=[]
41    for start in range(1,len(tokens)-1,3):
42        move=c.decode(list(tokens[start:start+3]))
43        if board.is_game_over(claim_draw=False) or move not in board.legal_moves:raise ValueError('post-terminal or illegal game')
44        if len(board.piece_map())<=5:
45            boards.append((board.copy(stack=True),len(moves),move.uci()))
46        board.push(move);moves.append(move.uci())
47    if len(moves)!=meta['plies'] or board.fen()!=meta['final_fen'] or hashlib.sha256(' '.join(moves).encode()).hexdigest()!=meta['sequence_sha256']:
48        raise ValueError('source game identity')
49    return boards
50
51def books():
52    protected,_=protection()
53    path=ROOT/'opening-inventory.sqlite'
54    if path.exists():raise ValueError('opening inventory exists; do not overwrite')
55    db=sqlite3.connect(path)
56    db.executescript('''PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL;
57      CREATE TABLE positions(k BLOB PRIMARY KEY,fen TEXT,protected INTEGER) WITHOUT ROWID;
58      CREATE TABLE provenance(k BLOB,book TEXT,first_line INTEGER,first_ply INTEGER,n INTEGER,endpoints INTEGER,
59        PRIMARY KEY(k,book)) WITHOUT ROWID;
60      CREATE TABLE edges(k BLOB,move TEXT,book TEXT,n INTEGER,PRIMARY KEY(k,move,book)) WITHOUT ROWID;''')
61    counts={}
62    for name in BOOKS:
63        stats=collections.Counter();start=time.monotonic()
64        with zipfile.ZipFile(ROOT/(name+'.zip')) as z,io.TextIOWrapper(z.open(name),encoding='utf-8') as f:
65            while (game:=chess.pgn.read_game(f)) is not None:
66                stats['lines']+=1;line=stats['lines'];board=game.board();pending=[]
67                moves=list(game.mainline_moves())
68                if game.errors or type(board)!=c.chess.Board or board.chess960 or not board.is_valid():
69                    stats['rejected_lines']+=1;continue
70                valid=True
71                for ply,move in enumerate(moves):
72                    if board.is_game_over(claim_draw=False) or move not in board.legal_moves:valid=False;break
73                    pending.append((c.position_key(board),board.fen(en_passant='fen'),ply,move.uci(),0))
74                    board.push(move)
75                if not valid:stats['rejected_lines']+=1;continue
76                pending.append((c.position_key(board),board.fen(en_passant='fen'),len(moves),None,1))
77                for key,fen,ply,move,endpoint in pending:
78                    if not db.execute('SELECT 1 FROM positions WHERE k=?',(key,)).fetchone():
79                        db.execute('INSERT INTO positions VALUES(?,?,?)',(key,fen,int(is_protected(protected,key))))
80                    db.execute('INSERT INTO provenance VALUES(?,?,?,?,1,?) ON CONFLICT(k,book) DO UPDATE SET n=n+1,endpoints=endpoints+excluded.endpoints',
81                               (key,name,line,ply,endpoint))
82                    if move:db.execute('INSERT INTO edges VALUES(?,?,?,1) ON CONFLICT(k,move,book) DO UPDATE SET n=n+1',(key,move,name))
83                stats['legal_lines']+=1;stats['decision_occurrences']+=len(moves)
84                if line%5000==0:
85                    db.commit();guard(32_000_000)
86                    print(json.dumps(dict(book=name,lines=line,elapsed=time.monotonic()-start)),flush=True)
87        db.commit()
88        stats['unique_positions_including_endpoints']=db.execute('SELECT COUNT(*) FROM provenance WHERE book=?',(name,)).fetchone()[0]
89        stats['unique_endpoints']=db.execute('SELECT COUNT(*) FROM provenance WHERE book=? AND endpoints>0',(name,)).fetchone()[0]
90        stats['unprotected_positions_including_endpoints']=db.execute('SELECT COUNT(*) FROM provenance q JOIN positions p USING(k) WHERE q.book=? AND p.protected=0',(name,)).fetchone()[0]
91        counts[name]=dict(stats)
92    result=dict(sources=counts,unique_positions=db.execute('SELECT COUNT(*) FROM positions').fetchone()[0],
93        protected_unique_positions=db.execute('SELECT COUNT(*) FROM positions WHERE protected=1').fetchone()[0],
94        unique_decision_positions=db.execute('SELECT COUNT(DISTINCT k) FROM edges').fetchone()[0],
95        unprotected_unique_decision_positions=db.execute('SELECT COUNT(DISTINCT e.k) FROM edges e JOIN positions p USING(k) WHERE p.protected=0').fetchone()[0],
96        label_kind='opening_supported candidates, not verified optimal moves',
97        independent_human_frequency='not measured by book occurrence counts',training_records_published=0)
98    db.close();result['sqlite_bytes']=path.stat().st_size;save('opening-inventory.json',result);print(json.dumps(result),flush=True)
99
100def family(board):
101    sides=[]
102    for color in (True,False):
103        sides.append('K'+''.join(p*len(board.pieces(t,color)) for t,p in [(5,'Q'),(4,'R'),(3,'B'),(2,'N'),(1,'P')]))
104    return chess.syzygy.normalize_tablename('v'.join(sides))
105
106def raw_probe(tb,board,children=False,full_history=False):
107    c.check_board(board)
108    if board.castling_rights or len(board.piece_map())>5:raise ValueError('unsupported tablebase board')
109    if board.is_game_over(claim_draw=False):raise ValueError('terminal has no decision')
110    result=dict(fen=board.fen(en_passant='fen'),halfmove_clock=board.halfmove_clock,
111                raw_wdl=tb.probe_wdl(board),raw_dtz=tb.probe_dtz(board),
112                repetition_status='known_full_game_history' if full_history else 'unknown',
113                label_scope='raw Syzygy zero-counter semantics; not actual-clock root move labels')
114    if children:
115        out=[]
116        for move in board.legal_moves:
117            child=board.copy(stack=True);zero=child.is_zeroing(move);child.push(move)
118            out.append(dict(move=move.uci(),zeroing=zero,halfmove_clock=child.halfmove_clock,
119                            raw_child_wdl=tb.probe_wdl(child),raw_child_dtz=tb.probe_dtz(child),
120                            checkmate=child.is_checkmate(),stalemate=child.is_stalemate()))
121        result['children']=out
122        # Preserve ties without presenting them as clock-correct winning moves.
123        best=max(-r['raw_child_wdl'] for r in out)
124        result['raw_wdl_tied_moves']=[r['move'] for r in out if -r['raw_child_wdl']==best]
125    return result
126
127def endgames():
128    protected,config=protection();seen=set();rows=[];counts=collections.Counter();families=collections.Counter()
129    source_sequences=set();early_samples=[]
130    tb=chess.syzygy.open_tablebase(str(ROOT),max_fds=32)
131    start=time.monotonic()
132    for source,ref in enumerate(config['sources']):
133        root=Path(ref['root']);m=json.loads((root/'manifest.json').read_text());n=0
134        shards=[s for s in m['shards'] if s['split']=='train']
135        shards.sort(key=lambda s:hashlib.sha256(('curriculum-v1:'+','.join(sorted(s['files']))).encode()).digest())
136        for shard in shards:
137            tokens,offsets,metas=_read_shard(root,shard)
138            for a,b,meta in zip(offsets,offsets[1:],metas):
139                if n>=2000 or len(rows)>=20000 or time.monotonic()-start>1200:break
140                n+=1;counts['source'+str(source)+'_games_scanned']+=1
141                if any(protected.execute('SELECT 1 FROM protected_games WHERE k=?',(x,)).fetchone() for x in aliases(meta)):
142                    counts['heldout_games']+=1;continue
143                try:boards=replay(tokens[a:b],meta)
144                except ValueError:counts['invalid_or_post_terminal_games']+=1;continue
145                if meta['sequence_sha256'] in source_sequences:counts['duplicate_games']+=1;continue
146                source_sequences.add(meta['sequence_sha256'])
147                early=c.chess.Board()
148                for j in range(a+1,min(b-1,a+1+16*3),3):
149                    move=c.decode(list(tokens[j:j+3]));key=c.position_key(early)
150                    early_samples.append([key.hex(),meta['sequence_sha256'],source,move.uci()])
151                    early.push(move)
152                for board,ply,move in boards:
153                    if len(rows)>=20000:break
154                    counts['endgame_occurrences']+=1;key=c.position_key(board)
155                    if is_protected(protected,key):counts['protected_positions']+=1;continue
156                    if key in seen:counts['duplicate_positions']+=1;continue
157                    if board.castling_rights:counts['unsupported_castling']+=1;continue
158                    seen.add(key);fam=family(board);families[fam]+=1
159                    probe=raw_probe(tb,board,children=len(rows)<250,full_history=True)
160                    probe.update(position_key=key.hex(),source_kind='human_main' if source==0 else 'human_elite',
161                        source_sequence_sha256=meta['sequence_sha256'],game_aliases=aliases(meta),ply=ply,
162                        actual_move=move,material_family=fam,history_uci=[m.uci() for m in c.genuine_history(board)],
163                        can_claim_fifty_moves=board.can_claim_fifty_moves(),can_claim_threefold=board.can_claim_threefold_repetition())
164                    rows.append(probe);counts['raw_wdl_'+str(probe['raw_wdl'])]+=1
165                    counts['zero_halfmove' if board.halfmove_clock==0 else 'nonzero_halfmove']+=1
166                if n>=2000 or len(rows)>=20000 or time.monotonic()-start>1200:break
167            print(json.dumps(dict(source=source,games=n,unique=len(rows),elapsed=time.monotonic()-start)),flush=True)
168            guard(32_000_000)
169            if n>=2000 or len(rows)>=20000 or time.monotonic()-start>1200:break
170        if len(rows)>=20000 or time.monotonic()-start>1200:break
171    guard(32_000_000)
172    p=ROOT/'endgame-candidates.jsonl'
173    with p.open('x') as f:
174        for row in rows:f.write(json.dumps(row,separators=(',',':'))+'\n')
175    result=dict(counts=dict(counts),unique_unprotected_candidates=len(rows),material_families=dict(families.most_common()),
176        candidate_file_bytes=p.stat().st_size,candidate_sha256=sha(p),elapsed_seconds=time.monotonic()-start,
177        source_game_cap_per_source=2000,actual_clock_policy_verified=False,training_records_published=0,
178        limitations=['bounded deterministic shard sample, not population frequency',
179                    'raw WDL/DTZ and child ties are not yet actual-clock exact policy labels'])
180    save('human-opening-sample.json',dict(games=len(source_sequences),rows=early_samples))
181    save('endgame-inventory.json',result);print(json.dumps(result),flush=True);tb.close()
182
183def support():
184    sample=json.loads((ROOT/'human-opening-sample.json').read_text())
185    opening=sqlite3.connect(ROOT/'opening-inventory.sqlite')
186    opening.execute('CREATE TABLE IF NOT EXISTS human_support(k BLOB,sequence TEXT,source INTEGER,move TEXT,PRIMARY KEY(k,sequence)) WITHOUT ROWID')
187    for k,seq,src,move in sample['rows']:
188        key=bytes.fromhex(k)
189        if opening.execute('SELECT 1 FROM positions WHERE k=? AND protected=0',(key,)).fetchone():
190            opening.execute('INSERT OR IGNORE INTO human_support VALUES(?,?,?,?)',(key,seq,src,move))
191    opening.commit()
192    supported=list(opening.execute('SELECT hex(k),COUNT(*) FROM human_support GROUP BY k HAVING COUNT(*)>=3 ORDER BY COUNT(*) DESC,k'))
193    save('opening-human-support.json',dict(sample_games=sample['games'],positions_with_at_least_three_games=len(supported),
194        positions_with_any_game=opening.execute('SELECT COUNT(DISTINCT k) FROM human_support').fetchone()[0],
195        supported_positions=[dict(position_key=k.lower(),distinct_games=n) for k,n in supported],
196        scope='first 16 plies of bounded existing-game sample; unprotected opening positions only',optimal_move_verified=False))
197    opening.close()
198
199def children():
200    """Complete raw child inventory without claiming rule-aware move policy."""
201    inventory=json.loads((ROOT/'endgame-inventory.json').read_text())
202    path=ROOT/'endgame-candidates.jsonl'
203    if sha(path)!=inventory['candidate_sha256']:raise ValueError('candidate identity changed')
204    temp=ROOT/'endgame-candidates.enriched.tmp';counts=collections.Counter()
205    tb=chess.syzygy.open_tablebase(str(ROOT),max_fds=32)
206    guard(100_000_000)
207    with path.open() as src,temp.open('x') as out:
208        for n,line in enumerate(src,1):
209            r=json.loads(line)
210            if 'children' not in r:
211                probe=raw_probe(tb,c.chess.Board(r['fen']),children=True)
212                r['children']=probe['children'];r['raw_wdl_tied_moves']=probe['raw_wdl_tied_moves']
213            counts['positions_with_at_most_two_raw_wdl_tied_moves']+=len(r['raw_wdl_tied_moves'])<=2
214            counts['positions_with_promotion_move']+=any(len(x['move'])==5 for x in r['children'])
215            counts['positions_with_stalemating_move']+=any(x['stalemate'] for x in r['children'])
216            out.write(json.dumps(r,separators=(',',':'))+'\n')
217            if n%1000==0:
218                out.flush();guard(10_000_000)
219                print(json.dumps(dict(child_positions=n)),flush=True)
220    tb.close();temp.replace(path)
221    inventory.update(candidate_file_bytes=path.stat().st_size,candidate_sha256=sha(path),
222        raw_child_inventory=dict(counts),all_candidate_children_probed=True)
223    save('endgame-inventory.json',inventory)
224
225if __name__=='__main__':
226    p=argparse.ArgumentParser();p.add_argument('phase',choices=['books','endgames','support','children']);args=p.parse_args();globals()[args.phase]()
227