reader.py

1"""Three sealed sources and an explicit pair schedule; no PGN work in training."""
2import sys,json,hashlib
3from pathlib import Path
4import numpy as np
5H=Path(__file__).resolve().parent;P=H.parent;BUILDER=Path('/work/source/2026-10-29/task')
6sys.path[1:1]=[str(BUILDER),str(P/'state-v2')]
7from book_pair_loader import BookPairs
8from records import DTYPE,sha,Records as SmallRecords
9from runtime_records import Records
10sys.path.insert(0,str(H))
11PAIR=np.dtype([('source','u1'),('row','<u4'),('choice','<u2')])
12SD=np.dtype([('kind','u1'),('index','<u4')])
13class Data:
14    def __init__(self,manifest,verify=True):
15        self.manifest=json.loads(Path(manifest).read_text());m=self.manifest
16        for path,h in m['bindings'].items():
17            if sha(path)!=h:raise ValueError('followup input changed: '+path)
18        self.schedule=np.memmap(m['schedule'],dtype=SD,mode='r')
19        if sha(m['schedule'])!=m['schedule_sha256'] or len(self.schedule)!=m['decisions']:raise ValueError('schedule integrity')
20        self.book=BookPairs(m['book'],verify=verify)
21        self.exact_pairs=np.memmap(Path(m['replay'])/'exact-pairs.bin',dtype=PAIR,mode='r')
22        self.exact={s:SmallRecords(m['exact'],s,0,verify=verify) for s in (0,1)}
23        self.metadata=[json.loads(x) for x in (Path(m['exact'])/'metadata.jsonl').read_text().splitlines()]
24        self.elite=Records(m['ordinary'],1,0,verify=verify)
25        self.sources={('book',2):self.book.records,('exact',0):self.exact[0],('exact',1):self.exact[1],('elite',1):self.elite}
26    def take(self,ids):
27        ids=np.asarray(ids,dtype=np.int64)
28        if ids.ndim!=1 or np.any(ids<0) or np.any(ids>=len(self.schedule)):raise IndexError('schedule index')
29        e=self.schedule[ids];out=np.empty(len(ids),dtype=DTYPE)
30        if np.any(e['kind']>2):raise ValueError('unknown source kind')
31        for kind in (0,1,2):
32            mask=e['kind']==kind
33            if not mask.any():continue
34            indices=e['index'][mask]
35            if kind==0:out[mask]=self.book.take(indices)
36            elif kind==2:out[mask]=self.elite.take(indices)
37            else:
38                if np.any(indices>=len(self.exact_pairs)):raise IndexError('exact pair')
39                pairs=self.exact_pairs[indices];rows=np.empty(len(pairs),dtype=DTYPE)
40                if np.any(pairs['source']>1):raise ValueError('exact source')
41                for s in (0,1):
42                    selected=pairs['source']==s
43                    if selected.any():rows[selected]=self.exact[s].take(pairs['row'][selected])
44                for i,pair in enumerate(pairs):
45                    target=self.metadata[int(rows[i]['game'])]['tied_target_triples'][int(pair['choice'])]
46                    rows[i]['y']=target;rows[i]['x'][-2:]=target[:2]
47                out[mask]=rows
48        return out
49    def close(self):
50        self.book.records.close();self.elite.close()
51
52class Ledger:
53    def __init__(self,data):
54        self.data=data;self.book_states=len(data.book.records);self.exact0=len(data.exact[0]);self.exactn=self.exact0+len(data.exact[1])
55        self.pair_offsets=(0,len(data.book),len(data.book)+len(data.exact_pairs))
56        self.pair_n=self.pair_offsets[2]+len(data.elite)
57        self.state_n=self.book_states+self.exactn+len(data.elite)
58    def snapshot(self,cursor):
59        import torch
60        if not 0<=cursor<=len(self.data.schedule):raise ValueError('cursor')
61        pairs=np.zeros(self.pair_n,dtype=np.uint32);states=np.zeros(self.state_n,dtype=np.uint32)
62        for start in range(0,cursor,65536):
63            e=self.data.schedule[start:min(cursor,start+65536)]
64            if np.any(e['kind']>2):raise ValueError('schedule kind')
65            for kind in (0,1,2):
66                ids=e['index'][e['kind']==kind].astype(np.int64)
67                np.add.at(pairs,ids+self.pair_offsets[kind],1)
68                if kind==0:state=self.data.book.pairs[ids]['row'].astype(np.int64)
69                elif kind==1:
70                    p=self.data.exact_pairs[ids];state=self.book_states+p['row'].astype(np.int64)+p['source'].astype(np.int64)*self.exact0
71                else:state=self.book_states+self.exactn+ids
72                np.add.at(states,state,1)
73        if np.any(pairs[:self.pair_offsets[1]]>3) or np.any(pairs[self.pair_offsets[2]:]>1) or np.any(states[:self.book_states]>33) or np.any(states[self.book_states:self.book_states+self.exactn]>90):raise ValueError('exposure ceiling')
74        return {'cursor':cursor,'pair_counts':torch.from_numpy(pairs),'state_counts':torch.from_numpy(states),
75            'sha256':hashlib.sha256(pairs.tobytes()+states.tobytes()).hexdigest(),'exposures':int(pairs.astype(np.uint64).sum()),
76            'unique_pairs':int(np.count_nonzero(pairs)),'unique_states':int(np.count_nonzero(states))}
77    def verify(self,value):
78        import torch
79        actual=self.snapshot(value['cursor'])
80        if any(value[k]!=actual[k] for k in ('sha256','exposures','unique_pairs','unique_states')) or any(not torch.equal(value[k],actual[k]) for k in ('pair_counts','state_counts')):raise ValueError('exposure ledger mismatch')
81        return actual['cursor']
82