book_pair_loader.py

python · 25 lines · Exact bytes

Version & provenance · data-cm7

Recorded SHA-256 04a55b0b7310a305cb40988355121e04e79d77605d235d26a1b4884afb13da04Rendered SHA-256 04a55b0b7310a305cb40988355121e04e79d77605d235d26a1b4884afb13da04

Apache-2.0 original project code; dependencies retain their own licenses. License notes. Line links are stable within this snapshot. CM-7 completed implementation source; original and public rendered hashes recorded. Host-specific paths use /work placeholders; inspect reproduction instructions before running. Separate Rust helper retains its GPL license.

1"""Every observed book continuation is an explicit training pair."""
2import json
3from pathlib import Path
4import numpy as np
5from build_curriculum import DTYPE,sha
6from runtime_records import Records
7PAIR_DTYPE=np.dtype([('row','<u4'),('choice','<u2')])
8class BookPairs:
9    def __init__(self,root,verify=True):
10        self.root=Path(root);m=json.loads((self.root/'manifest.json').read_text())
11        if m['label_readiness']!='independently_reviewed_ready':raise ValueError('book dataset not reviewed')
12        if sha(self.root/'metadata.jsonl')!=m['metadata_sha256']:raise ValueError('book alternatives changed')
13        if sha(self.root/'train-pairs.bin')!=m['pair_map_sha256']:raise ValueError('pair map changed')
14        self.records=Records(self.root,2,0,verify=verify)
15        self.pairs=np.memmap(self.root/'train-pairs.bin',dtype=PAIR_DTYPE,mode='r')
16        self.targets=[np.asarray(json.loads(line)['tied_target_triples'],dtype=np.uint16) for line in (self.root/'metadata.jsonl').open()]
17    def __len__(self):return len(self.pairs)
18    def take(self,indices):
19        ids=np.asarray(indices,dtype=np.int64)
20        if ids.ndim!=1 or np.any(ids<0) or np.any(ids>=len(self.pairs)):raise IndexError('book pair index')
21        entries=self.pairs[ids];rows=self.records.take(entries['row'])
22        for i,e in enumerate(entries):
23            target=self.targets[int(rows[i]['game'])][int(e['choice'])];rows[i]['y']=target;rows[i]['x'][-2:]=target[:2]
24        return rows
25