build_curriculum.py

1"""Bounded curriculum from sealed records; no training, no raw-data copies."""
2import collections,json,hashlib,shutil,time
3from pathlib import Path
4from inventory_inputs import ROOT,STATE,c,protection,sha
5from records import DTYPE,canonical,digest,atomic_json
6import numpy as np
7
8OUT=ROOT/'curriculum-v2'
9def phase(rows):
10    pieces=np.count_nonzero(rows['x'][:,51:115]!=128,axis=1)
11    return np.where(pieces<=10,2,np.where(rows['ply']<16,0,1)).astype(np.uint8)
12def guard(extra=0):
13    for p in (ROOT,Path(__file__).parent):
14        if shutil.disk_usage(p).free<50_000_000_000+extra:raise RuntimeError('free-space reserve')
15def inventory():
16    guard(6_000_000_000);db,_=protection();db.close()
17    m=json.loads((STATE/'manifest.json').read_text());stats=collections.Counter();nonpad=0
18    for f in m['files']:
19        p=STATE/f['name']
20        if sha(p)!=f['sha256']:raise ValueError('source hash')
21        rows=np.memmap(p,dtype=DTYPE,mode='r')
22        if len(rows)!=f['rows']:raise ValueError('source length')
23        if f['split']!=0:continue
24        for k,v in collections.Counter(phase(rows).tolist()).items():stats[f"{f['source']}:{k}"]+=v
25        nonpad+=int(rows['attention'].sum())
26    result=dict(source_manifest_sha256=sha(STATE/'manifest.json'),phase_counts=dict(stats),phase_ids={'0':'opening','1':'middlegame','2':'endgame <=10 pieces'},nonpadding_input_slots=nonpad)
27    Path(__file__).with_name('sealed-phase-inventory.json').write_text(json.dumps(result,indent=2));print(json.dumps(result),flush=True)
28def build():
29    from curriculum_loader import SCHEDULE_DTYPE
30    theory=ROOT/'theory-v3'
31    tm=json.loads((theory/'manifest.json').read_text())
32    if tm['label_readiness']!='independently_reviewed_ready':raise ValueError('theory review required')
33    if OUT.exists():raise ValueError('refuse overwrite')
34    guard(300_000_000);OUT.mkdir()
35    metadata=[json.loads(x) for x in (theory/'metadata.jsonl').read_text().splitlines()]
36    keys=np.array([bytes.fromhex(x['position_key']) for x in metadata],dtype='V32')
37    m=json.loads((STATE/'manifest.json').read_text());rng=np.random.default_rng(20261029)
38    ids={0:[],1:[]};offset={0:0,1:0};phases={0:[],1:[]};nonpad={0:[],1:[]}
39    for f in m['files']:
40        if f['split']!=0:continue
41        src=f['source'];rows=np.memmap(STATE/f['name'],dtype=DTYPE,mode='r')
42        keep=~np.isin(np.ascontiguousarray(rows['key']).view('V32').ravel(),keys)
43        ids[src].append(np.arange(offset[src],offset[src]+len(rows),dtype=np.uint32)[keep]);offset[src]+=len(rows)
44        phases[src].append(phase(rows)[keep]);nonpad[src].append(rows['attention'].sum(axis=1,dtype=np.uint16)[keep])
45    ids={s:np.concatenate(ids[s]) for s in (0,1)};n=min(map(len,ids.values()));stats={};input_nonpad=0;selected_phases={}
46    for src in (0,1):
47        p=np.concatenate(phases[src]);npad=np.concatenate(nonpad[src]);perm=rng.permutation(len(ids[src]))[:n]
48        ids[src]=ids[src][perm];stats[str(src)]={str(k):int(v) for k,v in collections.Counter(p[perm].tolist()).items()};input_nonpad+=int(npad[perm].sum())
49        selected_phases[src]=p[perm]
50        ids[src].tofile(OUT/f'ordinary-source{src}-indices.bin')
51    theory_n=tm['rows'];total=2*n+3*theory_n
52    schedule=np.memmap(OUT/'schedule.bin',dtype=SCHEDULE_DTYPE,mode='w+',shape=(total,));is_theory=np.zeros(total,dtype=bool)
53    theory_ids=np.concatenate([np.arange(f['rows'],dtype=np.uint32) for f in tm['files']]);theory_src=np.concatenate([np.full(f['rows'],f['source'],dtype=np.uint8) for f in tm['files']])
54    for exp in range(3):
55        lo=exp*total//3;hi=(exp+1)*total//3
56        places=lo+((np.arange(theory_n,dtype=np.int64)+1)*(hi-lo)//(theory_n+1))
57        perm=rng.permutation(theory_n);is_theory[places]=True
58        schedule['kind'][places]=1;schedule['source'][places]=theory_src[perm];schedule['row'][places]=theory_ids[perm];schedule['exposure'][places]=exp
59    ordinary_places=np.flatnonzero(~is_theory)
60    assert len(ordinary_places)==2*n
61    for src in (0,1):
62        places=ordinary_places[src::2];schedule['kind'][places]=0;schedule['source'][places]=src;schedule['row'][places]=ids[src];schedule['exposure'][places]=0
63    schedule.flush();del schedule
64    unique=2*n+theory_n;exposed=total
65    tags=np.memmap(OUT/'state-tags.bin',dtype='u1',mode='w+',shape=(unique,3))
66    for src in (0,1):tags[src*n:(src+1)*n,0]=src;tags[src*n:(src+1)*n,1]=selected_phases[src];tags[src*n:(src+1)*n,2]=0
67    tags[2*n:,0]=theory_src;tags[2*n:,1]=2;tags[2*n:,2]=2;tags.flush();del tags
68    stateids=np.memmap(OUT/'schedule-state-ids.bin',dtype='<u4',mode='w+',shape=(total,))
69    schedule=np.memmap(OUT/'schedule.bin',dtype=SCHEDULE_DTYPE,mode='r')
70    for src in (0,1):
71        mask=(schedule['kind']==0)&(schedule['source']==src);lookup=np.full(offset[src],2**32-1,dtype=np.uint32);lookup[ids[src]]=np.arange(src*n,(src+1)*n,dtype=np.uint32);stateids[mask]=lookup[schedule['row'][mask]]
72        mask=(schedule['kind']==1)&(schedule['source']==src);base=2*n+sum(f['rows'] for f in tm['files'] if f['source']<src);stateids[mask]=base+schedule['row'][mask]
73    stateids.flush();del stateids;del schedule
74    manifest=dict(format='littleply-indexed-curriculum-v1',readiness='pending_independent_review',roots=dict(ordinary=str(STATE),theory=str(theory)),dependencies=[dict(path=str(p),sha256=sha(p)) for p in [STATE/'manifest.json',theory/'manifest.json',theory/'metadata.jsonl',Path(__file__).with_name('curriculum_loader.py')]],schedule_dtype=SCHEDULE_DTYPE.descr,schedule_sha256=sha(OUT/'schedule.bin'),schedule_entries=total,ordinary_per_source=n,theory_unique=theory_n,theory_exposures=3,unique_decisions=unique,unique_serialized_slots=123*unique,unique_input_slots=122*unique,unique_supervised_targets=3*unique,unique_nonpadding_input_slots=input_nonpad+74*theory_n,unique_nonpadding_serialized_slots=input_nonpad+74*theory_n+unique,exposure_serialized_slots=123*exposed,exposure_input_slots=122*exposed,exposure_supervised_targets=3*exposed,exposure_nonpadding_input_slots=input_nonpad+74*theory_n*3,phase_counts=stats,phase_ids={'0':'opening >10pieces ply<16','1':'middlegame >10pieces ply>=16','2':'endgame <=10pieces'},phase_adjustment='retain available unique ordinary states; no duplication to fill15/60/25; equal sources after theory ownership exclusions',theory_spacing='one exposure in each global third, evenly spaced; distinct shuffled order; ordinarysource0/1alternate in remaining slots',tie_policy='exposure index modulo complete tied triples; update x[-2:] and y together',seed=20261029,validation_root=str(STATE),validation_split=1,heldout_journal=str(STATE/'journal.sqlite'),opening_verified_rows=0,ordinary_dropped_for_source_equality=abs(len(np.concatenate([np.array([],dtype=np.uint32)]))) if False else offset[0]+offset[1]-2*n-theory_n,estimated_hours_at_992=exposed/992/3600,training_launched=False)
75    assert unique*123<=2_000_000_000 and theory_n*123<=200_000_000
76    manifest['dependencies'].append(dict(path=str(Path(__file__)),sha256=sha(Path(__file__))))
77    from inventory_inputs import PROJECT
78    manifest['dependencies'].extend(dict(path=str(p),sha256=sha(p)) for p in [PROJECT/'state-v2'/'records.py',PROJECT/'state-v2'/'runtime_records.py',PROJECT/'state-v2'/'codec.py'])
79    manifest['sidecars']={name:dict(sha256=sha(OUT/name),bytes=(OUT/name).stat().st_size) for name in ('state-tags.bin','schedule-state-ids.bin','ordinary-source0-indices.bin','ordinary-source1-indices.bin')}
80    manifest['dense_state_id']='ordinary source0 selected-index order, then source1, then theory source0/source1 local row order'
81    manifest['state_tags_columns']=['source_u8','phase_u8','label_kind_u8'];manifest['label_kinds']={'0':'ordinary','1':'verified_opening','2':'exact_tablebase'}
82    validation=[]
83    for f in m['files']:
84        if f['split']!=1:continue
85        rows=np.memmap(STATE/f['name'],dtype=DTYPE,mode='r');v=np.zeros((len(rows),3),dtype='u1');v[:,0]=f['source'];v[:,1]=phase(rows);validation.append(v)
86    validation=np.concatenate(validation);validation.tofile(OUT/'validation-tags.bin')
87    manifest['validation_rows']=len(validation);manifest['validation_tag_order']='source manifest file order then row order, split1 only'
88    manifest['validation_phase_counts']={str(src):dict((str(k),int(v)) for k,v in collections.Counter(validation[validation[:,0]==src,1].tolist()).items()) for src in (0,1)}
89    manifest['sidecars']['validation-tags.bin']=dict(sha256=sha(OUT/'validation-tags.bin'),bytes=(OUT/'validation-tags.bin').stat().st_size)
90    manifest['ordinary_shuffle']='uniform deterministic permutation within each source, preserving measured available phase mix'
91    manifest['final_partial_global_batch']='drop without replacement or padding; report actual consumed schedule entries'
92    manifest['global_batch_512_usable_entries']=total//512*512
93    manifest['global_batch_512_dropped_tail']=total%512
94    atomic_json(OUT/'manifest.json',manifest);print(json.dumps(manifest),flush=True)
95
96if __name__=='__main__':
97    import sys
98    build() if '--build' in sys.argv else inventory()
99