stage_data.py

python · 40 lines · Sanitized derivative

Sanitized derivative: private paths or personal test identities use explicit placeholders. Original line count is preserved.

Version & provenance · cm8-training-v1

Recorded SHA-256 b8cc23345901d1f952d12f0b69bc84524a4b2c2e5decef4378ff149be2e96196Rendered SHA-256 3fadd11a17ed2dae13136b5ec3123f1868b2a3a415a4d6dcb39738bab3861740

Original LittlePly code: Apache-2.0. External Rust helper remains a separate GPL component. License notes. Line links are stable within this snapshot. Frozen research implementation; host paths redacted. Original and rendered digests identify the derivative; not a portable launcher or complete training payload.

1import sys,time,hashlib
2from pathlib import Path
3import numpy as np
4import torch
5H=Path(__file__).resolve().parent;P=H.parent
6sys.path.insert(0,str(P/'cm6-curriculum-v1'))
7from composite import Plan as BasePlan,start_prefetch,phase,tensors
8sys.path.insert(0,str(P/'state-v2'))
9from records import sha,digest
10
11def open_curriculum(root,adapter_directory):
12 sys.path.insert(0,str(adapter_directory))
13 from newrun_loader import RandomData
14 return RandomData(Path(root)/'manifest.json',verify=True)
15class Plan(BasePlan):
16 def load(self,step,pin):
17  began=time.perf_counter();ids=self.indices(step);rows=self.data.take(ids);tags=self.data.schedule[ids]['kind'].astype(np.int64)*3+phase(rows)
18  return tensors(rows,pin),torch.from_numpy(np.ascontiguousarray(tags)),time.perf_counter()-began
19class ExposureLedger:
20 def __init__(self,data):
21  self.data=data;self.po=[];self.so=[];p=s=0
22  for k in range(4):
23   self.po.append(p);self.so.append(s);p+=len(data.pairs[k]) if k<2 else len(data.sources[k]);s+=len(data.sources[k])
24  self.pn=p;self.sn=s
25 def snapshot(self,cursor):
26  if not 0<=cursor<=len(self.data.schedule):raise ValueError('cursor')
27  pairs=np.zeros(self.pn,dtype=np.uint32);states=np.zeros(self.sn,dtype=np.uint32);source=np.zeros(4,dtype=np.int64)
28  for start in range(0,cursor,65536):
29   e=self.data.schedule[start:min(cursor,start+65536)]
30   if np.any(e['kind']>3):raise ValueError('kind')
31   for k in range(4):
32    ids=e['index'][e['kind']==k].astype(np.int64);source[k]+=len(ids)
33    np.add.at(pairs,self.po[k]+ids,1);rows=self.data.pairs[k]['row'][ids] if k<2 else ids;np.add.at(states,self.so[k]+rows,1)
34  return {'cursor':cursor,'pair_counts':torch.from_numpy(pairs),'state_counts':torch.from_numpy(states),'source_counts':source.tolist(),'sha256':hashlib.sha256(pairs.tobytes()+states.tobytes()).hexdigest(),'unique_pairs':int(np.count_nonzero(pairs)),'unique_states':int(np.count_nonzero(states))}
35 def verify(self,v):
36  a=self.snapshot(v['cursor'])
37  if any(a[k]!=v[k] for k in ['sha256','unique_pairs','unique_states','source_counts']) or any(not torch.equal(a[k],v[k]) for k in ['pair_counts','state_counts']):raise ValueError('exposure mismatch')
38  return a['cursor']
39
40