evaluation.py
python · 30 lines · Sanitized derivative
Sanitized derivative: private paths or personal test identities use explicit placeholders. Original line count is preserved.
Version & provenance · cm8-training-v1
Recorded SHA-256 957c3151a23846b3ca1e32a8b61bba5ba627e5140b049f7e6b077c8e89585193Rendered SHA-256 549b95b3b8d342023409877c96dc3c028a3830ef31ea43d4d58fcc5b78bb185c
Original LittlePly code: Apache-2.0. External Rust helper remains a separate GPL component. License notes. Line links are stable within this snapshot. Frozen research implementation; host paths redacted. Original and rendered digests identify the derivative; not a portable launcher or complete training payload.
1import json,math
2from pathlib import Path
3import numpy as np
4import torch
5from stage_data import tensors,phase
6
7def evaluate(net,cfg,device):
8 root=Path(cfg['prepared_validation']);spec=json.loads((root/'manifest.json').read_text());groups={};weighted=0.;net.eval()
9 with torch.inference_mode():
10 for g in spec['groups']:
11 rows=np.load(root/(g['name']+'.npy'),allow_pickle=False);accepted=json.loads((root/(g['name']+'-accepted.json')).read_text());losses=[];hits=np.zeros(3);raw_hits=0;reserved=np.zeros(3);pieces=phase(rows)
12 for a in range(0,len(rows),64):
13 x,y,m=[v.to(device) for v in tensors(rows[a:a+64])]
14 with torch.autocast('cuda',dtype=torch.bfloat16,enabled=device.type=='cuda'):logits=net(x,m)
15 if not torch.isfinite(logits).all():raise FloatingPointError('validation logits')
16 losses.extend(torch.nn.functional.cross_entropy(logits.float().reshape(-1,512),y.reshape(-1),reduction='none').reshape(-1,3).sum(-1).cpu().tolist());hits+=(logits.argmax(-1)==y).sum(0).cpu().numpy()
17 probs=logits.float().softmax(-1);reserved+=(probs[:,:,72:128].sum(-1)+probs[:,:,379:].sum(-1)).sum(0).cpu().numpy()
18 xx=x[:,:120];mm=m[:,:120];pred=[]
19 for _ in range(3):
20 with torch.autocast('cuda',dtype=torch.bfloat16,enabled=device.type=='cuda'):z=net.next_logits(xx,mm)
21 if not torch.isfinite(z).all():raise FloatingPointError('raw logits')
22 t=z.argmax(-1);pred.append(t.cpu().tolist());xx=torch.cat([xx,t[:,None]],1);mm=torch.cat([mm,torch.ones_like(t[:,None],dtype=mm.dtype)],1)
23 raw_hits+=sum(list(p) in accepted[a+i] for i,p in enumerate(zip(*pred)))
24 ls=np.asarray(losses);nll=float(ls.mean()/3);weighted+=g['weight']*nll
25 groups[g['name']]={'decisions':len(rows),'nll':nll,'weight':g['weight'],'phase_nll':[float(ls[pieces==p].mean()/3) if np.any(pieces==p) else None for p in range(3)],'teacher_forced_token_accuracy':(hits/len(rows)).tolist(),'raw_accepted_move_accuracy':raw_hits/len(rows),'reserved_probability_teacher_forced':(reserved/len(rows)).tolist(),'accepted_move_definition':g['definition']}
26 net.train()
27 if not math.isfinite(weighted):raise FloatingPointError('validation NLL')
28 return {'nll':weighted,'groups':groups,'definition':spec['selection'],'accuracy_is_diagnostic_only':True}
29
30