verified_training_data.py

python · 24 lines · Exact bytes

Version & provenance · month-20261028

Recorded SHA-256 ad3952fe45d8721d05dbe80d6f077e69a4851a3fae7ee5114def2e33ccc57037Rendered SHA-256 ad3952fe45d8721d05dbe80d6f077e69a4851a3fae7ee5114def2e33ccc57037

Apache-2.0 for original LittlePly code, declared by the project owner on 2026-10-28. Historical source bytes and hashes are unchanged. Third-party components retain their own licenses; Lichess data remain CC0-1.0. License notes. Line links are stable within this snapshot. SHA-256 compared to persisted run identity/manifest. Unavailable files are never replaced with newer versions. Rendered derivatives preserve line numbers and redact private paths; original and rendered hashes are distinct.

1"""Fast training load after separately completed full legal reader validation."""
2import json
3from pathlib import Path
4from chessdata.convert import file_hash
5from chessdata.reader import _manifest,_read_shard
6
7
8def load(directory,receipt_path):
9    root=Path(directory);receipt=json.loads(Path(receipt_path).read_text())
10    required=('all_file_hashes_and_counts','all_games_legal_replayed','all_sequence_hashes_and_final_fens_match','deterministic_splits_verified')
11    if receipt.get('status')!='verified' or any(receipt.get(k) is not True for k in required):raise ValueError('incomplete integrity receipt')
12    if receipt.get('manifest_sha256')!=file_hash(root/'manifest.json'):raise ValueError('integrity receipt manifest mismatch')
13    manifest=_manifest(root)  # Rehash every actual shard, including held-out test.
14    if receipt.get('dataset_fingerprint')!=manifest['fingerprint']:raise ValueError('receipt fingerprint mismatch')
15    data={'train':[],'validation':[]}
16    for shard in manifest['shards']:
17        if shard['split'] not in data:continue
18        tokens,offsets,_=_read_shard(root,shard)
19        for a,b in zip(offsets,offsets[1:]):
20            value=tokens[a:b]
21            if not 2<=len(value)<=1025 or value[0]!=69 or value[-1]!=70:raise ValueError('full history boundary/length mismatch')
22            data[shard['split']].append(value)
23    return data
24