verified_training_data.py

python · 25 lines · Exact bytes

Version & provenance · foundation-20261029

Recorded SHA-256 fa00eb60a7c4c2061706ef94016f52bdef6cc04cbb2492ce8178dc02568fc71bRendered SHA-256 33f3c5376a9c96a08438ace6266bde24f8d85a90a92f99bafde9cc42eecd2dbc

Original project code: Apache-2.0. Dependency licenses remain separate. License notes. Line links are stable within this snapshot. Allowlisted foundation training and frozen evaluation source; recorded hashes refer to original files, rendered hashes identify any path-redacted derivatives. This subset is not a standalone training distribution; no host configs, credentials, optimizer state or dataset bulk included.

1"""Fast training load after separately completed full legal reader validation."""
2import json
3from pathlib import Path
4from chessdata.convert import file_hash
5from chessdata.reader import _manifest,_read_shard
6
7
8def load(directory,receipt_path):
9    root=Path(directory);receipt=json.loads(Path(receipt_path).read_text())
10    required=('all_file_hashes_and_counts','all_games_legal_replayed','all_sequence_hashes_and_final_fens_match','deterministic_splits_verified')
11    if receipt.get('status')!='verified' or any(receipt.get(k) is not True for k in required):raise ValueError('incomplete integrity receipt')
12    if receipt.get('manifest_sha256')!=file_hash(root/'manifest.json'):raise ValueError('integrity receipt manifest mismatch')
13    manifest=_manifest(root)  # Rehash every actual shard, including held-out test.
14    if receipt.get('dataset_fingerprint')!=manifest['fingerprint']:raise ValueError('receipt fingerprint mismatch')
15    if manifest['provenance'].get('backend') is not None and receipt.get('backend')!=manifest['provenance']['backend']:raise ValueError('integrity receipt backend mismatch')
16    data={'train':[],'validation':[]}
17    for shard in manifest['shards']:
18        if shard['split'] not in data:continue
19        tokens,offsets,_=_read_shard(root,shard)
20        for a,b in zip(offsets,offsets[1:]):
21            value=tokens[a:b]
22            if not 2<=len(value)<=1025 or value[0]!=69 or value[-1]!=70:raise ValueError('full history boundary/length mismatch')
23            data[shard['split']].append(value)
24    return data
25