curriculum_policy.py

1"""Offline preflight and deterministic selection policy. No downloads or training.
2
3Candidates are independently validated metadata, not raw FENs. This module does
4not compute legal moves, exact labels, repetition, or clock-aware outcomes.
5"""
6from collections import Counter
7from hashlib import sha256
8import re
9
10TABLE_CAP = 1_200_000_000
11TASK_CAP = 2_000_000_000
12FREE_RESERVE = 50_000_000_000
13SERIALIZED_TOKENS = 123
14MODEL_INPUT_TOKENS = 122
15TABLE_NAME = re.compile(r"K[QRBNP]*vK[QRBNP]*\.rtb[wz]\Z")
16
17
18def preflight(files, *, host_free, native_free, task_bytes, reusable=None):
19    """Require exact sizes for entire plan BEFORE any table payload download.
20
21    reusable maps verified existing filenames to sizes. The caller must hash and
22    verify them first. A failed/partial file is NOT reusable and counts as scratch.
23    The 1.2 GB ceiling applies conservatively to the complete table set.
24    """
25    reusable = reusable or {}
26    if any(type(v) is not int or v < 0 for v in
27           (host_free, native_free, task_bytes)):
28        raise ValueError('both filesystem free-space measurements are required')
29    names = set()
30    table_bytes = 0
31    missing_bytes = 0
32    for item in files:
33        name, size = item['name'], item['bytes']
34        if not TABLE_NAME.fullmatch(name):
35            raise ValueError('unsupported table name')
36        pieces = len(name.split('.')[0].replace('v', ''))
37        if not 3 <= pieces <= 5:
38            raise ValueError('only 3-5 pieces allowed')
39        if name in names or type(size) is not int or size <= 0:
40            raise ValueError('duplicate file or unknown size')
41        names.add(name)
42        table_bytes += size
43        if name in reusable:
44            if reusable[name] != size:
45                raise ValueError('reuse size mismatch')
46        else:
47            missing_bytes += size
48    if not names:
49        raise ValueError('empty table manifest')
50    if table_bytes > TABLE_CAP:
51        raise ValueError('table payload cap exceeded')
52    if task_bytes + missing_bytes > TASK_CAP:
53        raise ValueError('total task cap exceeded')
54    # Even if ext4 already has space, conservatively reserve full VHD growth.
55    if min(host_free, native_free) - missing_bytes < FREE_RESERVE:
56        raise ValueError('50 GB checkpoint headroom reserve')
57    return dict(table_bytes=table_bytes, new_bytes=missing_bytes,
58                projected_task_bytes=task_bytes + missing_bytes,
59                host_free_after_worst_case=host_free-missing_bytes,
60                native_free_after=native_free-missing_bytes)
61
62
63def select(candidates, *, protected_keys, protected_games, limit,
64           family_cap, source_kind, protection_verified, seed='curriculum-v1'):
65    """Select only after verified historical protection and candidate validation.
66
67    position_key must be the existing codec key (board/turn/castling/legal EP,
68    clocks and history ignored). Terminal and castling positions cannot become
69    tablebase targets. Exact outcome/move sets require upstream rule-aware probes.
70    Unknown repetition remains explicit; never invent history from a FEN.
71    """
72    if not protection_verified:
73        raise ValueError('historical protection not verified')
74    if source_kind not in ('human_main', 'human_elite', 'opening_supported', 'tablebase_exact'):
75        raise ValueError('label provenance required')
76    if type(limit) is not int or limit < 0 or type(family_cap) is not int or family_cap < 1:
77        raise ValueError('invalid selection bound')
78    eligible = []
79    rejected = Counter()
80    for c in candidates:
81        reason = None
82        if c.get('source_kind') != source_kind:
83            reason = 'provenance'
84        elif not c.get('legal_verified') or c.get('terminal', True):
85            reason = 'illegal_or_terminal'
86        elif c['position_key'] in protected_keys or set(c['game_aliases']) & protected_games:
87            reason = 'heldout'
88        elif source_kind == 'tablebase_exact' and (
89            not 3 <= c.get('pieces', 0) <= 5 or c.get('castling', True)
90            or not c.get('clock_verified') or not c.get('all_children_probed')
91            or c.get('wdl') not in (-2, -1, 0, 1, 2)
92            or type(c.get('dtz')) is not int
93            or c.get('repetition_status') not in ('known_clear', 'unknown')
94            or not c.get('preserving_moves')
95        ):
96            reason = 'unverified_tablebase_semantics'
97        elif source_kind == 'opening_supported' and c.get('distinct_game_support', 0) < 3:
98            reason = 'insufficient_game_support'
99        if reason:
100            rejected[reason] += 1
101        else:
102            eligible.append(c)
103    # Prefer practical occurrence, then critical decisions. Seeded stable order
104    # prevents input scan order deciding ties; UCI ties remain in each record.
105    def priority(c):
106        critical = 0 < len(c.get('preserving_moves', [])) <= 2
107        special = bool(c.get('promotion') or c.get('stalemate_risk'))
108        support = min(c.get('distinct_game_support', 0), 100)
109        return (-int(c.get('from_real_game', False)), -int(critical),
110                -int(special), -support,
111                sha256((seed + ':' + c['position_key']).encode()).hexdigest(),
112                str(c.get('candidate_id', '')))
113    selected, seen, families = [], set(), Counter()
114    for c in sorted(eligible, key=priority):
115        if c['position_key'] in seen:
116            rejected['duplicate'] += 1
117            continue
118        if len(selected) >= limit or families[c['material_family']] >= family_cap:
119            rejected['selection_cap'] += 1
120            continue
121        seen.add(c['position_key'])
122        families[c['material_family']] += 1
123        selected.append(c)
124    return selected, dict(rejected)
125