{
  "id": "paired-100-20261029",
  "title": "Foundation versus September: 100 games each",
  "date": "2026-10-29",
  "kind": "evaluation",
  "status": "completed",
  "hypothesis": "Compare two checkpoints under one frozen Sunfish protocol, separating model diagnostics from helper-assisted play.",
  "baseline": "September checkpoint 6133648b2926: 13 wins, 2 draws, 85 losses. Foundation checkpoint 857bc93bfffb: 21 wins, 9 draws, 69 losses and 1 unfinished at the 256-ply cap.",
  "data": "50 frozen opening prefixes, each played in both colors: 100 games per model, all 200 retained. 25 twenty-ply prefixes and 25 short prefixes. Matched validation: identical 7,000 September validation games / 1,595,704 prediction targets.",
  "config": {
    "suite_sha256": "0cd04bbde2fb3588e678f8c8c10946d65c1895c0beaefe28fc6d0bd1a93b55a6",
    "sunfish_commit": "e396948acbd80cdcbad3dccfb07e878fba9df8f4",
    "requested_nodes": 1000,
    "total_ply_cap": 256,
    "helper": "Same bounded three-ply helper for both models: 50,000 nodes / 1 second; NOT the later Rust iterative helper.",
    "book_autoplay": "Full standard-start prefix retained; excluded from model metrics."
  },
  "compute": "Matched FP32 legal-candidate scoring; fixed Sunfish node request can overshoot. Evaluation is separate from the four-hour training budget.",
  "results": [
    {
      "label": "Baseline matched validation NLL",
      "value": 0.6058686751172994
    },
    {
      "label": "Foundation matched validation NLL",
      "value": 0.5752401508322061
    },
    {
      "label": "Baseline raw-greedy legality (%)",
      "value": 92.32270657124269
    },
    {
      "label": "Foundation raw-greedy legality (%)",
      "value": 95.60163367891926
    },
    {
      "label": "Baseline helper overrides",
      "value": 20
    },
    {
      "label": "Foundation helper overrides",
      "value": 29
    }
  ],
  "limitations": [
    "The unfinished game remains unscored, never silently a draw. Candidate scheduled-100 score bounds are 25.5%–26.5%; baseline 14%.",
    "The original comparison report conservatively marks the comparison incomplete. These are observed results, not a general playing-strength or Elo conclusion.",
    "Legality is measured on different visited positions; it is not a matched-position causal comparison.",
    "Both policies include legal filtering and three-ply helper overrides; there is no model-only ablation. The newer Rust helper did not produce these results.",
    "The test split was not model-evaluated. Browser play still uses the September checkpoint and its three-ply helper."
  ],
  "checkpoint": "857bc93bfffb287dc67e2845a14c214a4691bc1a008dae1fa4d044e5fccc87d2",
  "sourceSnapshot": "foundation-20261029",
  "history": [],
  "evidence": [
    {
      "label": "Frozen comparison report",
      "sha256": "aa8359604e6c8e75cd02a481330e52fb1c676fc0bf7b54bd2366e7ad430fb7e8",
      "selectedFields": {
        "identical_protocol_verified": true,
        "all_games_scored_without_errors": false,
        "baseline": {
          "scheduled_games": 100,
          "audited_games": 100,
          "unattempted_or_unaudited": 0,
          "wins": 13,
          "draws": 2,
          "losses": 85,
          "unfinished": 0,
          "score_on_scored_games": 0.14,
          "fully_scored_pairs": 50,
          "pair_mean_score": 0.14,
          "pair_standard_error": 0.03857142857142857,
          "uncertainty_limit": "paired-opening standard error; small selected suite, not an Elo/generalization estimate",
          "model_queries": 3074,
          "helper_overrides": 20,
          "autoplay_excluded_from_model_metrics": true
        },
        "candidate": {
          "scheduled_games": 100,
          "audited_games": 100,
          "unattempted_or_unaudited": 0,
          "wins": 21,
          "draws": 9,
          "losses": 69,
          "unfinished": 1,
          "score_on_scored_games": 0.25757575757575757,
          "fully_scored_pairs": 49,
          "pair_mean_score": 0.2602040816326531,
          "pair_standard_error": 0.03923101147762679,
          "uncertainty_limit": "paired-opening standard error; small selected suite, not an Elo/generalization estimate",
          "model_queries": 3183,
          "helper_overrides": 29,
          "autoplay_excluded_from_model_metrics": true
        },
        "promotion_authorized": false,
        "comparison_status": "incomplete; no improvement conclusion",
        "evidence_for_improvement": false
      }
    }
  ]
}