{
  "schema_version": 1,
  "date_utc": "2026-10-06",
  "author": "Cisco Caceres",
  "scope": "developer-run local engineering pilot; not H1, frontier-model research result, external peer review or independent replication",
  "benchmark": "HumanEval (2021), MIT; 120 fixed pilot tasks and original 49-task public-qualified subset",
  "generation_model": {
    "name": "qwen3-coder:30b",
    "ollama_digest": "06c1097efce0431c2045fe7b2e5108366e43bee1b4603a7aded8f21689e90bca"
  },
  "verifier_model": {
    "name": "qwen3.5:9b",
    "ollama_digest": "6488c96fa5faab64bb65cbd30d4289e20e6130ef535a93ef9a49f42eda893ea7"
  },
  "collector_source_commit": "ab5cd6b56c2d38059e09687ffc12b8cec6778d4e",
  "postcollection_source_sha256": "97b51ff263ab733ab752369b321c37e58ad9bf3598c83f611e6dbccd8750f8c5",
  "collection_usage": {
    "requests": 960,
    "prompt_tokens": 276882,
    "returned_tokens": 100216,
    "judge_abstentions": 0,
    "external_usd": 0,
    "local_electricity_unmeasured": true
  },
  "generation_contract_failures": 4,
  "grading_slots": {
    "hidden": 480,
    "public": 196,
    "automatic_contract_failure_slots": 4
  },
  "candidate_passes": {
    "hidden": 437,
    "hidden_total": 480,
    "public": 188,
    "public_total": 196
  },
  "prospective_controls": {
    "hidden_passed": 120,
    "hidden_total": 120,
    "public_passed": 51,
    "public_total": 57,
    "original_public_subset": 49,
    "subset_expanded": false,
    "historical_case_byte_parity_claimed": false
  },
  "decision_count": 9288,
  "policy_seeds": [
    0,
    1,
    2
  ],
  "counterfactual_cached_replay": true,
  "single_comparator": "unfiltered first candidate in each task; on both task subsets",
  "public_subset_other_policies": "use public-test eligibility; no hidden tests enter selection",
  "results": [
    {
      "public_filter": false,
      "cap_tokens": 4608,
      "policy": "single",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 394.975,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 4608,
      "policy": "best_of_n",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 566.7166666666667,
      "cap_exhaustion_count": 360,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 4608,
      "policy": "soft_best_of_n",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 566.7166666666667,
      "cap_exhaustion_count": 360,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 4608,
      "policy": "bounded_best_of_poisson",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 566.7166666666667,
      "cap_exhaustion_count": 360,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 9216,
      "policy": "single",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 394.975,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 9216,
      "policy": "best_of_n",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 3109.391666666667,
      "cap_exhaustion_count": 9,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 9216,
      "policy": "soft_best_of_n",
      "tasks": 120,
      "success": 0.913888888888889,
      "paired_task_counts": {
        "candidate_better": 1,
        "candidate_worse": 0,
        "tied": 119
      },
      "mean_consumed_tokens": 3109.391666666667,
      "cap_exhaustion_count": 9,
      "paired_vs_single_descriptive": {
        "effect": 0.005555555555555555,
        "ci95": [
          0.0,
          0.016666666666666666
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 9216,
      "policy": "bounded_best_of_poisson",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 3109.391666666667,
      "cap_exhaustion_count": 9,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 18432,
      "policy": "single",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 394.975,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 18432,
      "policy": "best_of_n",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 3142.483333333333,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 18432,
      "policy": "soft_best_of_n",
      "tasks": 120,
      "success": 0.913888888888889,
      "paired_task_counts": {
        "candidate_better": 1,
        "candidate_worse": 0,
        "tied": 119
      },
      "mean_consumed_tokens": 3142.483333333333,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.005555555555555555,
        "ci95": [
          0.0,
          0.016666666666666666
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 18432,
      "policy": "bounded_best_of_poisson",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 3142.483333333333,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 36864,
      "policy": "single",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 394.975,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 36864,
      "policy": "best_of_n",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 3142.483333333333,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 36864,
      "policy": "soft_best_of_n",
      "tasks": 120,
      "success": 0.913888888888889,
      "paired_task_counts": {
        "candidate_better": 1,
        "candidate_worse": 0,
        "tied": 119
      },
      "mean_consumed_tokens": 3142.483333333333,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.005555555555555555,
        "ci95": [
          0.0,
          0.016666666666666666
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": false,
      "cap_tokens": 36864,
      "policy": "bounded_best_of_poisson",
      "tasks": 120,
      "success": 0.9083333333333333,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 120
      },
      "mean_consumed_tokens": 3142.483333333333,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 120,
        "clusters": 120,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 4608,
      "policy": "single",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 328.0816326530612,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 4608,
      "policy": "best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 541.8775510204082,
      "cap_exhaustion_count": 147,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 4608,
      "policy": "soft_best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 541.8775510204082,
      "cap_exhaustion_count": 147,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 4608,
      "policy": "bounded_best_of_poisson",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 541.8775510204082,
      "cap_exhaustion_count": 147,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 4608,
      "policy": "first_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 328.0816326530612,
      "cap_exhaustion_count": 6,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 4608,
      "policy": "uniform_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 328.0816326530612,
      "cap_exhaustion_count": 147,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 9216,
      "policy": "single",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 328.0816326530612,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 9216,
      "policy": "best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 9216,
      "policy": "soft_best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 9216,
      "policy": "bounded_best_of_poisson",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 9216,
      "policy": "first_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 372.7755102040816,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 9216,
      "policy": "uniform_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 1304.6122448979593,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 18432,
      "policy": "single",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 328.0816326530612,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 18432,
      "policy": "best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 18432,
      "policy": "soft_best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 18432,
      "policy": "bounded_best_of_poisson",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 18432,
      "policy": "first_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 372.7755102040816,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 18432,
      "policy": "uniform_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 1304.6122448979593,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 36864,
      "policy": "single",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 328.0816326530612,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 36864,
      "policy": "best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 36864,
      "policy": "soft_best_of_n",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 36864,
      "policy": "bounded_best_of_poisson",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 2540.530612244898,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 36864,
      "policy": "first_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 372.7755102040816,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    },
    {
      "public_filter": true,
      "cap_tokens": 36864,
      "policy": "uniform_public",
      "tasks": 49,
      "success": 0.9591836734693877,
      "paired_task_counts": {
        "candidate_better": 0,
        "candidate_worse": 0,
        "tied": 49
      },
      "mean_consumed_tokens": 1304.6122448979593,
      "cap_exhaustion_count": 0,
      "paired_vs_single_descriptive": {
        "effect": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "tasks": 49,
        "clusters": 49,
        "resamples": 10000,
        "seed": 20261006
      }
    }
  ],
  "posthoc_candidate_pool_diagnostic": {
    "schema_version": 1,
    "status": "posthoc diagnostic after frozen policy analysis; not prespecified primary endpoint",
    "tasks": 120,
    "candidates": 480,
    "candidate_passes": 437,
    "pool_counts": {
      "all_pass": 108,
      "all_fail": 10,
      "mixed": 2
    },
    "oracle_tasks_with_at_least_one_pass": 110,
    "unfiltered_first_candidate_tasks_passed": 109,
    "maximum_oracle_gain_over_first_candidate": 0.008333333333333304
  },
  "artifact_sha256": {
    "analysis": "4fcd3b98255aa15d7a6f58a303a159fb143f9a411c4bc4426c4d424bc67df3cb",
    "frozen_decisions": "da9a0cc203fbc44e7511a384cf2257f11becc18c06ab72d8735e44bfdf4d4d25",
    "grading_coverage": "a1633eba0b239bd81c4b984a56de3267f60d0f330c0e325d957895fec9e330c2",
    "case_manifest": "d5b3420c07d57099c630bdd457e55c614ee13ad03d97eeec02f4bb388ab98c9b",
    "independent_same_owner_metadata_audit": "3516212a250e466ae0dd5819a7a44f5d4d545559c1a79fe2fb81cabc7c24a445"
  },
  "interpretation": [
    "three seeds reuse each fixed four-candidate pool; not independent task samples",
    "task-cluster bootstrap intervals are conditional descriptive and multiple comparisons unadjusted",
    "two mixed-outcome candidate pools; oracle headroom only 1/120 versus first candidate",
    "same-owner separate-agent recomputation is not external review",
    "known public 2021 tasks cannot establish fresh-data generalization",
    "no data, model, policy or decision retuning after frozen selection"
  ],
  "source_access": "private or gated; aggregate metadata does not imply a public runnable source release"
}