{
  "cohort_id": "2cc80710990cbbfe8971b2b3d131095d5854074750f841cdd24aa642151e56b5",
  "category": "coding",
  "bank_version": "tested-best-bank-v2-reviewed",
  "bank_hash": "fce29b4d66cfddeab39632d451935271a37df595374b40eacd300201e3726291",
  "protocol_hash": "90b114dd8a70ecbf3aecc35f58cea1ba85c786554aee9835a38a1c518293da05",
  "grader_version": "deterministic-v2",
  "grader_hash": "62980813a06357fd89c914858db26855ffe2bd65a7e18f326d827fb6610e7e23",
  "run_start_utc": "2026-10-05T05:41:29.235497+00:00",
  "run_end_utc": "2026-10-05T05:54:25.801161+00:00",
  "run_start_vancouver": "2026-10-04T22:41:29.235497-07:00",
  "run_end_vancouver": "2026-10-04T22:54:25.801161-07:00",
  "cohort_coding_attempts": 108,
  "unique_tasks": 12,
  "repetitions_per_task_and_model": 3,
  "rows": [
    {
      "model_id": "gpt-6.1-sol",
      "route": "OpenAI direct API, Standard global",
      "settings": {
        "effort": "medium",
        "max_output_tokens": 8192,
        "surface": "OpenAI direct API, Standard global"
      },
      "attempts": 36,
      "passed": 36,
      "failed": 0,
      "pass_rate": 1.0,
      "historical_input_usd_per_million": 2.0,
      "historical_output_usd_per_million": 10.0,
      "input_tokens": 14502,
      "billable_output_tokens": 4208,
      "visible_output_tokens": 3975,
      "recorded_reasoning_tokens": 233,
      "estimated_total_microusd": 71084,
      "estimated_failed_microusd": 0,
      "estimated_passed_microusd": 71084,
      "estimated_total_usd": 0.071084,
      "estimated_failed_usd": 0.0,
      "failed_share_of_estimated_cost": 0.0,
      "mean_estimated_cost_per_attempt_usd": 0.0019745555555555554,
      "estimated_cost_per_strict_pass_usd": 0.0019745555555555554,
      "mean_estimated_cost_of_passing_attempts_usd": 0.0019745555555555554,
      "uplift_vs_mean_all_attempt_cost": 0.0,
      "failed_cost_loading_vs_passing_only_mean": 0.0,
      "failure_breakdown": {},
      "billed_total_usd": null,
      "cost_rankable": false,
      "task_results": [
        {
          "task_id": "coding-reconcile",
          "task_title": "Reconcile invoices and payments",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-dedup",
          "task_title": "Resolve duplicate event deliveries",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-intervals",
          "task_title": "Merge maintenance windows",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-redact",
          "task_title": "Redact nested customer records",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-rolling",
          "task_title": "Calculate seven-day activity totals",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-allocate",
          "task_title": "Allocate cents without rounding loss",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-csv",
          "task_title": "Parse and aggregate CSV purchases",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-dependencies",
          "task_title": "Order deployment dependencies",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-fifo",
          "task_title": "Value remaining FIFO inventory",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-percentile",
          "task_title": "Calculate service latency percentiles",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-slug",
          "task_title": "Normalize Unicode article slugs",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-rate",
          "task_title": "Apply a per-user rolling rate limit",
          "passed": 3,
          "attempts": 3
        }
      ]
    },
    {
      "model_id": "claude-sonnet-5-5",
      "route": "Claude direct API, Standard global",
      "settings": {
        "effort": "medium",
        "max_output_tokens": 8192,
        "surface": "Claude direct API, Standard global"
      },
      "attempts": 36,
      "passed": 31,
      "failed": 5,
      "pass_rate": 0.8611111111111112,
      "historical_input_usd_per_million": 2.0,
      "historical_output_usd_per_million": 10.0,
      "input_tokens": 22302,
      "billable_output_tokens": 7500,
      "visible_output_tokens": 7500,
      "recorded_reasoning_tokens": 0,
      "estimated_total_microusd": 119604,
      "estimated_failed_microusd": 29620,
      "estimated_passed_microusd": 89984,
      "estimated_total_usd": 0.119604,
      "estimated_failed_usd": 0.02962,
      "failed_share_of_estimated_cost": 0.24765058024815223,
      "mean_estimated_cost_per_attempt_usd": 0.0033223333333333334,
      "estimated_cost_per_strict_pass_usd": 0.003858193548387097,
      "mean_estimated_cost_of_passing_attempts_usd": 0.002902709677419355,
      "uplift_vs_mean_all_attempt_cost": 0.16129032258064524,
      "failed_cost_loading_vs_passing_only_mean": 0.32916963015647227,
      "failure_breakdown": {
        "execution_rejected": 5
      },
      "billed_total_usd": null,
      "cost_rankable": false,
      "task_results": [
        {
          "task_id": "coding-reconcile",
          "task_title": "Reconcile invoices and payments",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-dedup",
          "task_title": "Resolve duplicate event deliveries",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-intervals",
          "task_title": "Merge maintenance windows",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-redact",
          "task_title": "Redact nested customer records",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-rolling",
          "task_title": "Calculate seven-day activity totals",
          "passed": 0,
          "attempts": 3
        },
        {
          "task_id": "coding-allocate",
          "task_title": "Allocate cents without rounding loss",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-csv",
          "task_title": "Parse and aggregate CSV purchases",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-dependencies",
          "task_title": "Order deployment dependencies",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-fifo",
          "task_title": "Value remaining FIFO inventory",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-percentile",
          "task_title": "Calculate service latency percentiles",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-slug",
          "task_title": "Normalize Unicode article slugs",
          "passed": 2,
          "attempts": 3
        },
        {
          "task_id": "coding-rate",
          "task_title": "Apply a per-user rolling rate limit",
          "passed": 2,
          "attempts": 3
        }
      ]
    },
    {
      "model_id": "gemini-3.8-flash",
      "route": "Gemini Developer API, paid Standard",
      "settings": {
        "effort": "medium",
        "max_output_tokens": 8192,
        "surface": "Gemini Developer API, paid Standard"
      },
      "attempts": 36,
      "passed": 35,
      "failed": 1,
      "pass_rate": 0.9722222222222222,
      "historical_input_usd_per_million": 0.75,
      "historical_output_usd_per_million": 3.75,
      "input_tokens": 14466,
      "billable_output_tokens": 47932,
      "visible_output_tokens": 5178,
      "recorded_reasoning_tokens": 42754,
      "estimated_total_microusd": 190604,
      "estimated_failed_microusd": 3715,
      "estimated_passed_microusd": 186889,
      "estimated_total_usd": 0.190604,
      "estimated_failed_usd": 0.003715,
      "failed_share_of_estimated_cost": 0.019490671759249543,
      "mean_estimated_cost_per_attempt_usd": 0.005294555555555556,
      "estimated_cost_per_strict_pass_usd": 0.005445828571428571,
      "mean_estimated_cost_of_passing_attempts_usd": 0.005339685714285714,
      "uplift_vs_mean_all_attempt_cost": 0.02857142857142847,
      "failed_cost_loading_vs_passing_only_mean": 0.019878109466046692,
      "failure_breakdown": {
        "execution_rejected": 1
      },
      "billed_total_usd": null,
      "cost_rankable": false,
      "task_results": [
        {
          "task_id": "coding-reconcile",
          "task_title": "Reconcile invoices and payments",
          "passed": 2,
          "attempts": 3
        },
        {
          "task_id": "coding-dedup",
          "task_title": "Resolve duplicate event deliveries",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-intervals",
          "task_title": "Merge maintenance windows",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-redact",
          "task_title": "Redact nested customer records",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-rolling",
          "task_title": "Calculate seven-day activity totals",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-allocate",
          "task_title": "Allocate cents without rounding loss",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-csv",
          "task_title": "Parse and aggregate CSV purchases",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-dependencies",
          "task_title": "Order deployment dependencies",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-fifo",
          "task_title": "Value remaining FIFO inventory",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-percentile",
          "task_title": "Calculate service latency percentiles",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-slug",
          "task_title": "Normalize Unicode article slugs",
          "passed": 3,
          "attempts": 3
        },
        {
          "task_id": "coding-rate",
          "task_title": "Apply a per-user rolling rate limit",
          "passed": 3,
          "attempts": 3
        }
      ]
    }
  ],
  "rounding": "Ceiling to one microdollar independently for each attempt, then sum.",
  "evidence_boundary": "Recomputed from retained attempt export and usage fields; original full raw provider payloads are not present in this local working copy. Retained hashes identify those payloads; this audit does not reverify their bytes or replay execution grades.",
  "retry_boundary": "Three independent repetitions per task. One provider call per trial, no automatic paid retries. No repair strategy was tested.",
  "excluded_costs": [
    "settled invoice reconciliation",
    "human review and repair",
    "hardware",
    "taxes",
    "tool fees"
  ],
  "new_provider_calls": 0,
  "submitted_code_executed": false
}
