{
  "models": {
    "alibaba/qwen4-max": {
      "runs": 8,
      "complete_passes": 7,
      "cost_usd": "0.2850909",
      "median_seconds": 56.49439862500003,
      "median_turns": 6.0,
      "hidden_passed": 45,
      "hidden_total": 46,
      "tool_calls": 83,
      "visible_test_calls": 8
    },
    "anthropic/claude-opus-5.5": {
      "runs": 8,
      "complete_passes": 8,
      "cost_usd": "1.032804",
      "median_seconds": 40.15320118749514,
      "median_turns": 4.0,
      "hidden_passed": 46,
      "hidden_total": 46,
      "tool_calls": 86,
      "visible_test_calls": 8
    }
  },
  "runs": [
    {
      "task": "reconciliation",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 32.87431229099457,
      "usage_cost_usd": "0.086844",
      "turns": 4,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_duplicate (test_visible.Visible.test_duplicate) ... ok\ntest_refund (test_visible.Visible.test_refund) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_duplicate\", \"test_visible.Visible.test_refund\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_duplicate",
              "test_visible.Visible.test_refund"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_amounts (test_hidden.Hidden.test_bad_amounts) ... ok\ntest_conflict (test_hidden.Hidden.test_conflict) ... ok\ntest_currencies_and_negative (test_hidden.Hidden.test_currencies_and_negative) ... ok\ntest_header (test_hidden.Hidden.test_header) ... ok\ntest_invalid_currency (test_hidden.Hidden.test_invalid_currency) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_bad_amounts\", \"test_hidden.Hidden.test_conflict\", \"test_hidden.Hidden.test_currencies_and_negative\", \"test_hidden.Hidden.test_header\", \"test_hidden.Hidden.test_invalid_currency\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_bad_amounts",
              "test_hidden.Hidden.test_conflict",
              "test_hidden.Hidden.test_currencies_and_negative",
              "test_hidden.Hidden.test_header",
              "test_hidden.Hidden.test_invalid_currency"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "parse.py",
        "report.py"
      ],
      "run": "reconciliation-opus-r1",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 10,
      "run_tests_calls": 1,
      "api_input_tokens": 8171,
      "api_output_tokens": 2708,
      "model_response_seconds": 32.498389998989296
    },
    {
      "task": "reconciliation",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 38.0304941249924,
      "usage_cost_usd": "0.090020",
      "turns": 4,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_duplicate (test_visible.Visible.test_duplicate) ... ok\ntest_refund (test_visible.Visible.test_refund) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_duplicate\", \"test_visible.Visible.test_refund\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_duplicate",
              "test_visible.Visible.test_refund"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_amounts (test_hidden.Hidden.test_bad_amounts) ... ok\ntest_conflict (test_hidden.Hidden.test_conflict) ... ok\ntest_currencies_and_negative (test_hidden.Hidden.test_currencies_and_negative) ... ok\ntest_header (test_hidden.Hidden.test_header) ... ok\ntest_invalid_currency (test_hidden.Hidden.test_invalid_currency) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_bad_amounts\", \"test_hidden.Hidden.test_conflict\", \"test_hidden.Hidden.test_currencies_and_negative\", \"test_hidden.Hidden.test_header\", \"test_hidden.Hidden.test_invalid_currency\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_bad_amounts",
              "test_hidden.Hidden.test_conflict",
              "test_hidden.Hidden.test_currencies_and_negative",
              "test_hidden.Hidden.test_header",
              "test_hidden.Hidden.test_invalid_currency"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "parse.py",
        "report.py"
      ],
      "run": "reconciliation-opus-r2",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 10,
      "run_tests_calls": 1,
      "api_input_tokens": 8265,
      "api_output_tokens": 2848,
      "model_response_seconds": 37.77235387498513
    },
    {
      "task": "reconciliation",
      "model": "alibaba/qwen4-max",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 42.59341512499668,
      "usage_cost_usd": "0.0202389",
      "turns": 5,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_duplicate (test_visible.Visible.test_duplicate) ... ok\ntest_refund (test_visible.Visible.test_refund) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_duplicate\", \"test_visible.Visible.test_refund\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_duplicate",
              "test_visible.Visible.test_refund"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_amounts (test_hidden.Hidden.test_bad_amounts) ... ok\ntest_conflict (test_hidden.Hidden.test_conflict) ... ok\ntest_currencies_and_negative (test_hidden.Hidden.test_currencies_and_negative) ... ok\ntest_header (test_hidden.Hidden.test_header) ... ok\ntest_invalid_currency (test_hidden.Hidden.test_invalid_currency) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_bad_amounts\", \"test_hidden.Hidden.test_conflict\", \"test_hidden.Hidden.test_currencies_and_negative\", \"test_hidden.Hidden.test_header\", \"test_hidden.Hidden.test_invalid_currency\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_bad_amounts",
              "test_hidden.Hidden.test_conflict",
              "test_hidden.Hidden.test_currencies_and_negative",
              "test_hidden.Hidden.test_header",
              "test_hidden.Hidden.test_invalid_currency"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "parse.py",
        "report.py"
      ],
      "run": "reconciliation-qwen-r1",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 10,
      "run_tests_calls": 1,
      "api_input_tokens": 8481,
      "api_output_tokens": 2238,
      "model_response_seconds": 42.354682086006505
    },
    {
      "task": "reconciliation",
      "model": "alibaba/qwen4-max",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 38.13853412499884,
      "usage_cost_usd": "0.0200313",
      "turns": 5,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_duplicate (test_visible.Visible.test_duplicate) ... ok\ntest_refund (test_visible.Visible.test_refund) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_duplicate\", \"test_visible.Visible.test_refund\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_duplicate",
              "test_visible.Visible.test_refund"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_amounts (test_hidden.Hidden.test_bad_amounts) ... ok\ntest_conflict (test_hidden.Hidden.test_conflict) ... ok\ntest_currencies_and_negative (test_hidden.Hidden.test_currencies_and_negative) ... ok\ntest_header (test_hidden.Hidden.test_header) ... ok\ntest_invalid_currency (test_hidden.Hidden.test_invalid_currency) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_bad_amounts\", \"test_hidden.Hidden.test_conflict\", \"test_hidden.Hidden.test_currencies_and_negative\", \"test_hidden.Hidden.test_header\", \"test_hidden.Hidden.test_invalid_currency\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_bad_amounts",
              "test_hidden.Hidden.test_conflict",
              "test_hidden.Hidden.test_currencies_and_negative",
              "test_hidden.Hidden.test_header",
              "test_hidden.Hidden.test_invalid_currency"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "parse.py",
        "report.py"
      ],
      "run": "reconciliation-qwen-r2",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 10,
      "run_tests_calls": 1,
      "api_input_tokens": 8123,
      "api_output_tokens": 2068,
      "model_response_seconds": 37.88014679100888
    },
    {
      "task": "replay",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 29.218309957999736,
      "usage_cost_usd": "0.094932",
      "turns": 4,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_atomic (test_visible.Visible.test_atomic) ... ok\ntest_retry (test_visible.Visible.test_retry) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.002s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_atomic\", \"test_visible.Visible.test_retry\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_atomic",
              "test_visible.Visible.test_retry"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_and_rollback (test_hidden.Hidden.test_conflict_and_rollback) ... ok\ntest_invalid_bool (test_hidden.Hidden.test_invalid_bool) ... ok\ntest_persistence (test_hidden.Hidden.test_persistence) ... ok\ntest_within_batch_and_negative (test_hidden.Hidden.test_within_batch_and_negative) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.003s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_hidden.Hidden.test_conflict_and_rollback\", \"test_hidden.Hidden.test_invalid_bool\", \"test_hidden.Hidden.test_persistence\", \"test_hidden.Hidden.test_within_batch_and_negative\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_and_rollback",
              "test_hidden.Hidden.test_invalid_bool",
              "test_hidden.Hidden.test_persistence",
              "test_hidden.Hidden.test_within_batch_and_negative"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "events.py",
        "processor.py"
      ],
      "run": "replay-opus-r1",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 4,
      "hidden_failed": 0,
      "hidden_passed": 4,
      "tool_calls": 10,
      "run_tests_calls": 1,
      "api_input_tokens": 9183,
      "api_output_tokens": 2910,
      "model_response_seconds": 28.978939083986916
    },
    {
      "task": "replay",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 33.026289542001905,
      "usage_cost_usd": "0.103324",
      "turns": 4,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_atomic (test_visible.Visible.test_atomic) ... ok\ntest_retry (test_visible.Visible.test_retry) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.001s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_atomic\", \"test_visible.Visible.test_retry\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_atomic",
              "test_visible.Visible.test_retry"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_and_rollback (test_hidden.Hidden.test_conflict_and_rollback) ... ok\ntest_invalid_bool (test_hidden.Hidden.test_invalid_bool) ... ok\ntest_persistence (test_hidden.Hidden.test_persistence) ... ok\ntest_within_batch_and_negative (test_hidden.Hidden.test_within_batch_and_negative) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.003s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_hidden.Hidden.test_conflict_and_rollback\", \"test_hidden.Hidden.test_invalid_bool\", \"test_hidden.Hidden.test_persistence\", \"test_hidden.Hidden.test_within_batch_and_negative\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_and_rollback",
              "test_hidden.Hidden.test_invalid_bool",
              "test_hidden.Hidden.test_persistence",
              "test_hidden.Hidden.test_within_batch_and_negative"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "events.py",
        "processor.py"
      ],
      "run": "replay-opus-r2",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 4,
      "hidden_failed": 0,
      "hidden_passed": 4,
      "tool_calls": 10,
      "run_tests_calls": 1,
      "api_input_tokens": 9601,
      "api_output_tokens": 3246,
      "model_response_seconds": 32.72369116700429
    },
    {
      "task": "replay",
      "model": "alibaba/qwen4-max",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 31.290693625007407,
      "usage_cost_usd": "0.0161388",
      "turns": 5,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_atomic (test_visible.Visible.test_atomic) ... ok\ntest_retry (test_visible.Visible.test_retry) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.001s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_atomic\", \"test_visible.Visible.test_retry\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_atomic",
              "test_visible.Visible.test_retry"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_and_rollback (test_hidden.Hidden.test_conflict_and_rollback) ... ok\ntest_invalid_bool (test_hidden.Hidden.test_invalid_bool) ... ok\ntest_persistence (test_hidden.Hidden.test_persistence) ... ok\ntest_within_batch_and_negative (test_hidden.Hidden.test_within_batch_and_negative) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.004s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_hidden.Hidden.test_conflict_and_rollback\", \"test_hidden.Hidden.test_invalid_bool\", \"test_hidden.Hidden.test_persistence\", \"test_hidden.Hidden.test_within_batch_and_negative\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_and_rollback",
              "test_hidden.Hidden.test_invalid_bool",
              "test_hidden.Hidden.test_persistence",
              "test_hidden.Hidden.test_within_batch_and_negative"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "processor.py"
      ],
      "run": "replay-qwen-r1",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 4,
      "hidden_failed": 0,
      "hidden_passed": 4,
      "tool_calls": 8,
      "run_tests_calls": 1,
      "api_input_tokens": 7472,
      "api_output_tokens": 1420,
      "model_response_seconds": 31.04537358401285
    },
    {
      "task": "replay",
      "model": "alibaba/qwen4-max",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 38.73963579100382,
      "usage_cost_usd": "0.0192861",
      "turns": 6,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_atomic (test_visible.Visible.test_atomic) ... ok\ntest_retry (test_visible.Visible.test_retry) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.001s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_atomic\", \"test_visible.Visible.test_retry\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_atomic",
              "test_visible.Visible.test_retry"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_and_rollback (test_hidden.Hidden.test_conflict_and_rollback) ... ok\ntest_invalid_bool (test_hidden.Hidden.test_invalid_bool) ... ok\ntest_persistence (test_hidden.Hidden.test_persistence) ... ok\ntest_within_batch_and_negative (test_hidden.Hidden.test_within_batch_and_negative) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.003s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_hidden.Hidden.test_conflict_and_rollback\", \"test_hidden.Hidden.test_invalid_bool\", \"test_hidden.Hidden.test_persistence\", \"test_hidden.Hidden.test_within_batch_and_negative\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_and_rollback",
              "test_hidden.Hidden.test_invalid_bool",
              "test_hidden.Hidden.test_persistence",
              "test_hidden.Hidden.test_within_batch_and_negative"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "processor.py"
      ],
      "run": "replay-qwen-r2",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 4,
      "hidden_failed": 0,
      "hidden_passed": 4,
      "tool_calls": 9,
      "run_tests_calls": 1,
      "api_input_tokens": 9748,
      "api_output_tokens": 1685,
      "model_response_seconds": 38.482488626017584
    },
    {
      "task": "scheduler",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 54.050469958005124,
      "usage_cost_usd": "0.181756",
      "turns": 5,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_dependency (test_visible.Visible.test_dependency) ... ok\ntest_limit (test_visible.Visible.test_limit) ... ok\ntest_parallel_dependencies (test_visible.VisibleOptimal.test_parallel_dependencies) ... ok\ntest_waves (test_visible.VisiblePlan.test_waves) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_visible.Visible.test_dependency\", \"test_visible.Visible.test_limit\", \"test_visible.VisibleOptimal.test_parallel_dependencies\", \"test_visible.VisiblePlan.test_waves\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_visible.Visible.test_dependency",
              "test_visible.Visible.test_limit",
              "test_visible.VisibleOptimal.test_parallel_dependencies",
              "test_visible.VisiblePlan.test_waves"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_deps_and_missing (test_hidden.Hidden.test_deps_and_missing) ... ok\ntest_order_and_terminal (test_hidden.Hidden.test_order_and_terminal) ... ok\ntest_retry_progression (test_hidden.Hidden.test_retry_progression) ... ok\ntest_time_boundary (test_hidden.Hidden.test_time_boundary) ... ok\ntest_invalid_and_empty (test_hidden.HiddenOptimal.test_invalid_and_empty) ... ok\ntest_oracle_cases (test_hidden.HiddenOptimal.test_oracle_cases) ... ok\ntest_bad_graphs (test_hidden.HiddenPlan.test_bad_graphs) ... ok\ntest_diamond_and_no_mutation (test_hidden.HiddenPlan.test_diamond_and_no_mutation) ... ok\ntest_done_and_empty (test_hidden.HiddenPlan.test_done_and_empty) ... ok\n\n----------------------------------------------------------------------\nRan 9 tests in 0.068s\n\nOK\nKINGY_RESULT:{\"testsRun\": 9, \"test_ids\": [\"test_hidden.Hidden.test_deps_and_missing\", \"test_hidden.Hidden.test_order_and_terminal\", \"test_hidden.Hidden.test_retry_progression\", \"test_hidden.Hidden.test_time_boundary\", \"test_hidden.HiddenOptimal.test_invalid_and_empty\", \"test_hidden.HiddenOptimal.test_oracle_cases\", \"test_hidden.HiddenPlan.test_bad_graphs\", \"test_hidden.HiddenPlan.test_diamond_and_no_mutation\", \"test_hidden.HiddenPlan.test_done_and_empty\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 9,
            "test_ids": [
              "test_hidden.Hidden.test_deps_and_missing",
              "test_hidden.Hidden.test_order_and_terminal",
              "test_hidden.Hidden.test_retry_progression",
              "test_hidden.Hidden.test_time_boundary",
              "test_hidden.HiddenOptimal.test_invalid_and_empty",
              "test_hidden.HiddenOptimal.test_oracle_cases",
              "test_hidden.HiddenPlan.test_bad_graphs",
              "test_hidden.HiddenPlan.test_diamond_and_no_mutation",
              "test_hidden.HiddenPlan.test_done_and_empty"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "planner.py",
        "policy.py",
        "scheduler.py"
      ],
      "run": "scheduler-opus-r1",
      "visible_total": 4,
      "visible_failed": 0,
      "visible_passed": 4,
      "hidden_total": 9,
      "hidden_failed": 0,
      "hidden_passed": 9,
      "tool_calls": 12,
      "run_tests_calls": 1,
      "api_input_tokens": 17889,
      "api_output_tokens": 5510,
      "model_response_seconds": 53.745586709017516
    },
    {
      "task": "scheduler",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 71.89249979199667,
      "usage_cost_usd": "0.207900",
      "turns": 4,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_dependency (test_visible.Visible.test_dependency) ... ok\ntest_limit (test_visible.Visible.test_limit) ... ok\ntest_parallel_dependencies (test_visible.VisibleOptimal.test_parallel_dependencies) ... ok\ntest_waves (test_visible.VisiblePlan.test_waves) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_visible.Visible.test_dependency\", \"test_visible.Visible.test_limit\", \"test_visible.VisibleOptimal.test_parallel_dependencies\", \"test_visible.VisiblePlan.test_waves\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_visible.Visible.test_dependency",
              "test_visible.Visible.test_limit",
              "test_visible.VisibleOptimal.test_parallel_dependencies",
              "test_visible.VisiblePlan.test_waves"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_deps_and_missing (test_hidden.Hidden.test_deps_and_missing) ... ok\ntest_order_and_terminal (test_hidden.Hidden.test_order_and_terminal) ... ok\ntest_retry_progression (test_hidden.Hidden.test_retry_progression) ... ok\ntest_time_boundary (test_hidden.Hidden.test_time_boundary) ... ok\ntest_invalid_and_empty (test_hidden.HiddenOptimal.test_invalid_and_empty) ... ok\ntest_oracle_cases (test_hidden.HiddenOptimal.test_oracle_cases) ... ok\ntest_bad_graphs (test_hidden.HiddenPlan.test_bad_graphs) ... ok\ntest_diamond_and_no_mutation (test_hidden.HiddenPlan.test_diamond_and_no_mutation) ... ok\ntest_done_and_empty (test_hidden.HiddenPlan.test_done_and_empty) ... ok\n\n----------------------------------------------------------------------\nRan 9 tests in 0.069s\n\nOK\nKINGY_RESULT:{\"testsRun\": 9, \"test_ids\": [\"test_hidden.Hidden.test_deps_and_missing\", \"test_hidden.Hidden.test_order_and_terminal\", \"test_hidden.Hidden.test_retry_progression\", \"test_hidden.Hidden.test_time_boundary\", \"test_hidden.HiddenOptimal.test_invalid_and_empty\", \"test_hidden.HiddenOptimal.test_oracle_cases\", \"test_hidden.HiddenPlan.test_bad_graphs\", \"test_hidden.HiddenPlan.test_diamond_and_no_mutation\", \"test_hidden.HiddenPlan.test_done_and_empty\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 9,
            "test_ids": [
              "test_hidden.Hidden.test_deps_and_missing",
              "test_hidden.Hidden.test_order_and_terminal",
              "test_hidden.Hidden.test_retry_progression",
              "test_hidden.Hidden.test_time_boundary",
              "test_hidden.HiddenOptimal.test_invalid_and_empty",
              "test_hidden.HiddenOptimal.test_oracle_cases",
              "test_hidden.HiddenPlan.test_bad_graphs",
              "test_hidden.HiddenPlan.test_diamond_and_no_mutation",
              "test_hidden.HiddenPlan.test_done_and_empty"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "planner.py",
        "policy.py",
        "scheduler.py"
      ],
      "run": "scheduler-opus-r2",
      "visible_total": 4,
      "visible_failed": 0,
      "visible_passed": 4,
      "hidden_total": 9,
      "hidden_failed": 0,
      "hidden_passed": 9,
      "tool_calls": 12,
      "run_tests_calls": 1,
      "api_input_tokens": 15025,
      "api_output_tokens": 7390,
      "model_response_seconds": 71.58913766700425
    },
    {
      "task": "scheduler",
      "model": "alibaba/qwen4-max",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 117.1311302910035,
      "usage_cost_usd": "0.0461763",
      "turns": 7,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_dependency (test_visible.Visible.test_dependency) ... ok\ntest_limit (test_visible.Visible.test_limit) ... ok\ntest_parallel_dependencies (test_visible.VisibleOptimal.test_parallel_dependencies) ... ok\ntest_waves (test_visible.VisiblePlan.test_waves) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_visible.Visible.test_dependency\", \"test_visible.Visible.test_limit\", \"test_visible.VisibleOptimal.test_parallel_dependencies\", \"test_visible.VisiblePlan.test_waves\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_visible.Visible.test_dependency",
              "test_visible.Visible.test_limit",
              "test_visible.VisibleOptimal.test_parallel_dependencies",
              "test_visible.VisiblePlan.test_waves"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 1,
          "timeout": false,
          "output": "test_deps_and_missing (test_hidden.Hidden.test_deps_and_missing) ... ok\ntest_order_and_terminal (test_hidden.Hidden.test_order_and_terminal) ... ok\ntest_retry_progression (test_hidden.Hidden.test_retry_progression) ... ok\ntest_time_boundary (test_hidden.Hidden.test_time_boundary) ... ok\ntest_invalid_and_empty (test_hidden.HiddenOptimal.test_invalid_and_empty) ... ERROR\ntest_oracle_cases (test_hidden.HiddenOptimal.test_oracle_cases) ... ok\ntest_bad_graphs (test_hidden.HiddenPlan.test_bad_graphs) ... ok\ntest_diamond_and_no_mutation (test_hidden.HiddenPlan.test_diamond_and_no_mutation) ... ok\ntest_done_and_empty (test_hidden.HiddenPlan.test_done_and_empty) ... ok\n\n======================================================================\nERROR: test_invalid_and_empty (test_hidden.HiddenOptimal.test_invalid_and_empty)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n  File \"/private/var/folders/21/9q8f1tp90cb73x3p8gxw_pl40000gn/T/kingy-validate-kgh2lm_p/repo/_hidden_tests/test_hidden.py\", line 47, in test_invalid_and_empty\n    with self.assertRaises(ValueError):schedule(jobs,w)\n                                       ~~~~~~~~^^^^^^^^\n  File \"/private/var/folders/21/9q8f1tp90cb73x3p8gxw_pl40000gn/T/kingy-validate-kgh2lm_p/repo/planner.py\", line 64, in schedule\n    jid = j['id']\n          ~^^^^^^\nKeyError: 'id'\n\n----------------------------------------------------------------------\nRan 9 tests in 0.071s\n\nFAILED (errors=1)\nKINGY_RESULT:{\"testsRun\": 9, \"test_ids\": [\"test_hidden.Hidden.test_deps_and_missing\", \"test_hidden.Hidden.test_order_and_terminal\", \"test_hidden.Hidden.test_retry_progression\", \"test_hidden.Hidden.test_time_boundary\", \"test_hidden.HiddenOptimal.test_invalid_and_empty\", \"test_hidden.HiddenOptimal.test_oracle_cases\", \"test_hidden.HiddenPlan.test_bad_graphs\", \"test_hidden.HiddenPlan.test_diamond_and_no_mutation\", \"test_hidden.HiddenPlan.test_done_and_empty\"], \"failures\": [], \"errors\": [\"test_hidden.HiddenOptimal.test_invalid_and_empty\"], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 9,
            "test_ids": [
              "test_hidden.Hidden.test_deps_and_missing",
              "test_hidden.Hidden.test_order_and_terminal",
              "test_hidden.Hidden.test_retry_progression",
              "test_hidden.Hidden.test_time_boundary",
              "test_hidden.HiddenOptimal.test_invalid_and_empty",
              "test_hidden.HiddenOptimal.test_oracle_cases",
              "test_hidden.HiddenPlan.test_bad_graphs",
              "test_hidden.HiddenPlan.test_diamond_and_no_mutation",
              "test_hidden.HiddenPlan.test_done_and_empty"
            ],
            "failures": [],
            "errors": [
              "test_hidden.HiddenOptimal.test_invalid_and_empty"
            ],
            "skipped": []
          }
        }
      },
      "passed": false,
      "changed_files": [
        "planner.py",
        "policy.py",
        "scheduler.py"
      ],
      "run": "scheduler-qwen-r1",
      "visible_total": 4,
      "visible_failed": 0,
      "visible_passed": 4,
      "hidden_total": 9,
      "hidden_failed": 1,
      "hidden_passed": 8,
      "tool_calls": 12,
      "run_tests_calls": 1,
      "api_input_tokens": 24426,
      "api_output_tokens": 6497,
      "model_response_seconds": 116.81414399901405
    },
    {
      "task": "scheduler",
      "model": "alibaba/qwen4-max",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 272.9394354160031,
      "usage_cost_usd": "0.0955659",
      "turns": 7,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_dependency (test_visible.Visible.test_dependency) ... ok\ntest_limit (test_visible.Visible.test_limit) ... ok\ntest_parallel_dependencies (test_visible.VisibleOptimal.test_parallel_dependencies) ... ok\ntest_waves (test_visible.VisiblePlan.test_waves) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 0.000s\n\nOK\nKINGY_RESULT:{\"testsRun\": 4, \"test_ids\": [\"test_visible.Visible.test_dependency\", \"test_visible.Visible.test_limit\", \"test_visible.VisibleOptimal.test_parallel_dependencies\", \"test_visible.VisiblePlan.test_waves\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 4,
            "test_ids": [
              "test_visible.Visible.test_dependency",
              "test_visible.Visible.test_limit",
              "test_visible.VisibleOptimal.test_parallel_dependencies",
              "test_visible.VisiblePlan.test_waves"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_deps_and_missing (test_hidden.Hidden.test_deps_and_missing) ... ok\ntest_order_and_terminal (test_hidden.Hidden.test_order_and_terminal) ... ok\ntest_retry_progression (test_hidden.Hidden.test_retry_progression) ... ok\ntest_time_boundary (test_hidden.Hidden.test_time_boundary) ... ok\ntest_invalid_and_empty (test_hidden.HiddenOptimal.test_invalid_and_empty) ... ok\ntest_oracle_cases (test_hidden.HiddenOptimal.test_oracle_cases) ... ok\ntest_bad_graphs (test_hidden.HiddenPlan.test_bad_graphs) ... ok\ntest_diamond_and_no_mutation (test_hidden.HiddenPlan.test_diamond_and_no_mutation) ... ok\ntest_done_and_empty (test_hidden.HiddenPlan.test_done_and_empty) ... ok\n\n----------------------------------------------------------------------\nRan 9 tests in 0.070s\n\nOK\nKINGY_RESULT:{\"testsRun\": 9, \"test_ids\": [\"test_hidden.Hidden.test_deps_and_missing\", \"test_hidden.Hidden.test_order_and_terminal\", \"test_hidden.Hidden.test_retry_progression\", \"test_hidden.Hidden.test_time_boundary\", \"test_hidden.HiddenOptimal.test_invalid_and_empty\", \"test_hidden.HiddenOptimal.test_oracle_cases\", \"test_hidden.HiddenPlan.test_bad_graphs\", \"test_hidden.HiddenPlan.test_diamond_and_no_mutation\", \"test_hidden.HiddenPlan.test_done_and_empty\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 9,
            "test_ids": [
              "test_hidden.Hidden.test_deps_and_missing",
              "test_hidden.Hidden.test_order_and_terminal",
              "test_hidden.Hidden.test_retry_progression",
              "test_hidden.Hidden.test_time_boundary",
              "test_hidden.HiddenOptimal.test_invalid_and_empty",
              "test_hidden.HiddenOptimal.test_oracle_cases",
              "test_hidden.HiddenPlan.test_bad_graphs",
              "test_hidden.HiddenPlan.test_diamond_and_no_mutation",
              "test_hidden.HiddenPlan.test_done_and_empty"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "planner.py",
        "policy.py",
        "scheduler.py"
      ],
      "run": "scheduler-qwen-r2",
      "visible_total": 4,
      "visible_failed": 0,
      "visible_passed": 4,
      "hidden_total": 9,
      "hidden_failed": 0,
      "hidden_passed": 9,
      "tool_calls": 12,
      "run_tests_calls": 1,
      "api_input_tokens": 44713,
      "api_output_tokens": 14196,
      "model_response_seconds": 272.54421987599926
    },
    {
      "task": "sync",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 42.27590824999788,
      "usage_cost_usd": "0.131756",
      "turns": 4,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_page_atomic (test_visible.Visible.test_bad_page_atomic) ... ok\ntest_out_of_order (test_visible.Visible.test_out_of_order) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.001s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_bad_page_atomic\", \"test_visible.Visible.test_out_of_order\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_bad_page_atomic",
              "test_visible.Visible.test_out_of_order"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_rollback (test_hidden.Hidden.test_conflict_rollback) ... ok\ntest_cursor_conflict (test_hidden.Hidden.test_cursor_conflict) ... ok\ntest_older_cursor_conflict (test_hidden.Hidden.test_older_cursor_conflict) ... ok\ntest_replay_newer_and_validation (test_hidden.Hidden.test_replay_newer_and_validation) ... ok\ntest_restart_tombstone (test_hidden.Hidden.test_restart_tombstone) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.003s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_conflict_rollback\", \"test_hidden.Hidden.test_cursor_conflict\", \"test_hidden.Hidden.test_older_cursor_conflict\", \"test_hidden.Hidden.test_replay_newer_and_validation\", \"test_hidden.Hidden.test_restart_tombstone\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_rollback",
              "test_hidden.Hidden.test_cursor_conflict",
              "test_hidden.Hidden.test_older_cursor_conflict",
              "test_hidden.Hidden.test_replay_newer_and_validation",
              "test_hidden.Hidden.test_restart_tombstone"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "store.py",
        "sync.py",
        "validate.py"
      ],
      "run": "sync-opus-r1",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 11,
      "run_tests_calls": 1,
      "api_input_tokens": 10734,
      "api_output_tokens": 4441,
      "model_response_seconds": 42.02650570801052
    },
    {
      "task": "sync",
      "model": "anthropic/claude-opus-5.5",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 42.926635666997754,
      "usage_cost_usd": "0.136272",
      "turns": 4,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_page_atomic (test_visible.Visible.test_bad_page_atomic) ... ok\ntest_out_of_order (test_visible.Visible.test_out_of_order) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.002s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_bad_page_atomic\", \"test_visible.Visible.test_out_of_order\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_bad_page_atomic",
              "test_visible.Visible.test_out_of_order"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_rollback (test_hidden.Hidden.test_conflict_rollback) ... ok\ntest_cursor_conflict (test_hidden.Hidden.test_cursor_conflict) ... ok\ntest_older_cursor_conflict (test_hidden.Hidden.test_older_cursor_conflict) ... ok\ntest_replay_newer_and_validation (test_hidden.Hidden.test_replay_newer_and_validation) ... ok\ntest_restart_tombstone (test_hidden.Hidden.test_restart_tombstone) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.019s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_conflict_rollback\", \"test_hidden.Hidden.test_cursor_conflict\", \"test_hidden.Hidden.test_older_cursor_conflict\", \"test_hidden.Hidden.test_replay_newer_and_validation\", \"test_hidden.Hidden.test_restart_tombstone\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_rollback",
              "test_hidden.Hidden.test_cursor_conflict",
              "test_hidden.Hidden.test_older_cursor_conflict",
              "test_hidden.Hidden.test_replay_newer_and_validation",
              "test_hidden.Hidden.test_restart_tombstone"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "store.py",
        "sync.py",
        "validate.py"
      ],
      "run": "sync-opus-r2",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 11,
      "run_tests_calls": 1,
      "api_input_tokens": 10848,
      "api_output_tokens": 4644,
      "model_response_seconds": 42.58182524899894
    },
    {
      "task": "sync",
      "model": "alibaba/qwen4-max",
      "repeat": 1,
      "stop": "model-finished",
      "elapsed_seconds": 97.40064916601114,
      "usage_cost_usd": "0.0390162",
      "turns": 7,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_page_atomic (test_visible.Visible.test_bad_page_atomic) ... ok\ntest_out_of_order (test_visible.Visible.test_out_of_order) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.001s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_bad_page_atomic\", \"test_visible.Visible.test_out_of_order\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_bad_page_atomic",
              "test_visible.Visible.test_out_of_order"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_rollback (test_hidden.Hidden.test_conflict_rollback) ... ok\ntest_cursor_conflict (test_hidden.Hidden.test_cursor_conflict) ... ok\ntest_older_cursor_conflict (test_hidden.Hidden.test_older_cursor_conflict) ... ok\ntest_replay_newer_and_validation (test_hidden.Hidden.test_replay_newer_and_validation) ... ok\ntest_restart_tombstone (test_hidden.Hidden.test_restart_tombstone) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.003s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_conflict_rollback\", \"test_hidden.Hidden.test_cursor_conflict\", \"test_hidden.Hidden.test_older_cursor_conflict\", \"test_hidden.Hidden.test_replay_newer_and_validation\", \"test_hidden.Hidden.test_restart_tombstone\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_rollback",
              "test_hidden.Hidden.test_cursor_conflict",
              "test_hidden.Hidden.test_older_cursor_conflict",
              "test_hidden.Hidden.test_replay_newer_and_validation",
              "test_hidden.Hidden.test_restart_tombstone"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "store.py",
        "sync.py",
        "validate.py"
      ],
      "run": "sync-qwen-r1",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 11,
      "run_tests_calls": 1,
      "api_input_tokens": 20255,
      "api_output_tokens": 4837,
      "model_response_seconds": 97.13767120700504
    },
    {
      "task": "sync",
      "model": "alibaba/qwen4-max",
      "repeat": 2,
      "stop": "model-finished",
      "elapsed_seconds": 70.39538212500338,
      "usage_cost_usd": "0.0286374",
      "turns": 6,
      "grade": {
        "visible": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_bad_page_atomic (test_visible.Visible.test_bad_page_atomic) ... ok\ntest_out_of_order (test_visible.Visible.test_out_of_order) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.002s\n\nOK\nKINGY_RESULT:{\"testsRun\": 2, \"test_ids\": [\"test_visible.Visible.test_bad_page_atomic\", \"test_visible.Visible.test_out_of_order\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 2,
            "test_ids": [
              "test_visible.Visible.test_bad_page_atomic",
              "test_visible.Visible.test_out_of_order"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        },
        "hidden": {
          "exit_code": 0,
          "timeout": false,
          "output": "test_conflict_rollback (test_hidden.Hidden.test_conflict_rollback) ... ok\ntest_cursor_conflict (test_hidden.Hidden.test_cursor_conflict) ... ok\ntest_older_cursor_conflict (test_hidden.Hidden.test_older_cursor_conflict) ... ok\ntest_replay_newer_and_validation (test_hidden.Hidden.test_replay_newer_and_validation) ... ok\ntest_restart_tombstone (test_hidden.Hidden.test_restart_tombstone) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.004s\n\nOK\nKINGY_RESULT:{\"testsRun\": 5, \"test_ids\": [\"test_hidden.Hidden.test_conflict_rollback\", \"test_hidden.Hidden.test_cursor_conflict\", \"test_hidden.Hidden.test_older_cursor_conflict\", \"test_hidden.Hidden.test_replay_newer_and_validation\", \"test_hidden.Hidden.test_restart_tombstone\"], \"failures\": [], \"errors\": [], \"skipped\": []}\n",
          "output_truncated": false,
          "sandbox": "macOS sandbox-exec deny-default, no network, read allowlist, scratch-only writes",
          "tests": {
            "testsRun": 5,
            "test_ids": [
              "test_hidden.Hidden.test_conflict_rollback",
              "test_hidden.Hidden.test_cursor_conflict",
              "test_hidden.Hidden.test_older_cursor_conflict",
              "test_hidden.Hidden.test_replay_newer_and_validation",
              "test_hidden.Hidden.test_restart_tombstone"
            ],
            "failures": [],
            "errors": [],
            "skipped": []
          }
        }
      },
      "passed": true,
      "changed_files": [
        "store.py",
        "sync.py",
        "validate.py"
      ],
      "run": "sync-qwen-r2",
      "visible_total": 2,
      "visible_failed": 0,
      "visible_passed": 2,
      "hidden_total": 5,
      "hidden_failed": 0,
      "hidden_passed": 5,
      "tool_calls": 11,
      "run_tests_calls": 1,
      "api_input_tokens": 13721,
      "api_output_tokens": 3979,
      "model_response_seconds": 70.09181870998873
    }
  ],
  "all_runs": 16,
  "cost_usd": "1.3178949"
}
