{
  "repetitions": [
    {
      "repetition": 1,
      "attempts": 40,
      "answerable_count": 30,
      "answerable_passes": 30,
      "unanswerable_count": 10,
      "correct_abstentions": 10,
      "citation_count": 35,
      "valid_citation_count": 35,
      "answer_correctness": 1.0,
      "citation_precision": 1.0,
      "abstention_accuracy": 1.0,
      "qualified": true,
      "by_category": {
        "consecutive-page procedure": {
          "attempts": 5,
          "passed": 5
        },
        "current versus archived": {
          "attempts": 5,
          "passed": 5
        },
        "interval extraction": {
          "attempts": 3,
          "passed": 3
        },
        "limit and measurement scope": {
          "attempts": 1,
          "passed": 1
        },
        "money and per-item basis": {
          "attempts": 5,
          "passed": 5
        },
        "per-booking count": {
          "attempts": 1,
          "passed": 1
        },
        "role-bound thresholds": {
          "attempts": 5,
          "passed": 5
        },
        "single-page person": {
          "attempts": 5,
          "passed": 5
        },
        "whole-corpus absence": {
          "attempts": 10,
          "passed": 10
        }
      },
      "model_calls": 40,
      "model_request_count": 89,
      "guard_only_answers": 0
    },
    {
      "repetition": 2,
      "attempts": 40,
      "answerable_count": 30,
      "answerable_passes": 30,
      "unanswerable_count": 10,
      "correct_abstentions": 10,
      "citation_count": 35,
      "valid_citation_count": 35,
      "answer_correctness": 1.0,
      "citation_precision": 1.0,
      "abstention_accuracy": 1.0,
      "qualified": true,
      "by_category": {
        "consecutive-page procedure": {
          "attempts": 5,
          "passed": 5
        },
        "current versus archived": {
          "attempts": 5,
          "passed": 5
        },
        "interval extraction": {
          "attempts": 3,
          "passed": 3
        },
        "limit and measurement scope": {
          "attempts": 1,
          "passed": 1
        },
        "money and per-item basis": {
          "attempts": 5,
          "passed": 5
        },
        "per-booking count": {
          "attempts": 1,
          "passed": 1
        },
        "role-bound thresholds": {
          "attempts": 5,
          "passed": 5
        },
        "single-page person": {
          "attempts": 5,
          "passed": 5
        },
        "whole-corpus absence": {
          "attempts": 10,
          "passed": 10
        }
      },
      "model_calls": 40,
      "model_request_count": 89,
      "guard_only_answers": 0
    },
    {
      "repetition": 3,
      "attempts": 40,
      "answerable_count": 30,
      "answerable_passes": 30,
      "unanswerable_count": 10,
      "correct_abstentions": 10,
      "citation_count": 35,
      "valid_citation_count": 35,
      "answer_correctness": 1.0,
      "citation_precision": 1.0,
      "abstention_accuracy": 1.0,
      "qualified": true,
      "by_category": {
        "consecutive-page procedure": {
          "attempts": 5,
          "passed": 5
        },
        "current versus archived": {
          "attempts": 5,
          "passed": 5
        },
        "interval extraction": {
          "attempts": 3,
          "passed": 3
        },
        "limit and measurement scope": {
          "attempts": 1,
          "passed": 1
        },
        "money and per-item basis": {
          "attempts": 5,
          "passed": 5
        },
        "per-booking count": {
          "attempts": 1,
          "passed": 1
        },
        "role-bound thresholds": {
          "attempts": 5,
          "passed": 5
        },
        "single-page person": {
          "attempts": 5,
          "passed": 5
        },
        "whole-corpus absence": {
          "attempts": 10,
          "passed": 10
        }
      },
      "model_calls": 40,
      "model_request_count": 89,
      "guard_only_answers": 0
    }
  ],
  "complete_planned_evaluation": true,
  "qualified": true,
  "measuredAt": "2026-10-19T03:23:21Z",
  "runner_sha256": "3bcc03ffef818fa7fb6062ae17f20aa09d2fcc25fc9d57572c77ec35bc1ef02b",
  "freeze_sha256": "2217abc61fe8062af322dccb71a371ab7825286f5f9857e6e35203a349fadd81",
  "sessions": [
    {
      "repetition": 1,
      "runtime": {
        "path": "[TEST_ROOT]/runtime/llama-b11429/llama-server",
        "sha256": "1bf5602a2be50097c44282a6ecc63234d94ee8f96d39fd45fed6eff4e74b6b41",
        "version": "version: 0.6.0-dev (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000101 for Darwin arm64"
      },
      "cold_process_to_health_seconds": 1.548835917012184,
      "sampled_server_peak_rss_KiB": 4885888,
      "rss_samples": 386,
      "rss_sample_interval_seconds": 1,
      "before": {
        "os": {
          "exit_code": 0,
          "output": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0.1\nBuildVersion:\t\t26A434",
          "error": ""
        },
        "memory": {
          "exit_code": 0,
          "output": "The system has 17179869184 (1048576 pages with a page size of 16384).\nSystem-wide memory free percentage: 85%",
          "error": ""
        },
        "swap": {
          "exit_code": 0,
          "output": "total = 0.00M  used = 0.00M  free = 0.00M  (encrypted)",
          "error": ""
        },
        "disk": {
          "exit_code": 0,
          "output": "Filesystem        Size    Used   Avail Capacity iused ifree %iused  Mounted on\n/dev/disk3s1s1   228Gi    13Gi   176Gi     7%    484k  1.9G    0%   /",
          "error": ""
        }
      },
      "after": {
        "os": {
          "exit_code": 0,
          "output": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0.1\nBuildVersion:\t\t26A434",
          "error": ""
        },
        "memory": {
          "exit_code": 0,
          "output": "The system has 17179869184 (1048576 pages with a page size of 16384).\nSystem-wide memory free percentage: 64%",
          "error": ""
        },
        "swap": {
          "exit_code": 0,
          "output": "total = 0.00M  used = 0.00M  free = 0.00M  (encrypted)",
          "error": ""
        },
        "disk": {
          "exit_code": 0,
          "output": "Filesystem        Size    Used   Avail Capacity iused ifree %iused  Mounted on\n/dev/disk3s1s1   228Gi    13Gi   177Gi     7%    484k  1.9G    0%   /",
          "error": ""
        }
      },
      "attempts": 40,
      "failures": 0
    },
    {
      "repetition": 2,
      "runtime": {
        "path": "[TEST_ROOT]/runtime/llama-b11429/llama-server",
        "sha256": "1bf5602a2be50097c44282a6ecc63234d94ee8f96d39fd45fed6eff4e74b6b41",
        "version": "version: 0.6.0-dev (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000101 for Darwin arm64"
      },
      "cold_process_to_health_seconds": 1.544222707991139,
      "sampled_server_peak_rss_KiB": 4886256,
      "rss_samples": 390,
      "rss_sample_interval_seconds": 1,
      "before": {
        "os": {
          "exit_code": 0,
          "output": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0.1\nBuildVersion:\t\t26A434",
          "error": ""
        },
        "memory": {
          "exit_code": 0,
          "output": "The system has 17179869184 (1048576 pages with a page size of 16384).\nSystem-wide memory free percentage: 87%",
          "error": ""
        },
        "swap": {
          "exit_code": 0,
          "output": "total = 0.00M  used = 0.00M  free = 0.00M  (encrypted)",
          "error": ""
        },
        "disk": {
          "exit_code": 0,
          "output": "Filesystem        Size    Used   Avail Capacity iused ifree %iused  Mounted on\n/dev/disk3s1s1   228Gi    13Gi   177Gi     7%    484k  1.9G    0%   /",
          "error": ""
        }
      },
      "after": {
        "os": {
          "exit_code": 0,
          "output": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0.1\nBuildVersion:\t\t26A434",
          "error": ""
        },
        "memory": {
          "exit_code": 0,
          "output": "The system has 17179869184 (1048576 pages with a page size of 16384).\nSystem-wide memory free percentage: 64%",
          "error": ""
        },
        "swap": {
          "exit_code": 0,
          "output": "total = 0.00M  used = 0.00M  free = 0.00M  (encrypted)",
          "error": ""
        },
        "disk": {
          "exit_code": 0,
          "output": "Filesystem        Size    Used   Avail Capacity iused ifree %iused  Mounted on\n/dev/disk3s1s1   228Gi    13Gi   176Gi     7%    484k  1.9G    0%   /",
          "error": ""
        }
      },
      "attempts": 40,
      "failures": 0
    },
    {
      "repetition": 3,
      "runtime": {
        "path": "[TEST_ROOT]/runtime/llama-b11429/llama-server",
        "sha256": "1bf5602a2be50097c44282a6ecc63234d94ee8f96d39fd45fed6eff4e74b6b41",
        "version": "version: 0.6.0-dev (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000101 for Darwin arm64"
      },
      "cold_process_to_health_seconds": 1.5418534170021303,
      "sampled_server_peak_rss_KiB": 4889024,
      "rss_samples": 391,
      "rss_sample_interval_seconds": 1,
      "before": {
        "os": {
          "exit_code": 0,
          "output": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0.1\nBuildVersion:\t\t26A434",
          "error": ""
        },
        "memory": {
          "exit_code": 0,
          "output": "The system has 17179869184 (1048576 pages with a page size of 16384).\nSystem-wide memory free percentage: 87%",
          "error": ""
        },
        "swap": {
          "exit_code": 0,
          "output": "total = 0.00M  used = 0.00M  free = 0.00M  (encrypted)",
          "error": ""
        },
        "disk": {
          "exit_code": 0,
          "output": "Filesystem        Size    Used   Avail Capacity iused ifree %iused  Mounted on\n/dev/disk3s1s1   228Gi    13Gi   176Gi     7%    484k  1.9G    0%   /",
          "error": ""
        }
      },
      "after": {
        "os": {
          "exit_code": 0,
          "output": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0.1\nBuildVersion:\t\t26A434",
          "error": ""
        },
        "memory": {
          "exit_code": 0,
          "output": "The system has 17179869184 (1048576 pages with a page size of 16384).\nSystem-wide memory free percentage: 64%",
          "error": ""
        },
        "swap": {
          "exit_code": 0,
          "output": "total = 0.00M  used = 0.00M  free = 0.00M  (encrypted)",
          "error": ""
        },
        "disk": {
          "exit_code": 0,
          "output": "Filesystem        Size    Used   Avail Capacity iused ifree %iused  Mounted on\n/dev/disk3s1s1   228Gi    13Gi   176Gi     7%    484k  1.9G    0%   /",
          "error": ""
        }
      },
      "attempts": 40,
      "failures": 0
    }
  ],
  "latency": {
    "count": 120,
    "min": 5.0372827499959385,
    "max": 18.910133332989062,
    "p50": 8.695213770508417,
    "p95": 18.452097874993342,
    "metric": "Complete pipeline answer_seconds: retrieval plus model HTTP/selection/resolution ; source selection and semantic sufficiency phases; excludes import/index and cold start."
  },
  "normalization_applied_attempts": 0,
  "note": "40 unique held-out questions repeated in three fresh model sessions; not120 independent questions/users. All planned attempts and pipeline corrections retained.",
  "model_calls": 120,
  "model_request_count": 267,
  "guard_only_answers": 0
}