{ "version": "kingy-paired-document-analysis/1.1.0", "grade_version": "kingy-common-document-grade/1.2.0", "planned_unique_questions": 40, "planned_repetitions": [ 1, 2, 3 ], "planned_attempts_per_system": 120, "integrity_issues": [], "complete_planned_evaluation": true, "all_semantic_judgments_resolved": false, "systems": { "A": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 3, "lower": 0.8666666666666667, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 0, "lower": 0.8666666666666667, "upper": 0.8666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 38, "total": 42, "unresolved": 0, "lower": 0.9047619047619048, "upper": 0.9047619047619048 }, "unresolved_questions": 3, "explicit_page_citations": 38, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "heldout-33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.636813416989753 ] }, { "question_id": "heldout-36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3495562089956366 ] }, { "question_id": "heldout-12", "row_indices": [ 36 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9924647499865387 ] }, { "question_id": "heldout-04", "row_indices": [ 16 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.165912875003414 ] }, { "question_id": "heldout-14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.145693833997939 ] }, { "question_id": "heldout-21", "row_indices": [ 15 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4775298339955043 ] }, { "question_id": "heldout-27", "row_indices": [ 38 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.9235311669763178 ] }, { "question_id": "heldout-26", "row_indices": [ 12 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.530061708006542 ] }, { "question_id": "heldout-34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5685942909913138 ] }, { "question_id": "heldout-28", "row_indices": [ 8 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7066478329943493 ] }, { "question_id": "heldout-07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.096409334015334 ] }, { "question_id": "heldout-09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.865960458002519 ] }, { "question_id": "heldout-35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.6362162079894915 ] }, { "question_id": "heldout-02", "row_indices": [ 17 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.384059707983397 ] }, { "question_id": "heldout-31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.8604643330036197 ] }, { "question_id": "heldout-37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.336942292022286 ] }, { "question_id": "heldout-30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.632482374989195 ] }, { "question_id": "heldout-22", "row_indices": [ 9 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.94570237500011 ] }, { "question_id": "heldout-15", "row_indices": [ 34 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.086040875001345 ] }, { "question_id": "heldout-25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.010730540991062 ] }, { "question_id": "heldout-05", "row_indices": [ 2 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.520921957999235 ] }, { "question_id": "heldout-08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.3903537089936435 ] }, { "question_id": "heldout-18", "row_indices": [ 32 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.908561042015208 ] }, { "question_id": "heldout-10", "row_indices": [ 26 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5043819999846164 ] }, { "question_id": "heldout-11", "row_indices": [ 29 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.036660457990365 ] }, { "question_id": "heldout-16", "row_indices": [ 6 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.8140477080014534 ] }, { "question_id": "heldout-20", "row_indices": [ 24 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.4041547079978045 ] }, { "question_id": "heldout-29", "row_indices": [ 5 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.302947582997149 ] }, { "question_id": "heldout-32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7352241250046063 ] }, { "question_id": "heldout-03", "row_indices": [ 35 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.015138999995543 ] }, { "question_id": "heldout-38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.8259617919975426 ] }, { "question_id": "heldout-19", "row_indices": [ 37 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6319284579949453 ] }, { "question_id": "heldout-06", "row_indices": [ 39 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.1712853749922942 ] }, { "question_id": "heldout-23", "row_indices": [ 31 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.8901268749905284 ] }, { "question_id": "heldout-24", "row_indices": [ 18 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.382717708009295 ] }, { "question_id": "heldout-39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.407791208010167 ] }, { "question_id": "heldout-01", "row_indices": [ 20 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.081111792009324 ] }, { "question_id": "heldout-17", "row_indices": [ 30 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.0026685830089264 ] }, { "question_id": "heldout-40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5065356669947505 ] }, { "question_id": "heldout-13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.632441291993018 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 3, "lower": 0.8666666666666667, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 0, "lower": 0.8666666666666667, "upper": 0.8666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 38, "total": 42, "unresolved": 0, "lower": 0.9047619047619048, "upper": 0.9047619047619048 }, "unresolved_questions": 3, "explicit_page_citations": 38, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "heldout-33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.174166916985996 ] }, { "question_id": "heldout-36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4191770420002285 ] }, { "question_id": "heldout-12", "row_indices": [ 49 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2423023330047727 ] }, { "question_id": "heldout-04", "row_indices": [ 59 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7125646249915008 ] }, { "question_id": "heldout-14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.64696987500065 ] }, { "question_id": "heldout-21", "row_indices": [ 69 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2790573749807663 ] }, { "question_id": "heldout-27", "row_indices": [ 47 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.222385416011093 ] }, { "question_id": "heldout-26", "row_indices": [ 51 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2187443749862723 ] }, { "question_id": "heldout-34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4886583750194404 ] }, { "question_id": "heldout-28", "row_indices": [ 46 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2598525829962455 ] }, { "question_id": "heldout-07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3237904169945978 ] }, { "question_id": "heldout-09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2822044170170557 ] }, { "question_id": "heldout-35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.296284290991025 ] }, { "question_id": "heldout-02", "row_indices": [ 42 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.931772625015583 ] }, { "question_id": "heldout-31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.3436109589820262 ] }, { "question_id": "heldout-37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.3664022909943014 ] }, { "question_id": "heldout-30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4323818330012728 ] }, { "question_id": "heldout-22", "row_indices": [ 52 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2621045829728246 ] }, { "question_id": "heldout-15", "row_indices": [ 61 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5484832919901237 ] }, { "question_id": "heldout-25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.335402708995389 ] }, { "question_id": "heldout-05", "row_indices": [ 70 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.524492582975654 ] }, { "question_id": "heldout-08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5328053339908365 ] }, { "question_id": "heldout-18", "row_indices": [ 65 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4157322910032235 ] }, { "question_id": "heldout-10", "row_indices": [ 71 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5244419170194305 ] }, { "question_id": "heldout-11", "row_indices": [ 63 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5340842500154395 ] }, { "question_id": "heldout-16", "row_indices": [ 62 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.3509967499994673 ] }, { "question_id": "heldout-20", "row_indices": [ 45 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7909144170116633 ] }, { "question_id": "heldout-29", "row_indices": [ 64 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.571869958017487 ] }, { "question_id": "heldout-32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2505160420259926 ] }, { "question_id": "heldout-03", "row_indices": [ 58 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.215775416989345 ] }, { "question_id": "heldout-38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.2966934170108289 ] }, { "question_id": "heldout-19", "row_indices": [ 54 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.976216834009392 ] }, { "question_id": "heldout-06", "row_indices": [ 56 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.3226067910145503 ] }, { "question_id": "heldout-23", "row_indices": [ 40 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.3920002079976257 ] }, { "question_id": "heldout-24", "row_indices": [ 50 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.3990704589814413 ] }, { "question_id": "heldout-39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.435122458002297 ] }, { "question_id": "heldout-01", "row_indices": [ 48 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4013785000133794 ] }, { "question_id": "heldout-17", "row_indices": [ 67 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.439856208016863 ] }, { "question_id": "heldout-40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.297828834009124 ] }, { "question_id": "heldout-13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8321481670136563 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 3, "lower": 0.8666666666666667, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 0, "lower": 0.8666666666666667, "upper": 0.8666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 38, "total": 42, "unresolved": 0, "lower": 0.9047619047619048, "upper": 0.9047619047619048 }, "unresolved_questions": 3, "explicit_page_citations": 38, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "heldout-33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.1796442499908153 ] }, { "question_id": "heldout-36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4434868750104215 ] }, { "question_id": "heldout-12", "row_indices": [ 113 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2360869580006693 ] }, { "question_id": "heldout-04", "row_indices": [ 87 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7102763750008307 ] }, { "question_id": "heldout-14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6443391669890843 ] }, { "question_id": "heldout-21", "row_indices": [ 117 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.272413416998461 ] }, { "question_id": "heldout-27", "row_indices": [ 111 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2412760840088595 ] }, { "question_id": "heldout-26", "row_indices": [ 88 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2331962089810986 ] }, { "question_id": "heldout-34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.49324554100167 ] }, { "question_id": "heldout-28", "row_indices": [ 86 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2545603339967784 ] }, { "question_id": "heldout-07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3453246249991935 ] }, { "question_id": "heldout-09", "row_indices": [ 102 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.282090958004119 ] }, { "question_id": "heldout-35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.293425167008536 ] }, { "question_id": "heldout-02", "row_indices": [ 98 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9323652909952216 ] }, { "question_id": "heldout-31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.3433942499978002 ] }, { "question_id": "heldout-37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.377816917025484 ] }, { "question_id": "heldout-30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.44228975000442 ] }, { "question_id": "heldout-22", "row_indices": [ 89 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2745265839912463 ] }, { "question_id": "heldout-15", "row_indices": [ 114 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.532383750018198 ] }, { "question_id": "heldout-25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3622731659852434 ] }, { "question_id": "heldout-05", "row_indices": [ 83 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5251582080090884 ] }, { "question_id": "heldout-08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.536856999999145 ] }, { "question_id": "heldout-18", "row_indices": [ 80 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4318876669858582 ] }, { "question_id": "heldout-10", "row_indices": [ 112 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.523794959008228 ] }, { "question_id": "heldout-11", "row_indices": [ 107 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.520506083004875 ] }, { "question_id": "heldout-16", "row_indices": [ 81 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.350382582983002 ] }, { "question_id": "heldout-20", "row_indices": [ 95 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8036688750144094 ] }, { "question_id": "heldout-29", "row_indices": [ 97 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5650471670087427 ] }, { "question_id": "heldout-32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2452507500129286 ] }, { "question_id": "heldout-03", "row_indices": [ 110 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2170672500215005 ] }, { "question_id": "heldout-38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.2971440000110306 ] }, { "question_id": "heldout-19", "row_indices": [ 108 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.9770287919964176 ] }, { "question_id": "heldout-06", "row_indices": [ 82 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.317723084008321 ] }, { "question_id": "heldout-23", "row_indices": [ 85 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.3873465829819907 ] }, { "question_id": "heldout-24", "row_indices": [ 96 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4009059999953024 ] }, { "question_id": "heldout-39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.43218716600677 ] }, { "question_id": "heldout-01", "row_indices": [ 105 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.405261917010648 ] }, { "question_id": "heldout-17", "row_indices": [ 104 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4438932919874787 ] }, { "question_id": "heldout-40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.2960567499976605 ] }, { "question_id": "heldout-13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.803498416004004 ] } ] } ], "qualified": false }, "B": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 10, "total": 30, "unresolved": 13, "lower": 0.3333333333333333, "upper": 0.7666666666666667 }, "supported_answer_success": { "passed": 9, "total": 30, "unresolved": 11, "lower": 0.3, "upper": 0.6666666666666666 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 4, "lower": 0.6, "upper": 1.0 }, "citation_precision": { "passed": 38, "total": 48, "unresolved": 4, "lower": 0.7916666666666666, "upper": 0.875 }, "unresolved_questions": 17, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 38, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "qualified": false, "attempt_audit": [ { "question_id": "heldout-33", "row_indices": [ 1 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.001 ] }, { "question_id": "heldout-36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.849 ] }, { "question_id": "heldout-12", "row_indices": [ 36 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.41 ] }, { "question_id": "heldout-04", "row_indices": [ 16 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.761 ] }, { "question_id": "heldout-14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.861 ] }, { "question_id": "heldout-21", "row_indices": [ 15 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.073 ] }, { "question_id": "heldout-27", "row_indices": [ 38 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.785 ] }, { "question_id": "heldout-26", "row_indices": [ 12 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.515 ] }, { "question_id": "heldout-34", "row_indices": [ 23 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.24 ] }, { "question_id": "heldout-28", "row_indices": [ 8 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.64 ] }, { "question_id": "heldout-07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.862 ] }, { "question_id": "heldout-09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.47 ] }, { "question_id": "heldout-35", "row_indices": [ 11 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.359 ] }, { "question_id": "heldout-02", "row_indices": [ 17 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.751 ] }, { "question_id": "heldout-31", "row_indices": [ 25 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.153 ] }, { "question_id": "heldout-37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.908 ] }, { "question_id": "heldout-30", "row_indices": [ 27 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.913 ] }, { "question_id": "heldout-22", "row_indices": [ 9 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.336 ] }, { "question_id": "heldout-15", "row_indices": [ 34 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.791 ] }, { "question_id": "heldout-25", "row_indices": [ 14 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.108 ] }, { "question_id": "heldout-05", "row_indices": [ 2 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.744 ] }, { "question_id": "heldout-08", "row_indices": [ 28 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.919 ] }, { "question_id": "heldout-18", "row_indices": [ 32 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.098 ] }, { "question_id": "heldout-10", "row_indices": [ 26 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.384 ] }, { "question_id": "heldout-11", "row_indices": [ 29 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.82 ] }, { "question_id": "heldout-16", "row_indices": [ 6 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.399 ] }, { "question_id": "heldout-20", "row_indices": [ 24 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.673 ] }, { "question_id": "heldout-29", "row_indices": [ 5 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.491 ] }, { "question_id": "heldout-32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.475 ] }, { "question_id": "heldout-03", "row_indices": [ 35 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.525 ] }, { "question_id": "heldout-38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.128 ] }, { "question_id": "heldout-19", "row_indices": [ 37 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.896 ] }, { "question_id": "heldout-06", "row_indices": [ 39 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.353 ] }, { "question_id": "heldout-23", "row_indices": [ 31 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.026 ] }, { "question_id": "heldout-24", "row_indices": [ 18 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.773 ] }, { "question_id": "heldout-39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.721 ] }, { "question_id": "heldout-01", "row_indices": [ 20 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.467 ] }, { "question_id": "heldout-17", "row_indices": [ 30 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.579 ] }, { "question_id": "heldout-40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.891 ] }, { "question_id": "heldout-13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.614 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 10, "total": 30, "unresolved": 13, "lower": 0.3333333333333333, "upper": 0.7666666666666667 }, "supported_answer_success": { "passed": 9, "total": 30, "unresolved": 11, "lower": 0.3, "upper": 0.6666666666666666 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 4, "lower": 0.6, "upper": 1.0 }, "citation_precision": { "passed": 38, "total": 48, "unresolved": 4, "lower": 0.7916666666666666, "upper": 0.875 }, "unresolved_questions": 17, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 38, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "qualified": false, "attempt_audit": [ { "question_id": "heldout-33", "row_indices": [ 79 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.941 ] }, { "question_id": "heldout-36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.721 ] }, { "question_id": "heldout-12", "row_indices": [ 49 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.699 ] }, { "question_id": "heldout-04", "row_indices": [ 59 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.71 ] }, { "question_id": "heldout-14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.113 ] }, { "question_id": "heldout-21", "row_indices": [ 69 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.557 ] }, { "question_id": "heldout-27", "row_indices": [ 47 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.496 ] }, { "question_id": "heldout-26", "row_indices": [ 51 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.41 ] }, { "question_id": "heldout-34", "row_indices": [ 75 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.524 ] }, { "question_id": "heldout-28", "row_indices": [ 46 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.186 ] }, { "question_id": "heldout-07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.917 ] }, { "question_id": "heldout-09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.464 ] }, { "question_id": "heldout-35", "row_indices": [ 60 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.282 ] }, { "question_id": "heldout-02", "row_indices": [ 42 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.67 ] }, { "question_id": "heldout-31", "row_indices": [ 53 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.15 ] }, { "question_id": "heldout-37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.988 ] }, { "question_id": "heldout-30", "row_indices": [ 41 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.017 ] }, { "question_id": "heldout-22", "row_indices": [ 52 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.383 ] }, { "question_id": "heldout-15", "row_indices": [ 61 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.239 ] }, { "question_id": "heldout-25", "row_indices": [ 74 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.408 ] }, { "question_id": "heldout-05", "row_indices": [ 70 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.058 ] }, { "question_id": "heldout-08", "row_indices": [ 73 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.32 ] }, { "question_id": "heldout-18", "row_indices": [ 65 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.939 ] }, { "question_id": "heldout-10", "row_indices": [ 71 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.281 ] }, { "question_id": "heldout-11", "row_indices": [ 63 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.017 ] }, { "question_id": "heldout-16", "row_indices": [ 62 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.286 ] }, { "question_id": "heldout-20", "row_indices": [ 45 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.688 ] }, { "question_id": "heldout-29", "row_indices": [ 64 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.123 ] }, { "question_id": "heldout-32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.337 ] }, { "question_id": "heldout-03", "row_indices": [ 58 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.671 ] }, { "question_id": "heldout-38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.61 ] }, { "question_id": "heldout-19", "row_indices": [ 54 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.996 ] }, { "question_id": "heldout-06", "row_indices": [ 56 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.058 ] }, { "question_id": "heldout-23", "row_indices": [ 40 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.61 ] }, { "question_id": "heldout-24", "row_indices": [ 50 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.133 ] }, { "question_id": "heldout-39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.858 ] }, { "question_id": "heldout-01", "row_indices": [ 48 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.838 ] }, { "question_id": "heldout-17", "row_indices": [ 67 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.043 ] }, { "question_id": "heldout-40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.72 ] }, { "question_id": "heldout-13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.703 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 10, "total": 30, "unresolved": 13, "lower": 0.3333333333333333, "upper": 0.7666666666666667 }, "supported_answer_success": { "passed": 9, "total": 30, "unresolved": 11, "lower": 0.3, "upper": 0.6666666666666666 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 4, "lower": 0.6, "upper": 1.0 }, "citation_precision": { "passed": 38, "total": 48, "unresolved": 4, "lower": 0.7916666666666666, "upper": 0.875 }, "unresolved_questions": 17, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 38, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "qualified": false, "attempt_audit": [ { "question_id": "heldout-33", "row_indices": [ 115 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.424 ] }, { "question_id": "heldout-36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.91 ] }, { "question_id": "heldout-12", "row_indices": [ 113 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.75 ] }, { "question_id": "heldout-04", "row_indices": [ 87 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.316 ] }, { "question_id": "heldout-14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.974 ] }, { "question_id": "heldout-21", "row_indices": [ 117 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.665 ] }, { "question_id": "heldout-27", "row_indices": [ 111 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.85 ] }, { "question_id": "heldout-26", "row_indices": [ 88 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.221 ] }, { "question_id": "heldout-34", "row_indices": [ 90 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.318 ] }, { "question_id": "heldout-28", "row_indices": [ 86 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.816 ] }, { "question_id": "heldout-07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.238 ] }, { "question_id": "heldout-09", "row_indices": [ 102 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.887 ] }, { "question_id": "heldout-35", "row_indices": [ 93 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.022 ] }, { "question_id": "heldout-02", "row_indices": [ 98 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.873 ] }, { "question_id": "heldout-31", "row_indices": [ 119 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.145 ] }, { "question_id": "heldout-37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.917 ] }, { "question_id": "heldout-30", "row_indices": [ 109 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.29 ] }, { "question_id": "heldout-22", "row_indices": [ 89 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.554 ] }, { "question_id": "heldout-15", "row_indices": [ 114 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.907 ] }, { "question_id": "heldout-25", "row_indices": [ 106 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.966 ] }, { "question_id": "heldout-05", "row_indices": [ 83 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.869 ] }, { "question_id": "heldout-08", "row_indices": [ 116 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.051 ] }, { "question_id": "heldout-18", "row_indices": [ 80 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.738 ] }, { "question_id": "heldout-10", "row_indices": [ 112 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.919 ] }, { "question_id": "heldout-11", "row_indices": [ 107 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.788 ] }, { "question_id": "heldout-16", "row_indices": [ 81 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.759 ] }, { "question_id": "heldout-20", "row_indices": [ 95 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.767 ] }, { "question_id": "heldout-29", "row_indices": [ 97 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.056 ] }, { "question_id": "heldout-32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.534 ] }, { "question_id": "heldout-03", "row_indices": [ 110 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.876 ] }, { "question_id": "heldout-38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.702 ] }, { "question_id": "heldout-19", "row_indices": [ 108 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.36 ] }, { "question_id": "heldout-06", "row_indices": [ 82 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.675 ] }, { "question_id": "heldout-23", "row_indices": [ 85 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.89 ] }, { "question_id": "heldout-24", "row_indices": [ 96 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.515 ] }, { "question_id": "heldout-39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.825 ] }, { "question_id": "heldout-01", "row_indices": [ 105 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.993 ] }, { "question_id": "heldout-17", "row_indices": [ 104 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.004 ] }, { "question_id": "heldout-40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.725 ] }, { "question_id": "heldout-13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.334 ] } ] } ], "qualified": false } }, "paired_effects_A_minus_B": { "complete_question_success": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.15, 0.525 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.025, 0.7 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "supported_answer_success": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.2, 0.5666666666666667 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.7666666666666667 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "canonical_fact_correctness": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.1, 0.6333333333333333 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.1, 0.8 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "correct_abstention": { "unique_questions": 10, "effect_bounds_A_minus_B": [ 0.0, 0.4 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.7 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "emitted_citation_precision": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.029761904761904767, 0.11309523809523814 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.132449790045528, 0.29166666666666663 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." } }, "bootstrap_seed": 271828, "bootstrap_draws": 10000, "comparative_claim_gate": { "eligible_for_review": false, "withhold_reasons": [ "Unreviewed semantic/citation ambiguity remains." ], "automatic_winner": null, "p_values": null, "scope": "Fixed synthetic corpus on the declared single Mac; case-resampling sensitivity does not establish a population or market-wide winner." }, "session_provenance_complete": true, "input_sha256": { "comparison_freeze": "b0dde55a2f92f156f588d5df510832b2ea874e3da41e072ab096564269356d43", "references": "3482343d9dfa0ff31edcc521e35cff23ce8d25b1bb195ae028157aec9ac10f0b", "questions": "2c09fe76f96bfcfc59d06f3517c119d071b8555031200baefc1cd5d92f4d4486", "analysis_source": "5412933b74ee67dd5877a53643548948c585fc15c333c4906e5c8e718e66c1bc" } }