{ "version": "kingy-paired-document-analysis/1.1.0", "grade_version": "kingy-common-document-grade/1.5.0", "planned_unique_questions": 40, "planned_repetitions": [ 1, 2, 3 ], "planned_attempts_per_system": 120, "integrity_issues": [], "complete_planned_evaluation": true, "all_semantic_judgments_resolved": false, "systems": { "A": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 28, "total": 30, "unresolved": 2, "lower": 0.9333333333333333, "upper": 1.0 }, "supported_answer_success": { "passed": 28, "total": 30, "unresolved": 1, "lower": 0.9333333333333333, "upper": 0.9666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 34, "total": 35, "unresolved": 0, "lower": 0.9714285714285714, "upper": 0.9714285714285714 }, "unresolved_questions": 2, "explicit_page_citations": 34, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": true, "abstention_accuracy": true }, "qualified": true, "attempt_audit": [ { "question_id": "v20-q19", "row_indices": [ 37 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.278437665998354 ] }, { "question_id": "v20-q04", "row_indices": [ 16 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.85171654199803 ] }, { "question_id": "v20-q17", "row_indices": [ 30 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.354396417002135 ] }, { "question_id": "v20-q30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.519191874998796 ] }, { "question_id": "v20-q21", "row_indices": [ 15 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.373060916994291 ] }, { "question_id": "v20-q28", "row_indices": [ 8 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.159678542004258 ] }, { "question_id": "v20-q36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.434070249997603 ] }, { "question_id": "v20-q03", "row_indices": [ 35 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.866285957999935 ] }, { "question_id": "v20-q16", "row_indices": [ 6 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9891143330023624 ] }, { "question_id": "v20-q26", "row_indices": [ 12 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.630747208000685 ] }, { "question_id": "v20-q12", "row_indices": [ 36 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.519381958998565 ] }, { "question_id": "v20-q35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.485502584000642 ] }, { "question_id": "v20-q22", "row_indices": [ 9 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.033785584004363 ] }, { "question_id": "v20-q15", "row_indices": [ 34 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.312630375003209 ] }, { "question_id": "v20-q34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.341939790996548 ] }, { "question_id": "v20-q11", "row_indices": [ 29 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.370410791001632 ] }, { "question_id": "v20-q05", "row_indices": [ 2 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.230467583001882 ] }, { "question_id": "v20-q25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.552071167003305 ] }, { "question_id": "v20-q24", "row_indices": [ 18 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.18540375000157 ] }, { "question_id": "v20-q29", "row_indices": [ 5 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.721146708005108 ] }, { "question_id": "v20-q02", "row_indices": [ 17 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.161626542001613 ] }, { "question_id": "v20-q20", "row_indices": [ 24 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.761339707998559 ] }, { "question_id": "v20-q33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.061599542001204 ] }, { "question_id": "v20-q37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.935954665997997 ] }, { "question_id": "v20-q23", "row_indices": [ 31 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.958212666999316 ] }, { "question_id": "v20-q32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.052076374995522 ] }, { "question_id": "v20-q08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.828426125000988 ] }, { "question_id": "v20-q07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.426338500001293 ] }, { "question_id": "v20-q13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.236181583000871 ] }, { "question_id": "v20-q09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.739663042004395 ] }, { "question_id": "v20-q27", "row_indices": [ 38 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.545273666997673 ] }, { "question_id": "v20-q38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.6109999580003205 ] }, { "question_id": "v20-q01", "row_indices": [ 20 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.232620458999008 ] }, { "question_id": "v20-q06", "row_indices": [ 39 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.62807291700301 ] }, { "question_id": "v20-q39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.055515874999401 ] }, { "question_id": "v20-q18", "row_indices": [ 32 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.329500874999212 ] }, { "question_id": "v20-q10", "row_indices": [ 26 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.8693574580029235 ] }, { "question_id": "v20-q31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.14645895799913 ] }, { "question_id": "v20-q40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.406251166998118 ] }, { "question_id": "v20-q14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.6213728329967125 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 28, "total": 30, "unresolved": 2, "lower": 0.9333333333333333, "upper": 1.0 }, "supported_answer_success": { "passed": 28, "total": 30, "unresolved": 1, "lower": 0.9333333333333333, "upper": 0.9666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 34, "total": 35, "unresolved": 0, "lower": 0.9714285714285714, "upper": 0.9714285714285714 }, "unresolved_questions": 2, "explicit_page_citations": 34, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": true, "abstention_accuracy": true }, "qualified": true, "attempt_audit": [ { "question_id": "v20-q19", "row_indices": [ 54 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.740196291997563 ] }, { "question_id": "v20-q04", "row_indices": [ 59 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.100968624996312 ] }, { "question_id": "v20-q17", "row_indices": [ 67 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.191510292002931 ] }, { "question_id": "v20-q30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.519565250004234 ] }, { "question_id": "v20-q21", "row_indices": [ 69 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8491394169977866 ] }, { "question_id": "v20-q28", "row_indices": [ 46 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6496552919998066 ] }, { "question_id": "v20-q36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7376118339961977 ] }, { "question_id": "v20-q03", "row_indices": [ 58 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.130954208994808 ] }, { "question_id": "v20-q16", "row_indices": [ 62 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.430393709000782 ] }, { "question_id": "v20-q26", "row_indices": [ 51 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.065957166996668 ] }, { "question_id": "v20-q12", "row_indices": [ 49 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7589196659973823 ] }, { "question_id": "v20-q35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9354184999974677 ] }, { "question_id": "v20-q22", "row_indices": [ 52 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5546205840000766 ] }, { "question_id": "v20-q15", "row_indices": [ 61 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7743818329981877 ] }, { "question_id": "v20-q34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8396694580005715 ] }, { "question_id": "v20-q11", "row_indices": [ 63 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.220349083996553 ] }, { "question_id": "v20-q05", "row_indices": [ 70 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.675088666997908 ] }, { "question_id": "v20-q25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.903826667003159 ] }, { "question_id": "v20-q24", "row_indices": [ 50 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.9816915830015205 ] }, { "question_id": "v20-q29", "row_indices": [ 64 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.586875999993936 ] }, { "question_id": "v20-q02", "row_indices": [ 42 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.698233583003457 ] }, { "question_id": "v20-q20", "row_indices": [ 45 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.130046917001891 ] }, { "question_id": "v20-q33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5649701250004 ] }, { "question_id": "v20-q37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4374060420013848 ] }, { "question_id": "v20-q23", "row_indices": [ 40 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4658720000006724 ] }, { "question_id": "v20-q32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5529799580035615 ] }, { "question_id": "v20-q08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.131052374999854 ] }, { "question_id": "v20-q07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.767274499994528 ] }, { "question_id": "v20-q13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.687800250001601 ] }, { "question_id": "v20-q09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.163926083994738 ] }, { "question_id": "v20-q27", "row_indices": [ 47 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9311781249998603 ] }, { "question_id": "v20-q38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.93695870800002 ] }, { "question_id": "v20-q01", "row_indices": [ 48 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7057094170013443 ] }, { "question_id": "v20-q06", "row_indices": [ 56 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9181133750025765 ] }, { "question_id": "v20-q39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5490334999994957 ] }, { "question_id": "v20-q18", "row_indices": [ 65 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.758083833003184 ] }, { "question_id": "v20-q10", "row_indices": [ 71 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.2291900839991285 ] }, { "question_id": "v20-q31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.644865000002028 ] }, { "question_id": "v20-q40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.868318041000748 ] }, { "question_id": "v20-q14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.003806291002547 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 28, "total": 30, "unresolved": 2, "lower": 0.9333333333333333, "upper": 1.0 }, "supported_answer_success": { "passed": 28, "total": 30, "unresolved": 1, "lower": 0.9333333333333333, "upper": 0.9666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 34, "total": 35, "unresolved": 0, "lower": 0.9714285714285714, "upper": 0.9714285714285714 }, "unresolved_questions": 2, "explicit_page_citations": 34, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": true, "abstention_accuracy": true }, "qualified": true, "attempt_audit": [ { "question_id": "v20-q19", "row_indices": [ 108 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7165998329946888 ] }, { "question_id": "v20-q04", "row_indices": [ 87 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.109484666994831 ] }, { "question_id": "v20-q17", "row_indices": [ 104 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.098139916000946 ] }, { "question_id": "v20-q30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.5380274999988615 ] }, { "question_id": "v20-q21", "row_indices": [ 117 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8472949170027277 ] }, { "question_id": "v20-q28", "row_indices": [ 86 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.631551792001119 ] }, { "question_id": "v20-q36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7266127499970025 ] }, { "question_id": "v20-q03", "row_indices": [ 110 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.129725332997623 ] }, { "question_id": "v20-q16", "row_indices": [ 81 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4170090000043274 ] }, { "question_id": "v20-q26", "row_indices": [ 88 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.059974084004352 ] }, { "question_id": "v20-q12", "row_indices": [ 113 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6993150830021477 ] }, { "question_id": "v20-q35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8778549160051625 ] }, { "question_id": "v20-q22", "row_indices": [ 89 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.53255808299582 ] }, { "question_id": "v20-q15", "row_indices": [ 114 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7817240419972222 ] }, { "question_id": "v20-q34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8028252080039238 ] }, { "question_id": "v20-q11", "row_indices": [ 107 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.20108195800276 ] }, { "question_id": "v20-q05", "row_indices": [ 83 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.631243916002859 ] }, { "question_id": "v20-q25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8986104580035317 ] }, { "question_id": "v20-q24", "row_indices": [ 96 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.944941415997164 ] }, { "question_id": "v20-q29", "row_indices": [ 97 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.563864707997709 ] }, { "question_id": "v20-q02", "row_indices": [ 98 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6841086669955985 ] }, { "question_id": "v20-q20", "row_indices": [ 95 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.1093802909963415 ] }, { "question_id": "v20-q33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5664886250015115 ] }, { "question_id": "v20-q37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.440372290999221 ] }, { "question_id": "v20-q23", "row_indices": [ 85 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4125556250000955 ] }, { "question_id": "v20-q32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.537635250002495 ] }, { "question_id": "v20-q08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.147559083001397 ] }, { "question_id": "v20-q07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7565375000049244 ] }, { "question_id": "v20-q13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.680767375000869 ] }, { "question_id": "v20-q09", "row_indices": [ 102 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.137047791999066 ] }, { "question_id": "v20-q27", "row_indices": [ 111 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9646672919989214 ] }, { "question_id": "v20-q38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9102783749985974 ] }, { "question_id": "v20-q01", "row_indices": [ 105 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.685002625003108 ] }, { "question_id": "v20-q06", "row_indices": [ 82 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9560736249986803 ] }, { "question_id": "v20-q39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.520498124998994 ] }, { "question_id": "v20-q18", "row_indices": [ 80 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.744309499998053 ] }, { "question_id": "v20-q10", "row_indices": [ 112 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.233207833000051 ] }, { "question_id": "v20-q31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6404818749942933 ] }, { "question_id": "v20-q40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.882726832998742 ] }, { "question_id": "v20-q14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.967029999999795 ] } ] } ], "qualified": true }, "B": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 16, "total": 30, "unresolved": 13, "lower": 0.5333333333333333, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 14, "total": 30, "unresolved": 9, "lower": 0.4666666666666667, "upper": 0.7666666666666667 }, "abstention_accuracy": { "passed": 9, "total": 10, "unresolved": 1, "lower": 0.9, "upper": 1.0 }, "citation_precision": { "passed": 26, "total": 34, "unresolved": 3, "lower": 0.7647058823529411, "upper": 0.8529411764705882 }, "unresolved_questions": 15, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 26, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "v20-q19", "row_indices": [ 37 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.758 ] }, { "question_id": "v20-q04", "row_indices": [ 16 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.553 ] }, { "question_id": "v20-q17", "row_indices": [ 30 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.845 ] }, { "question_id": "v20-q30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.493 ] }, { "question_id": "v20-q21", "row_indices": [ 15 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.988 ] }, { "question_id": "v20-q28", "row_indices": [ 8 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.498 ] }, { "question_id": "v20-q36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.592 ] }, { "question_id": "v20-q03", "row_indices": [ 35 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.629 ] }, { "question_id": "v20-q16", "row_indices": [ 6 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.613 ] }, { "question_id": "v20-q26", "row_indices": [ 12 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.337 ] }, { "question_id": "v20-q12", "row_indices": [ 36 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.969 ] }, { "question_id": "v20-q35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.47 ] }, { "question_id": "v20-q22", "row_indices": [ 9 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.174 ] }, { "question_id": "v20-q15", "row_indices": [ 34 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.444 ] }, { "question_id": "v20-q34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.205 ] }, { "question_id": "v20-q11", "row_indices": [ 29 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.648 ] }, { "question_id": "v20-q05", "row_indices": [ 2 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.326 ] }, { "question_id": "v20-q25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.533 ] }, { "question_id": "v20-q24", "row_indices": [ 18 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.218 ] }, { "question_id": "v20-q29", "row_indices": [ 5 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.367 ] }, { "question_id": "v20-q02", "row_indices": [ 17 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.275 ] }, { "question_id": "v20-q20", "row_indices": [ 24 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.642 ] }, { "question_id": "v20-q33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.501 ] }, { "question_id": "v20-q37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.428 ] }, { "question_id": "v20-q23", "row_indices": [ 31 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.642 ] }, { "question_id": "v20-q32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.429 ] }, { "question_id": "v20-q08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.204 ] }, { "question_id": "v20-q07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.893 ] }, { "question_id": "v20-q13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.58 ] }, { "question_id": "v20-q09", "row_indices": [ 10 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.796 ] }, { "question_id": "v20-q27", "row_indices": [ 38 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.137 ] }, { "question_id": "v20-q38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.948 ] }, { "question_id": "v20-q01", "row_indices": [ 20 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.577 ] }, { "question_id": "v20-q06", "row_indices": [ 39 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.182 ] }, { "question_id": "v20-q39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.504 ] }, { "question_id": "v20-q18", "row_indices": [ 32 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.768 ] }, { "question_id": "v20-q10", "row_indices": [ 26 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.872 ] }, { "question_id": "v20-q31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.66 ] }, { "question_id": "v20-q40", "row_indices": [ 21 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.458 ] }, { "question_id": "v20-q14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.534 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 16, "total": 30, "unresolved": 13, "lower": 0.5333333333333333, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 14, "total": 30, "unresolved": 9, "lower": 0.4666666666666667, "upper": 0.7666666666666667 }, "abstention_accuracy": { "passed": 9, "total": 10, "unresolved": 1, "lower": 0.9, "upper": 1.0 }, "citation_precision": { "passed": 26, "total": 34, "unresolved": 3, "lower": 0.7647058823529411, "upper": 0.8529411764705882 }, "unresolved_questions": 15, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 26, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "v20-q19", "row_indices": [ 54 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.819 ] }, { "question_id": "v20-q04", "row_indices": [ 59 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.329 ] }, { "question_id": "v20-q17", "row_indices": [ 67 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.01 ] }, { "question_id": "v20-q30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.716 ] }, { "question_id": "v20-q21", "row_indices": [ 69 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.453 ] }, { "question_id": "v20-q28", "row_indices": [ 46 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.725 ] }, { "question_id": "v20-q36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.473 ] }, { "question_id": "v20-q03", "row_indices": [ 58 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.838 ] }, { "question_id": "v20-q16", "row_indices": [ 62 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.517 ] }, { "question_id": "v20-q26", "row_indices": [ 51 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.252 ] }, { "question_id": "v20-q12", "row_indices": [ 49 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.528 ] }, { "question_id": "v20-q35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.095 ] }, { "question_id": "v20-q22", "row_indices": [ 52 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.028 ] }, { "question_id": "v20-q15", "row_indices": [ 61 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.713 ] }, { "question_id": "v20-q34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.653 ] }, { "question_id": "v20-q11", "row_indices": [ 63 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.739 ] }, { "question_id": "v20-q05", "row_indices": [ 70 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.469 ] }, { "question_id": "v20-q25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.731 ] }, { "question_id": "v20-q24", "row_indices": [ 50 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.09 ] }, { "question_id": "v20-q29", "row_indices": [ 64 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.689 ] }, { "question_id": "v20-q02", "row_indices": [ 42 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.039 ] }, { "question_id": "v20-q20", "row_indices": [ 45 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.865 ] }, { "question_id": "v20-q33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.314 ] }, { "question_id": "v20-q37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.634 ] }, { "question_id": "v20-q23", "row_indices": [ 40 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.779 ] }, { "question_id": "v20-q32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.377 ] }, { "question_id": "v20-q08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.861 ] }, { "question_id": "v20-q07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.488 ] }, { "question_id": "v20-q13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.561 ] }, { "question_id": "v20-q09", "row_indices": [ 66 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.462 ] }, { "question_id": "v20-q27", "row_indices": [ 47 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.384 ] }, { "question_id": "v20-q38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.124 ] }, { "question_id": "v20-q01", "row_indices": [ 48 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.352 ] }, { "question_id": "v20-q06", "row_indices": [ 56 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.346 ] }, { "question_id": "v20-q39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.722 ] }, { "question_id": "v20-q18", "row_indices": [ 65 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.754 ] }, { "question_id": "v20-q10", "row_indices": [ 71 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.271 ] }, { "question_id": "v20-q31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.735 ] }, { "question_id": "v20-q40", "row_indices": [ 44 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.528 ] }, { "question_id": "v20-q14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.15 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 16, "total": 30, "unresolved": 13, "lower": 0.5333333333333333, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 14, "total": 30, "unresolved": 9, "lower": 0.4666666666666667, "upper": 0.7666666666666667 }, "abstention_accuracy": { "passed": 9, "total": 10, "unresolved": 1, "lower": 0.9, "upper": 1.0 }, "citation_precision": { "passed": 26, "total": 34, "unresolved": 3, "lower": 0.7647058823529411, "upper": 0.8529411764705882 }, "unresolved_questions": 15, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 26, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "v20-q19", "row_indices": [ 108 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.326 ] }, { "question_id": "v20-q04", "row_indices": [ 87 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.151 ] }, { "question_id": "v20-q17", "row_indices": [ 104 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 14.459 ] }, { "question_id": "v20-q30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.48 ] }, { "question_id": "v20-q21", "row_indices": [ 117 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.732 ] }, { "question_id": "v20-q28", "row_indices": [ 86 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.247 ] }, { "question_id": "v20-q36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.535 ] }, { "question_id": "v20-q03", "row_indices": [ 110 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 14.385 ] }, { "question_id": "v20-q16", "row_indices": [ 81 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.242 ] }, { "question_id": "v20-q26", "row_indices": [ 88 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.449 ] }, { "question_id": "v20-q12", "row_indices": [ 113 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.199 ] }, { "question_id": "v20-q35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.293 ] }, { "question_id": "v20-q22", "row_indices": [ 89 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.748 ] }, { "question_id": "v20-q15", "row_indices": [ 114 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.118 ] }, { "question_id": "v20-q34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.095 ] }, { "question_id": "v20-q11", "row_indices": [ 107 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 12.284 ] }, { "question_id": "v20-q05", "row_indices": [ 83 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.014 ] }, { "question_id": "v20-q25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.958 ] }, { "question_id": "v20-q24", "row_indices": [ 96 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.355 ] }, { "question_id": "v20-q29", "row_indices": [ 97 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.22 ] }, { "question_id": "v20-q02", "row_indices": [ 98 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.525 ] }, { "question_id": "v20-q20", "row_indices": [ 95 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.688 ] }, { "question_id": "v20-q33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 15.777 ] }, { "question_id": "v20-q37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.725 ] }, { "question_id": "v20-q23", "row_indices": [ 85 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.463 ] }, { "question_id": "v20-q32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.289 ] }, { "question_id": "v20-q08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.352 ] }, { "question_id": "v20-q07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.481 ] }, { "question_id": "v20-q13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.694 ] }, { "question_id": "v20-q09", "row_indices": [ 102 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.591 ] }, { "question_id": "v20-q27", "row_indices": [ 111 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.001 ] }, { "question_id": "v20-q38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.941 ] }, { "question_id": "v20-q01", "row_indices": [ 105 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.531 ] }, { "question_id": "v20-q06", "row_indices": [ 82 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.24 ] }, { "question_id": "v20-q39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.717 ] }, { "question_id": "v20-q18", "row_indices": [ 80 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.499 ] }, { "question_id": "v20-q10", "row_indices": [ 112 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 14.82 ] }, { "question_id": "v20-q31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.673 ] }, { "question_id": "v20-q40", "row_indices": [ 118 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.225 ] }, { "question_id": "v20-q14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 12.712 ] } ] } ], "qualified": false } }, "paired_effects_A_minus_B": { "complete_question_success": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.125, 0.4 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.55 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "supported_answer_success": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.16666666666666666, 0.5 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.6666666666666666 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "canonical_fact_correctness": { "unique_questions": 30, "effect_bounds_A_minus_B": [ -0.03333333333333333, 0.4666666666666667 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.13333333333333333, 0.6333333333333333 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "correct_abstention": { "unique_questions": 10, "effect_bounds_A_minus_B": [ 0.0, 0.1 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.3 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "emitted_citation_precision": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.11848739495798322, 0.2067226890756303 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.02777777777777779, 0.34375 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." } }, "bootstrap_seed": 271828, "bootstrap_draws": 10000, "comparative_claim_gate": { "eligible_for_review": false, "withhold_reasons": [ "Unreviewed semantic/citation ambiguity remains." ], "automatic_winner": null, "p_values": null, "scope": "Fixed synthetic corpus on the declared single Mac; case-resampling sensitivity does not establish a population or market-wide winner." }, "session_provenance_complete": true, "input_sha256": { "comparison_freeze": "87fcd2d85a9505f48b0e916ccec56b530816ee23d392ab17e5fa1a4731d14ab3", "references": "f3b8625134788d0d6367f97eeb092979ad9e5595921696e532a6806cd44b8e04", "questions": "1fff76507bf75696b93e184d0af92291f7f289bb42880f6c7073d423345faf93", "analysis_source": "a11a2dd93ff574bd55ccc39cc92bbcb15079028e73c169790496cff3b43e4e41" } }