{ "version": "kingy-paired-document-analysis/1.1.0", "grade_version": "kingy-common-document-grade/1.4.0", "planned_unique_questions": 40, "planned_repetitions": [ 1, 2, 3 ], "planned_attempts_per_system": 120, "integrity_issues": [], "complete_planned_evaluation": true, "all_semantic_judgments_resolved": false, "systems": { "A": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 24, "total": 30, "unresolved": 0, "lower": 0.8, "upper": 0.8 }, "supported_answer_success": { "passed": 23, "total": 30, "unresolved": 1, "lower": 0.7666666666666667, "upper": 0.8 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 27, "total": 28, "unresolved": 1, "lower": 0.9642857142857143, "upper": 1.0 }, "unresolved_questions": 1, "explicit_page_citations": 27, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V19-Q32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.456090249994304 ] }, { "question_id": "V19-Q29", "row_indices": [ 5 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.738023083016742 ] }, { "question_id": "V19-Q28", "row_indices": [ 8 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.258850749989506 ] }, { "question_id": "V19-Q27", "row_indices": [ 38 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.476257583999541 ] }, { "question_id": "V19-Q21", "row_indices": [ 15 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.364723666978534 ] }, { "question_id": "V19-Q31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.723389542021323 ] }, { "question_id": "V19-Q03", "row_indices": [ 35 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.6876438750186935 ] }, { "question_id": "V19-Q06", "row_indices": [ 39 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.9762109169969335 ] }, { "question_id": "V19-Q10", "row_indices": [ 26 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.206018540950026 ] }, { "question_id": "V19-Q23", "row_indices": [ 31 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.171583165996708 ] }, { "question_id": "V19-Q33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.622545958030969 ] }, { "question_id": "V19-Q24", "row_indices": [ 18 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.260318582993932 ] }, { "question_id": "V19-Q36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.37097770901164 ] }, { "question_id": "V19-Q02", "row_indices": [ 17 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.888800000015181 ] }, { "question_id": "V19-Q17", "row_indices": [ 30 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.1107155829668045 ] }, { "question_id": "V19-Q20", "row_indices": [ 24 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.137335666979197 ] }, { "question_id": "V19-Q07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.00618912500795 ] }, { "question_id": "V19-Q30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.89305316703394 ] }, { "question_id": "V19-Q15", "row_indices": [ 34 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.866158791992348 ] }, { "question_id": "V19-Q13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.119112083979417 ] }, { "question_id": "V19-Q14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.413528875040356 ] }, { "question_id": "V19-Q19", "row_indices": [ 37 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.185925875033718 ] }, { "question_id": "V19-Q11", "row_indices": [ 29 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.107318166003097 ] }, { "question_id": "V19-Q04", "row_indices": [ 16 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.806305583973881 ] }, { "question_id": "V19-Q35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.101432333001867 ] }, { "question_id": "V19-Q08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.748720749979839 ] }, { "question_id": "V19-Q38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.720423375023529 ] }, { "question_id": "V19-Q40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.007767457980663 ] }, { "question_id": "V19-Q18", "row_indices": [ 32 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.892946582986042 ] }, { "question_id": "V19-Q34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.29394812497776 ] }, { "question_id": "V19-Q25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.354678458999842 ] }, { "question_id": "V19-Q12", "row_indices": [ 36 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.790998582961038 ] }, { "question_id": "V19-Q22", "row_indices": [ 9 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.6757438750355504 ] }, { "question_id": "V19-Q16", "row_indices": [ 6 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.642559625033755 ] }, { "question_id": "V19-Q01", "row_indices": [ 20 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.520986375049688 ] }, { "question_id": "V19-Q37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.2095282090012915 ] }, { "question_id": "V19-Q26", "row_indices": [ 12 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.807647916022688 ] }, { "question_id": "V19-Q05", "row_indices": [ 2 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.808426708972547 ] }, { "question_id": "V19-Q39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.394660874968395 ] }, { "question_id": "V19-Q09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.427064749994315 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 24, "total": 30, "unresolved": 0, "lower": 0.8, "upper": 0.8 }, "supported_answer_success": { "passed": 23, "total": 30, "unresolved": 1, "lower": 0.7666666666666667, "upper": 0.8 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 27, "total": 28, "unresolved": 1, "lower": 0.9642857142857143, "upper": 1.0 }, "unresolved_questions": 1, "explicit_page_citations": 27, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V19-Q32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9132139579996874 ] }, { "question_id": "V19-Q29", "row_indices": [ 64 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.541269207998994 ] }, { "question_id": "V19-Q28", "row_indices": [ 46 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.921231042000727 ] }, { "question_id": "V19-Q27", "row_indices": [ 47 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8391945420007687 ] }, { "question_id": "V19-Q21", "row_indices": [ 69 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.650336749997223 ] }, { "question_id": "V19-Q31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7399721250003495 ] }, { "question_id": "V19-Q03", "row_indices": [ 58 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.599350708002021 ] }, { "question_id": "V19-Q06", "row_indices": [ 56 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.510950333999062 ] }, { "question_id": "V19-Q10", "row_indices": [ 71 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5453252920015075 ] }, { "question_id": "V19-Q23", "row_indices": [ 40 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6436558329987747 ] }, { "question_id": "V19-Q33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.568572125001083 ] }, { "question_id": "V19-Q24", "row_indices": [ 50 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4852482079986657 ] }, { "question_id": "V19-Q36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.05175970900018 ] }, { "question_id": "V19-Q02", "row_indices": [ 42 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4126532080008474 ] }, { "question_id": "V19-Q17", "row_indices": [ 67 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6965952499995183 ] }, { "question_id": "V19-Q20", "row_indices": [ 45 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.523142875001213 ] }, { "question_id": "V19-Q07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7029641250010172 ] }, { "question_id": "V19-Q30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.9434071670002595 ] }, { "question_id": "V19-Q15", "row_indices": [ 61 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.606827042000077 ] }, { "question_id": "V19-Q13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.772564582999621 ] }, { "question_id": "V19-Q14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6157479159992363 ] }, { "question_id": "V19-Q19", "row_indices": [ 54 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5377855410006305 ] }, { "question_id": "V19-Q11", "row_indices": [ 63 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6590270830020017 ] }, { "question_id": "V19-Q04", "row_indices": [ 59 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.118073958001332 ] }, { "question_id": "V19-Q35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9493862920026004 ] }, { "question_id": "V19-Q08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.016847165999934 ] }, { "question_id": "V19-Q38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3083130000013625 ] }, { "question_id": "V19-Q40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3136277919984423 ] }, { "question_id": "V19-Q18", "row_indices": [ 65 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.748537249997753 ] }, { "question_id": "V19-Q34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4532378330004576 ] }, { "question_id": "V19-Q25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.787626666999131 ] }, { "question_id": "V19-Q12", "row_indices": [ 49 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.20913279099841 ] }, { "question_id": "V19-Q22", "row_indices": [ 52 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5743448749999516 ] }, { "question_id": "V19-Q16", "row_indices": [ 62 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.55525787499937 ] }, { "question_id": "V19-Q01", "row_indices": [ 48 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5457530000021507 ] }, { "question_id": "V19-Q37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.490040583001246 ] }, { "question_id": "V19-Q26", "row_indices": [ 51 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.422845583001617 ] }, { "question_id": "V19-Q05", "row_indices": [ 70 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7261680830015393 ] }, { "question_id": "V19-Q39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.358341416998883 ] }, { "question_id": "V19-Q09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.348765208000259 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 24, "total": 30, "unresolved": 0, "lower": 0.8, "upper": 0.8 }, "supported_answer_success": { "passed": 23, "total": 30, "unresolved": 1, "lower": 0.7666666666666667, "upper": 0.8 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 27, "total": 28, "unresolved": 1, "lower": 0.9642857142857143, "upper": 1.0 }, "unresolved_questions": 1, "explicit_page_citations": 27, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V19-Q32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.666249166999478 ] }, { "question_id": "V19-Q29", "row_indices": [ 97 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.471811084000365 ] }, { "question_id": "V19-Q28", "row_indices": [ 86 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.384190374999889 ] }, { "question_id": "V19-Q27", "row_indices": [ 111 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.731723459000932 ] }, { "question_id": "V19-Q21", "row_indices": [ 117 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8350643329977174 ] }, { "question_id": "V19-Q31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6722466249993886 ] }, { "question_id": "V19-Q03", "row_indices": [ 110 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6099177910000435 ] }, { "question_id": "V19-Q06", "row_indices": [ 82 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5900335000005725 ] }, { "question_id": "V19-Q10", "row_indices": [ 112 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5283914169995114 ] }, { "question_id": "V19-Q23", "row_indices": [ 85 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.80678641600025 ] }, { "question_id": "V19-Q33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8752699169999687 ] }, { "question_id": "V19-Q24", "row_indices": [ 96 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4930500829977973 ] }, { "question_id": "V19-Q36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3921305000003485 ] }, { "question_id": "V19-Q02", "row_indices": [ 98 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5018811249974533 ] }, { "question_id": "V19-Q17", "row_indices": [ 104 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.698861125001713 ] }, { "question_id": "V19-Q20", "row_indices": [ 95 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.698258041000372 ] }, { "question_id": "V19-Q07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.8290570829994977 ] }, { "question_id": "V19-Q30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.024350249997951 ] }, { "question_id": "V19-Q15", "row_indices": [ 114 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.110735624999506 ] }, { "question_id": "V19-Q13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7619894160015974 ] }, { "question_id": "V19-Q14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6049985839999863 ] }, { "question_id": "V19-Q19", "row_indices": [ 108 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4244550000003073 ] }, { "question_id": "V19-Q11", "row_indices": [ 107 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6417158340009337 ] }, { "question_id": "V19-Q04", "row_indices": [ 87 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.932093250001344 ] }, { "question_id": "V19-Q35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.689147166998737 ] }, { "question_id": "V19-Q08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.375949042001594 ] }, { "question_id": "V19-Q38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.2973984999989625 ] }, { "question_id": "V19-Q40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.610177875001682 ] }, { "question_id": "V19-Q18", "row_indices": [ 80 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.826049792001868 ] }, { "question_id": "V19-Q34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4718483750002633 ] }, { "question_id": "V19-Q25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.790317709001101 ] }, { "question_id": "V19-Q12", "row_indices": [ 113 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5116446250031004 ] }, { "question_id": "V19-Q22", "row_indices": [ 89 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5413505420001457 ] }, { "question_id": "V19-Q16", "row_indices": [ 81 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6255454999991343 ] }, { "question_id": "V19-Q01", "row_indices": [ 105 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5796387910013436 ] }, { "question_id": "V19-Q37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.661505333002424 ] }, { "question_id": "V19-Q26", "row_indices": [ 88 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.6823848339990946 ] }, { "question_id": "V19-Q05", "row_indices": [ 83 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.894903708998754 ] }, { "question_id": "V19-Q39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.397810834001575 ] }, { "question_id": "V19-Q09", "row_indices": [ 102 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3669365829991875 ] } ] } ], "qualified": false }, "B": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 7, "total": 30, "unresolved": 18, "lower": 0.23333333333333334, "upper": 0.8333333333333334 }, "supported_answer_success": { "passed": 7, "total": 30, "unresolved": 15, "lower": 0.23333333333333334, "upper": 0.7333333333333333 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 31, "total": 35, "unresolved": 0, "lower": 0.8857142857142857, "upper": 0.8857142857142857 }, "unresolved_questions": 18, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 31, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V19-Q32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 20.873 ] }, { "question_id": "V19-Q29", "row_indices": [ 5 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.543 ] }, { "question_id": "V19-Q28", "row_indices": [ 8 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.253 ] }, { "question_id": "V19-Q27", "row_indices": [ 38 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.948 ] }, { "question_id": "V19-Q21", "row_indices": [ 15 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.054 ] }, { "question_id": "V19-Q31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.534 ] }, { "question_id": "V19-Q03", "row_indices": [ 35 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.647 ] }, { "question_id": "V19-Q06", "row_indices": [ 39 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.26 ] }, { "question_id": "V19-Q10", "row_indices": [ 26 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.702 ] }, { "question_id": "V19-Q23", "row_indices": [ 31 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 14.849 ] }, { "question_id": "V19-Q33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.252 ] }, { "question_id": "V19-Q24", "row_indices": [ 18 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 15.616 ] }, { "question_id": "V19-Q36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.841 ] }, { "question_id": "V19-Q02", "row_indices": [ 17 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.74 ] }, { "question_id": "V19-Q17", "row_indices": [ 30 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 16.363 ] }, { "question_id": "V19-Q20", "row_indices": [ 24 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 20.413 ] }, { "question_id": "V19-Q07", "row_indices": [ 0 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.212 ] }, { "question_id": "V19-Q30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 14.0 ] }, { "question_id": "V19-Q15", "row_indices": [ 34 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.869 ] }, { "question_id": "V19-Q13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 15.176 ] }, { "question_id": "V19-Q14", "row_indices": [ 13 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 21.345 ] }, { "question_id": "V19-Q19", "row_indices": [ 37 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.507 ] }, { "question_id": "V19-Q11", "row_indices": [ 29 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.517 ] }, { "question_id": "V19-Q04", "row_indices": [ 16 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.146 ] }, { "question_id": "V19-Q35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.936 ] }, { "question_id": "V19-Q08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 15.917 ] }, { "question_id": "V19-Q38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 15.463 ] }, { "question_id": "V19-Q40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.258 ] }, { "question_id": "V19-Q18", "row_indices": [ 32 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.28 ] }, { "question_id": "V19-Q34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.18 ] }, { "question_id": "V19-Q25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 19.752 ] }, { "question_id": "V19-Q12", "row_indices": [ 36 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.432 ] }, { "question_id": "V19-Q22", "row_indices": [ 9 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 17.221 ] }, { "question_id": "V19-Q16", "row_indices": [ 6 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.477 ] }, { "question_id": "V19-Q01", "row_indices": [ 20 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 18.86 ] }, { "question_id": "V19-Q37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 12.55 ] }, { "question_id": "V19-Q26", "row_indices": [ 12 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 40.455 ] }, { "question_id": "V19-Q05", "row_indices": [ 2 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.045 ] }, { "question_id": "V19-Q39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.569 ] }, { "question_id": "V19-Q09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 14.439 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 7, "total": 30, "unresolved": 18, "lower": 0.23333333333333334, "upper": 0.8333333333333334 }, "supported_answer_success": { "passed": 7, "total": 30, "unresolved": 15, "lower": 0.23333333333333334, "upper": 0.7333333333333333 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 31, "total": 35, "unresolved": 0, "lower": 0.8857142857142857, "upper": 0.8857142857142857 }, "unresolved_questions": 18, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 31, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V19-Q32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.706 ] }, { "question_id": "V19-Q29", "row_indices": [ 64 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.632 ] }, { "question_id": "V19-Q28", "row_indices": [ 46 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.698 ] }, { "question_id": "V19-Q27", "row_indices": [ 47 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.259 ] }, { "question_id": "V19-Q21", "row_indices": [ 69 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.338 ] }, { "question_id": "V19-Q31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.434 ] }, { "question_id": "V19-Q03", "row_indices": [ 58 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.289 ] }, { "question_id": "V19-Q06", "row_indices": [ 56 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.296 ] }, { "question_id": "V19-Q10", "row_indices": [ 71 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.288 ] }, { "question_id": "V19-Q23", "row_indices": [ 40 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.297 ] }, { "question_id": "V19-Q33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.102 ] }, { "question_id": "V19-Q24", "row_indices": [ 50 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.44 ] }, { "question_id": "V19-Q36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.407 ] }, { "question_id": "V19-Q02", "row_indices": [ 42 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.17 ] }, { "question_id": "V19-Q17", "row_indices": [ 67 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.916 ] }, { "question_id": "V19-Q20", "row_indices": [ 45 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.503 ] }, { "question_id": "V19-Q07", "row_indices": [ 43 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.182 ] }, { "question_id": "V19-Q30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.526 ] }, { "question_id": "V19-Q15", "row_indices": [ 61 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.72 ] }, { "question_id": "V19-Q13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.657 ] }, { "question_id": "V19-Q14", "row_indices": [ 68 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.352 ] }, { "question_id": "V19-Q19", "row_indices": [ 54 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.522 ] }, { "question_id": "V19-Q11", "row_indices": [ 63 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.227 ] }, { "question_id": "V19-Q04", "row_indices": [ 59 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.163 ] }, { "question_id": "V19-Q35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.517 ] }, { "question_id": "V19-Q08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.855 ] }, { "question_id": "V19-Q38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.468 ] }, { "question_id": "V19-Q40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.068 ] }, { "question_id": "V19-Q18", "row_indices": [ 65 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.785 ] }, { "question_id": "V19-Q34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.842 ] }, { "question_id": "V19-Q25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.476 ] }, { "question_id": "V19-Q12", "row_indices": [ 49 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.36 ] }, { "question_id": "V19-Q22", "row_indices": [ 52 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.707 ] }, { "question_id": "V19-Q16", "row_indices": [ 62 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.271 ] }, { "question_id": "V19-Q01", "row_indices": [ 48 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.446 ] }, { "question_id": "V19-Q37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.869 ] }, { "question_id": "V19-Q26", "row_indices": [ 51 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.834 ] }, { "question_id": "V19-Q05", "row_indices": [ 70 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.63 ] }, { "question_id": "V19-Q39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.15 ] }, { "question_id": "V19-Q09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.46 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 6, "total": 30, "unresolved": 18, "lower": 0.2, "upper": 0.8 }, "supported_answer_success": { "passed": 6, "total": 30, "unresolved": 15, "lower": 0.2, "upper": 0.7 }, "abstention_accuracy": { "passed": 9, "total": 10, "unresolved": 0, "lower": 0.9, "upper": 0.9 }, "citation_precision": { "passed": 30, "total": 34, "unresolved": 0, "lower": 0.8823529411764706, "upper": 0.8823529411764706 }, "unresolved_questions": 18, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 30, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V19-Q32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.454 ] }, { "question_id": "V19-Q29", "row_indices": [ 97 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.654 ] }, { "question_id": "V19-Q28", "row_indices": [ 86 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 13.544 ] }, { "question_id": "V19-Q27", "row_indices": [ 111 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.579 ] }, { "question_id": "V19-Q21", "row_indices": [ 117 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.24 ] }, { "question_id": "V19-Q31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.476 ] }, { "question_id": "V19-Q03", "row_indices": [ 110 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.342 ] }, { "question_id": "V19-Q06", "row_indices": [ 82 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 15.151 ] }, { "question_id": "V19-Q10", "row_indices": [ 112 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.782 ] }, { "question_id": "V19-Q23", "row_indices": [ 85 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 12.995 ] }, { "question_id": "V19-Q33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.485 ] }, { "question_id": "V19-Q24", "row_indices": [ 96 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.14 ] }, { "question_id": "V19-Q36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.923 ] }, { "question_id": "V19-Q02", "row_indices": [ 98 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 18.601 ] }, { "question_id": "V19-Q17", "row_indices": [ 104 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.042 ] }, { "question_id": "V19-Q20", "row_indices": [ 95 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.506 ] }, { "question_id": "V19-Q07", "row_indices": [ 94 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.6 ] }, { "question_id": "V19-Q30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.463 ] }, { "question_id": "V19-Q15", "row_indices": [ 114 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.635 ] }, { "question_id": "V19-Q13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.082 ] }, { "question_id": "V19-Q14", "row_indices": [ 99 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.931 ] }, { "question_id": "V19-Q19", "row_indices": [ 108 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.576 ] }, { "question_id": "V19-Q11", "row_indices": [ 107 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.491 ] }, { "question_id": "V19-Q04", "row_indices": [ 87 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.763 ] }, { "question_id": "V19-Q35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.24 ] }, { "question_id": "V19-Q08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.367 ] }, { "question_id": "V19-Q38", "row_indices": [ 101 ], "status": "fail", "error_rows": [ 101 ], "timeout_rows": [ 101 ], "errors": [ "Complete native answer exceeded frozen 90-second deadline." ], "measured_answer_seconds": [ 92.536 ] }, { "question_id": "V19-Q40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.447 ] }, { "question_id": "V19-Q18", "row_indices": [ 80 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.765 ] }, { "question_id": "V19-Q34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.735 ] }, { "question_id": "V19-Q25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.206 ] }, { "question_id": "V19-Q12", "row_indices": [ 113 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.939 ] }, { "question_id": "V19-Q22", "row_indices": [ 89 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.114 ] }, { "question_id": "V19-Q16", "row_indices": [ 81 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 17.89 ] }, { "question_id": "V19-Q01", "row_indices": [ 105 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.811 ] }, { "question_id": "V19-Q37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.828 ] }, { "question_id": "V19-Q26", "row_indices": [ 88 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.828 ] }, { "question_id": "V19-Q05", "row_indices": [ 83 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.31 ] }, { "question_id": "V19-Q39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.225 ] }, { "question_id": "V19-Q09", "row_indices": [ 102 ], "status": "fail", "error_rows": [ 102 ], "timeout_rows": [], "errors": [ "Expected one original native generation." ], "measured_answer_seconds": [] } ] } ], "qualified": false } }, "paired_effects_A_minus_B": { "complete_question_success": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.04166666666666667, 0.4416666666666667 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.10833333333333332, 0.6083333333333333 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "supported_answer_success": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.044444444444444446, 0.5777777777777777 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.15555555555555556, 0.7666666666666667 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "canonical_fact_correctness": { "unique_questions": 30, "effect_bounds_A_minus_B": [ -0.02222222222222222, 0.5777777777777777 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.2, 0.7666666666666667 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "correct_abstention": { "unique_questions": 10, "effect_bounds_A_minus_B": [ 0.03333333333333334, 0.03333333333333334 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.1 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "emitted_citation_precision": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.07967032967032972, 0.11538461538461542 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.048720553089791674, 0.23764907740774038 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." } }, "bootstrap_seed": 271828, "bootstrap_draws": 10000, "comparative_claim_gate": { "eligible_for_review": false, "withhold_reasons": [ "Unreviewed semantic/citation ambiguity remains." ], "automatic_winner": null, "p_values": null, "scope": "Fixed synthetic corpus on the declared single Mac; case-resampling sensitivity does not establish a population or market-wide winner." }, "session_provenance_complete": true, "input_sha256": { "comparison_freeze": "d1abac923f15a5b6c4a10e51b3f17938789cf99747a218e91ca5d622d22a8f6f", "references": "31df50ca439b27fbf546a2fb12bb0113278cb50fb0d1cba0470b734a54f24158", "questions": "677e88becba7b22d057e5f172421686c61cecf79d170eb0b56d1c9232d51be90", "analysis_source": "5412933b74ee67dd5877a53643548948c585fc15c333c4906e5c8e718e66c1bc" } }