{ "version": "kingy-paired-document-analysis/1.1.0", "grade_version": "kingy-common-document-grade/1.5.0", "planned_unique_questions": 40, "planned_repetitions": [ 1, 2, 3 ], "planned_attempts_per_system": 120, "integrity_issues": [], "complete_planned_evaluation": true, "all_semantic_judgments_resolved": false, "systems": { "A": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 30, "total": 30, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "supported_answer_success": { "passed": 30, "total": 30, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 35, "total": 35, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "unresolved_questions": 0, "explicit_page_citations": 35, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": true }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": true, "abstention_accuracy": true }, "qualified": true, "attempt_audit": [ { "question_id": "V24-Q03", "row_indices": [ 35 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.554732459000661 ] }, { "question_id": "V24-Q37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.947923333995277 ] }, { "question_id": "V24-Q04", "row_indices": [ 16 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.228079292006441 ] }, { "question_id": "V24-Q18", "row_indices": [ 32 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.310176375001902 ] }, { "question_id": "V24-Q01", "row_indices": [ 20 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.25351487500302 ] }, { "question_id": "V24-Q02", "row_indices": [ 17 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.8087857920036186 ] }, { "question_id": "V24-Q16", "row_indices": [ 6 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.029993332995218 ] }, { "question_id": "V24-Q23", "row_indices": [ 31 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.978202500002226 ] }, { "question_id": "V24-Q39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.01393770800496 ] }, { "question_id": "V24-Q13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.093915292003658 ] }, { "question_id": "V24-Q19", "row_indices": [ 37 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.058735583996167 ] }, { "question_id": "V24-Q26", "row_indices": [ 12 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.930935750002391 ] }, { "question_id": "V24-Q31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.113105124997674 ] }, { "question_id": "V24-Q15", "row_indices": [ 34 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.577935250010341 ] }, { "question_id": "V24-Q28", "row_indices": [ 8 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.233194583008299 ] }, { "question_id": "V24-Q20", "row_indices": [ 24 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.713137625003583 ] }, { "question_id": "V24-Q17", "row_indices": [ 30 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.25245904098847 ] }, { "question_id": "V24-Q32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.875173458000063 ] }, { "question_id": "V24-Q24", "row_indices": [ 18 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.159978707990376 ] }, { "question_id": "V24-Q08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.761612791000516 ] }, { "question_id": "V24-Q06", "row_indices": [ 39 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.529093584002112 ] }, { "question_id": "V24-Q10", "row_indices": [ 26 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.0163057500030845 ] }, { "question_id": "V24-Q27", "row_indices": [ 38 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.309338874998502 ] }, { "question_id": "V24-Q09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.480043958988972 ] }, { "question_id": "V24-Q34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.013660625001648 ] }, { "question_id": "V24-Q38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.957433999996283 ] }, { "question_id": "V24-Q25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.978525166006875 ] }, { "question_id": "V24-Q40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.87119225000788 ] }, { "question_id": "V24-Q30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.2153712090075715 ] }, { "question_id": "V24-Q11", "row_indices": [ 29 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.500647374996333 ] }, { "question_id": "V24-Q36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.762351333003608 ] }, { "question_id": "V24-Q12", "row_indices": [ 36 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.303009082999779 ] }, { "question_id": "V24-Q35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.896788042009575 ] }, { "question_id": "V24-Q21", "row_indices": [ 15 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.239173416994163 ] }, { "question_id": "V24-Q14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.725958750001155 ] }, { "question_id": "V24-Q05", "row_indices": [ 2 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.89339574999758 ] }, { "question_id": "V24-Q29", "row_indices": [ 5 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.662651416991139 ] }, { "question_id": "V24-Q22", "row_indices": [ 9 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.987042791006388 ] }, { "question_id": "V24-Q33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.8820614169962937 ] }, { "question_id": "V24-Q07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.158318417001283 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 30, "total": 30, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "supported_answer_success": { "passed": 30, "total": 30, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 35, "total": 35, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "unresolved_questions": 0, "explicit_page_citations": 35, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": true }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": true, "abstention_accuracy": true }, "qualified": true, "attempt_audit": [ { "question_id": "V24-Q03", "row_indices": [ 58 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.384165625000605 ] }, { "question_id": "V24-Q37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.801689958010684 ] }, { "question_id": "V24-Q04", "row_indices": [ 59 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.039978707995033 ] }, { "question_id": "V24-Q18", "row_indices": [ 65 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.103512292000232 ] }, { "question_id": "V24-Q01", "row_indices": [ 48 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.081677875001333 ] }, { "question_id": "V24-Q02", "row_indices": [ 42 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.7275275829888415 ] }, { "question_id": "V24-Q16", "row_indices": [ 62 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.7305579999956535 ] }, { "question_id": "V24-Q23", "row_indices": [ 40 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.659548584008007 ] }, { "question_id": "V24-Q39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.753327291997266 ] }, { "question_id": "V24-Q13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.965303082994069 ] }, { "question_id": "V24-Q19", "row_indices": [ 54 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.906634500002838 ] }, { "question_id": "V24-Q26", "row_indices": [ 51 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.902494958994794 ] }, { "question_id": "V24-Q31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.799608625005931 ] }, { "question_id": "V24-Q15", "row_indices": [ 61 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.245142541985842 ] }, { "question_id": "V24-Q28", "row_indices": [ 46 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.044326916991849 ] }, { "question_id": "V24-Q20", "row_indices": [ 45 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.677840624994133 ] }, { "question_id": "V24-Q17", "row_indices": [ 67 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.583365250000497 ] }, { "question_id": "V24-Q32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.615988457997446 ] }, { "question_id": "V24-Q24", "row_indices": [ 50 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.030156374996295 ] }, { "question_id": "V24-Q08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.608094499999424 ] }, { "question_id": "V24-Q06", "row_indices": [ 56 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.231160666007781 ] }, { "question_id": "V24-Q10", "row_indices": [ 71 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.002155832989956 ] }, { "question_id": "V24-Q27", "row_indices": [ 47 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.110831625002902 ] }, { "question_id": "V24-Q09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.287065957993036 ] }, { "question_id": "V24-Q34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.8609011660009855 ] }, { "question_id": "V24-Q38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.6483028329967055 ] }, { "question_id": "V24-Q25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.771231250007986 ] }, { "question_id": "V24-Q40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.672461832989939 ] }, { "question_id": "V24-Q30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.06862716699834 ] }, { "question_id": "V24-Q11", "row_indices": [ 63 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.309787291000248 ] }, { "question_id": "V24-Q36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.782520374996238 ] }, { "question_id": "V24-Q12", "row_indices": [ 49 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.122475457988912 ] }, { "question_id": "V24-Q35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.716392000002088 ] }, { "question_id": "V24-Q21", "row_indices": [ 69 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.072211999999126 ] }, { "question_id": "V24-Q14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.427066541000386 ] }, { "question_id": "V24-Q05", "row_indices": [ 70 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.339400832992396 ] }, { "question_id": "V24-Q29", "row_indices": [ 64 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.078832457991666 ] }, { "question_id": "V24-Q22", "row_indices": [ 52 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.862373000010848 ] }, { "question_id": "V24-Q33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.824609582996345 ] }, { "question_id": "V24-Q07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.948091915997793 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 30, "total": 30, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "supported_answer_success": { "passed": 30, "total": 30, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 35, "total": 35, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "unresolved_questions": 0, "explicit_page_citations": 35, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": true }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": true, "abstention_accuracy": true }, "qualified": true, "attempt_audit": [ { "question_id": "V24-Q03", "row_indices": [ 110 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.3726980420033215 ] }, { "question_id": "V24-Q37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.799100165997515 ] }, { "question_id": "V24-Q04", "row_indices": [ 87 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.066170292004244 ] }, { "question_id": "V24-Q18", "row_indices": [ 80 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.131860000008601 ] }, { "question_id": "V24-Q01", "row_indices": [ 105 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.099909208001918 ] }, { "question_id": "V24-Q02", "row_indices": [ 98 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.721642750009778 ] }, { "question_id": "V24-Q16", "row_indices": [ 81 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.821485499996925 ] }, { "question_id": "V24-Q23", "row_indices": [ 85 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.594951999999466 ] }, { "question_id": "V24-Q39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.753734124999028 ] }, { "question_id": "V24-Q13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.89293866600201 ] }, { "question_id": "V24-Q19", "row_indices": [ 108 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.917161332996329 ] }, { "question_id": "V24-Q26", "row_indices": [ 88 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.785535250004614 ] }, { "question_id": "V24-Q31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.8745879999914905 ] }, { "question_id": "V24-Q15", "row_indices": [ 114 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.286905166998622 ] }, { "question_id": "V24-Q28", "row_indices": [ 86 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.0735454999958165 ] }, { "question_id": "V24-Q20", "row_indices": [ 95 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.587890499999048 ] }, { "question_id": "V24-Q17", "row_indices": [ 104 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.462256333004916 ] }, { "question_id": "V24-Q32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.682882791006705 ] }, { "question_id": "V24-Q24", "row_indices": [ 96 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.985583166999277 ] }, { "question_id": "V24-Q08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.577455250007915 ] }, { "question_id": "V24-Q06", "row_indices": [ 82 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.28322433300491 ] }, { "question_id": "V24-Q10", "row_indices": [ 112 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.83934083400527 ] }, { "question_id": "V24-Q27", "row_indices": [ 111 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.151846042004763 ] }, { "question_id": "V24-Q09", "row_indices": [ 102 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.312166959003662 ] }, { "question_id": "V24-Q34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.889499791999697 ] }, { "question_id": "V24-Q38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.649864624996553 ] }, { "question_id": "V24-Q25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.783260208001593 ] }, { "question_id": "V24-Q40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.714570374999312 ] }, { "question_id": "V24-Q30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.10741512500681 ] }, { "question_id": "V24-Q11", "row_indices": [ 107 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.300472667004215 ] }, { "question_id": "V24-Q36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.692989125003805 ] }, { "question_id": "V24-Q12", "row_indices": [ 113 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.1484028339909855 ] }, { "question_id": "V24-Q35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.673613750012009 ] }, { "question_id": "V24-Q21", "row_indices": [ 117 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.077488084003562 ] }, { "question_id": "V24-Q14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.4779284999967786 ] }, { "question_id": "V24-Q05", "row_indices": [ 83 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.28744362499856 ] }, { "question_id": "V24-Q29", "row_indices": [ 97 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.24926341700484 ] }, { "question_id": "V24-Q22", "row_indices": [ 89 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.861512708012015 ] }, { "question_id": "V24-Q33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.833602457991219 ] }, { "question_id": "V24-Q07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.952315417001955 ] } ] } ], "qualified": true }, "B": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 29, "total": 30, "unresolved": 1, "lower": 0.9666666666666667, "upper": 1.0 }, "supported_answer_success": { "passed": 27, "total": 30, "unresolved": 1, "lower": 0.9, "upper": 0.9333333333333333 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 31, "total": 35, "unresolved": 2, "lower": 0.8857142857142857, "upper": 0.9428571428571428 }, "unresolved_questions": 3, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 31, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V24-Q03", "row_indices": [ 35 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.236 ] }, { "question_id": "V24-Q37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.993 ] }, { "question_id": "V24-Q04", "row_indices": [ 16 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.993 ] }, { "question_id": "V24-Q18", "row_indices": [ 32 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.031 ] }, { "question_id": "V24-Q01", "row_indices": [ 20 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.076 ] }, { "question_id": "V24-Q02", "row_indices": [ 17 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.264 ] }, { "question_id": "V24-Q16", "row_indices": [ 6 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.054 ] }, { "question_id": "V24-Q23", "row_indices": [ 31 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.054 ] }, { "question_id": "V24-Q39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.638 ] }, { "question_id": "V24-Q13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.477 ] }, { "question_id": "V24-Q19", "row_indices": [ 37 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.62 ] }, { "question_id": "V24-Q26", "row_indices": [ 12 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.936 ] }, { "question_id": "V24-Q31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.407 ] }, { "question_id": "V24-Q15", "row_indices": [ 34 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.267 ] }, { "question_id": "V24-Q28", "row_indices": [ 8 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.772 ] }, { "question_id": "V24-Q20", "row_indices": [ 24 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.468 ] }, { "question_id": "V24-Q17", "row_indices": [ 30 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.52 ] }, { "question_id": "V24-Q32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.938 ] }, { "question_id": "V24-Q24", "row_indices": [ 18 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.137 ] }, { "question_id": "V24-Q08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.431 ] }, { "question_id": "V24-Q06", "row_indices": [ 39 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.472 ] }, { "question_id": "V24-Q10", "row_indices": [ 26 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.379 ] }, { "question_id": "V24-Q27", "row_indices": [ 38 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.852 ] }, { "question_id": "V24-Q09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.721 ] }, { "question_id": "V24-Q34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.129 ] }, { "question_id": "V24-Q38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.116 ] }, { "question_id": "V24-Q25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.728 ] }, { "question_id": "V24-Q40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.403 ] }, { "question_id": "V24-Q30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.618 ] }, { "question_id": "V24-Q11", "row_indices": [ 29 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.678 ] }, { "question_id": "V24-Q36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.427 ] }, { "question_id": "V24-Q12", "row_indices": [ 36 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.788 ] }, { "question_id": "V24-Q35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.028 ] }, { "question_id": "V24-Q21", "row_indices": [ 15 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.812 ] }, { "question_id": "V24-Q14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.542 ] }, { "question_id": "V24-Q05", "row_indices": [ 2 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.699 ] }, { "question_id": "V24-Q29", "row_indices": [ 5 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.397 ] }, { "question_id": "V24-Q22", "row_indices": [ 9 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.343 ] }, { "question_id": "V24-Q33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.607 ] }, { "question_id": "V24-Q07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.509 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 29, "total": 30, "unresolved": 1, "lower": 0.9666666666666667, "upper": 1.0 }, "supported_answer_success": { "passed": 27, "total": 30, "unresolved": 1, "lower": 0.9, "upper": 0.9333333333333333 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 31, "total": 35, "unresolved": 2, "lower": 0.8857142857142857, "upper": 0.9428571428571428 }, "unresolved_questions": 3, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 31, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V24-Q03", "row_indices": [ 58 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.015 ] }, { "question_id": "V24-Q37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.21 ] }, { "question_id": "V24-Q04", "row_indices": [ 59 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.589 ] }, { "question_id": "V24-Q18", "row_indices": [ 65 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.278 ] }, { "question_id": "V24-Q01", "row_indices": [ 48 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.511 ] }, { "question_id": "V24-Q02", "row_indices": [ 42 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.903 ] }, { "question_id": "V24-Q16", "row_indices": [ 62 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.921 ] }, { "question_id": "V24-Q23", "row_indices": [ 40 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.076 ] }, { "question_id": "V24-Q39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.773 ] }, { "question_id": "V24-Q13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.179 ] }, { "question_id": "V24-Q19", "row_indices": [ 54 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.352 ] }, { "question_id": "V24-Q26", "row_indices": [ 51 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.324 ] }, { "question_id": "V24-Q31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.257 ] }, { "question_id": "V24-Q15", "row_indices": [ 61 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.39 ] }, { "question_id": "V24-Q28", "row_indices": [ 46 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.177 ] }, { "question_id": "V24-Q20", "row_indices": [ 45 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.363 ] }, { "question_id": "V24-Q17", "row_indices": [ 67 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.661 ] }, { "question_id": "V24-Q32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.816 ] }, { "question_id": "V24-Q24", "row_indices": [ 50 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.242 ] }, { "question_id": "V24-Q08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.405 ] }, { "question_id": "V24-Q06", "row_indices": [ 56 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.186 ] }, { "question_id": "V24-Q10", "row_indices": [ 71 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.65 ] }, { "question_id": "V24-Q27", "row_indices": [ 47 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.644 ] }, { "question_id": "V24-Q09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.045 ] }, { "question_id": "V24-Q34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.721 ] }, { "question_id": "V24-Q38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.2 ] }, { "question_id": "V24-Q25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.453 ] }, { "question_id": "V24-Q40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.664 ] }, { "question_id": "V24-Q30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.577 ] }, { "question_id": "V24-Q11", "row_indices": [ 63 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.743 ] }, { "question_id": "V24-Q36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.053 ] }, { "question_id": "V24-Q12", "row_indices": [ 49 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.927 ] }, { "question_id": "V24-Q35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.274 ] }, { "question_id": "V24-Q21", "row_indices": [ 69 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.232 ] }, { "question_id": "V24-Q14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.444 ] }, { "question_id": "V24-Q05", "row_indices": [ 70 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.123 ] }, { "question_id": "V24-Q29", "row_indices": [ 64 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.922 ] }, { "question_id": "V24-Q22", "row_indices": [ 52 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.173 ] }, { "question_id": "V24-Q33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.307 ] }, { "question_id": "V24-Q07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.571 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 29, "total": 30, "unresolved": 1, "lower": 0.9666666666666667, "upper": 1.0 }, "supported_answer_success": { "passed": 27, "total": 30, "unresolved": 1, "lower": 0.9, "upper": 0.9333333333333333 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 31, "total": 35, "unresolved": 2, "lower": 0.8857142857142857, "upper": 0.9428571428571428 }, "unresolved_questions": 3, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 31, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": true, "citation_precision": false, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "V24-Q03", "row_indices": [ 110 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.796 ] }, { "question_id": "V24-Q37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.093 ] }, { "question_id": "V24-Q04", "row_indices": [ 87 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.927 ] }, { "question_id": "V24-Q18", "row_indices": [ 80 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.749 ] }, { "question_id": "V24-Q01", "row_indices": [ 105 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.724 ] }, { "question_id": "V24-Q02", "row_indices": [ 98 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.816 ] }, { "question_id": "V24-Q16", "row_indices": [ 81 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.835 ] }, { "question_id": "V24-Q23", "row_indices": [ 85 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.879 ] }, { "question_id": "V24-Q39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.561 ] }, { "question_id": "V24-Q13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.946 ] }, { "question_id": "V24-Q19", "row_indices": [ 108 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.406 ] }, { "question_id": "V24-Q26", "row_indices": [ 88 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.938 ] }, { "question_id": "V24-Q31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.169 ] }, { "question_id": "V24-Q15", "row_indices": [ 114 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.199 ] }, { "question_id": "V24-Q28", "row_indices": [ 86 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.892 ] }, { "question_id": "V24-Q20", "row_indices": [ 95 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.621 ] }, { "question_id": "V24-Q17", "row_indices": [ 104 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.015 ] }, { "question_id": "V24-Q32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.495 ] }, { "question_id": "V24-Q24", "row_indices": [ 96 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.556 ] }, { "question_id": "V24-Q08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.611 ] }, { "question_id": "V24-Q06", "row_indices": [ 82 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.138 ] }, { "question_id": "V24-Q10", "row_indices": [ 112 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.444 ] }, { "question_id": "V24-Q27", "row_indices": [ 111 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.15 ] }, { "question_id": "V24-Q09", "row_indices": [ 102 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.921 ] }, { "question_id": "V24-Q34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.928 ] }, { "question_id": "V24-Q38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.35 ] }, { "question_id": "V24-Q25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.114 ] }, { "question_id": "V24-Q40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.512 ] }, { "question_id": "V24-Q30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.953 ] }, { "question_id": "V24-Q11", "row_indices": [ 107 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.344 ] }, { "question_id": "V24-Q36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.036 ] }, { "question_id": "V24-Q12", "row_indices": [ 113 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.158 ] }, { "question_id": "V24-Q35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.582 ] }, { "question_id": "V24-Q21", "row_indices": [ 117 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.447 ] }, { "question_id": "V24-Q14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.76 ] }, { "question_id": "V24-Q05", "row_indices": [ 83 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.416 ] }, { "question_id": "V24-Q29", "row_indices": [ 97 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.631 ] }, { "question_id": "V24-Q22", "row_indices": [ 89 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.318 ] }, { "question_id": "V24-Q33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.358 ] }, { "question_id": "V24-Q07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.843 ] } ] } ], "qualified": false } }, "paired_effects_A_minus_B": { "complete_question_success": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.05, 0.075 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.175 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "supported_answer_success": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.06666666666666667, 0.1 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.23333333333333334 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "canonical_fact_correctness": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.0, 0.03333333333333333 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.1 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "correct_abstention": { "unique_questions": 10, "effect_bounds_A_minus_B": [ 0.0, 0.0 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.0 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "emitted_citation_precision": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.05714285714285716, 0.11428571428571432 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.0, 0.2727272727272727 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." } }, "bootstrap_seed": 271828, "bootstrap_draws": 10000, "comparative_claim_gate": { "eligible_for_review": false, "withhold_reasons": [ "Unreviewed semantic/citation ambiguity remains." ], "automatic_winner": null, "p_values": null, "scope": "Fixed synthetic corpus on the declared single Mac; case-resampling sensitivity does not establish a population or market-wide winner." }, "measurement_integrity_gate": { "passed": true, "all240OriginalsRetained": true, "sealSha256": "55c6073e5e88de4fefd694d2dc91ea9b2d2e0f6ffd8fad90d654cca6546cbf00" }, "session_provenance_complete": true, "input_sha256": { "comparison_freeze": "f968a685190b843d121167b1031130e79a8daaf5a86da13effef37cb7af45936", "references": "6c5e3d55ba5af2b5ad0b580e1421ffa9bb35a7b44b9a353774497c9ef9757b98", "questions": "b00b40659e59072dd38b6327aa5a1f14da82ba9fc4a3d0f99396a17f0bd274be", "analysis_source": "76d6edb07935cf993c878f2d616a4a1c5f404b1dba81913dd62765b29d1e8ba5" } }