{ "version": "kingy-paired-document-analysis/1.1.0", "grade_version": "kingy-common-document-grade/1.1.0", "planned_unique_questions": 40, "planned_repetitions": [ 1, 2, 3 ], "planned_attempts_per_system": 120, "integrity_issues": [], "complete_planned_evaluation": true, "all_semantic_judgments_resolved": false, "systems": { "A": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 2, "lower": 0.8666666666666667, "upper": 0.9333333333333333 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 1, "lower": 0.8666666666666667, "upper": 0.9 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 37, "total": 38, "unresolved": 0, "lower": 0.9736842105263158, "upper": 0.9736842105263158 }, "unresolved_questions": 2, "explicit_page_citations": 37, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "held-28", "row_indices": [ 8 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.33072191698011 ] }, { "question_id": "held-11", "row_indices": [ 29 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4414830839959905 ] }, { "question_id": "held-27", "row_indices": [ 38 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.096474207995925 ] }, { "question_id": "held-38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6763773749989923 ] }, { "question_id": "held-40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4624059580091853 ] }, { "question_id": "held-13", "row_indices": [ 7 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6032090420194436 ] }, { "question_id": "held-30", "row_indices": [ 27 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.61663237499306 ] }, { "question_id": "held-09", "row_indices": [ 10 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6524891250010114 ] }, { "question_id": "held-36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.623149833001662 ] }, { "question_id": "held-15", "row_indices": [ 34 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4506341250089463 ] }, { "question_id": "held-18", "row_indices": [ 32 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.647518374986248 ] }, { "question_id": "held-32", "row_indices": [ 33 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.518032541993307 ] }, { "question_id": "held-08", "row_indices": [ 28 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6125361250014976 ] }, { "question_id": "held-29", "row_indices": [ 5 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5136542079853825 ] }, { "question_id": "held-19", "row_indices": [ 37 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5521292079938576 ] }, { "question_id": "held-05", "row_indices": [ 2 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6548168750014156 ] }, { "question_id": "held-25", "row_indices": [ 14 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.6697844589943998 ] }, { "question_id": "held-31", "row_indices": [ 25 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.391522249992704 ] }, { "question_id": "held-37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.667346207977971 ] }, { "question_id": "held-23", "row_indices": [ 31 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3172280410071835 ] }, { "question_id": "held-01", "row_indices": [ 20 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5041971660102718 ] }, { "question_id": "held-16", "row_indices": [ 6 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.819263375014998 ] }, { "question_id": "held-39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.520687749987701 ] }, { "question_id": "held-21", "row_indices": [ 15 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9339249999902677 ] }, { "question_id": "held-22", "row_indices": [ 9 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.022966958000325 ] }, { "question_id": "held-24", "row_indices": [ 18 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.285688291012775 ] }, { "question_id": "held-04", "row_indices": [ 16 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6611803749983665 ] }, { "question_id": "held-33", "row_indices": [ 1 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6883222500036936 ] }, { "question_id": "held-03", "row_indices": [ 35 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.477899624995189 ] }, { "question_id": "held-06", "row_indices": [ 39 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5355131249816623 ] }, { "question_id": "held-17", "row_indices": [ 30 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5533066249918193 ] }, { "question_id": "held-02", "row_indices": [ 17 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.710998208989622 ] }, { "question_id": "held-34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.485158291005064 ] }, { "question_id": "held-10", "row_indices": [ 26 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.532282166997902 ] }, { "question_id": "held-20", "row_indices": [ 24 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5627464580175 ] }, { "question_id": "held-12", "row_indices": [ 36 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4965933749917895 ] }, { "question_id": "held-07", "row_indices": [ 0 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4704826249799225 ] }, { "question_id": "held-14", "row_indices": [ 13 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.581105999997817 ] }, { "question_id": "held-35", "row_indices": [ 11 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.602056875010021 ] }, { "question_id": "held-26", "row_indices": [ 12 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.593339208018733 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 2, "lower": 0.8666666666666667, "upper": 0.9333333333333333 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 1, "lower": 0.8666666666666667, "upper": 0.9 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 37, "total": 38, "unresolved": 0, "lower": 0.9736842105263158, "upper": 0.9736842105263158 }, "unresolved_questions": 2, "explicit_page_citations": 37, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "held-28", "row_indices": [ 46 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3924704160017427 ] }, { "question_id": "held-11", "row_indices": [ 63 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4916890419845004 ] }, { "question_id": "held-27", "row_indices": [ 47 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.003020333999302 ] }, { "question_id": "held-38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.615780417021597 ] }, { "question_id": "held-40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.514514833019348 ] }, { "question_id": "held-13", "row_indices": [ 77 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.518320417002542 ] }, { "question_id": "held-30", "row_indices": [ 41 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5620153330091853 ] }, { "question_id": "held-09", "row_indices": [ 66 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.607625584001653 ] }, { "question_id": "held-36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6573432089935523 ] }, { "question_id": "held-15", "row_indices": [ 61 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.528535708988784 ] }, { "question_id": "held-18", "row_indices": [ 65 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6695435420260765 ] }, { "question_id": "held-32", "row_indices": [ 55 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.54769424998085 ] }, { "question_id": "held-08", "row_indices": [ 73 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6197782910021488 ] }, { "question_id": "held-29", "row_indices": [ 64 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4354570420109667 ] }, { "question_id": "held-19", "row_indices": [ 54 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.555502458009869 ] }, { "question_id": "held-05", "row_indices": [ 70 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5880295419774484 ] }, { "question_id": "held-25", "row_indices": [ 74 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.733615832985379 ] }, { "question_id": "held-31", "row_indices": [ 53 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.4394259999971837 ] }, { "question_id": "held-37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7189500000094995 ] }, { "question_id": "held-23", "row_indices": [ 40 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.397369125013938 ] }, { "question_id": "held-01", "row_indices": [ 48 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5350581250095274 ] }, { "question_id": "held-16", "row_indices": [ 62 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.76043724999181 ] }, { "question_id": "held-39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.620824084006017 ] }, { "question_id": "held-21", "row_indices": [ 69 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.991417500015814 ] }, { "question_id": "held-22", "row_indices": [ 52 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9865453749953303 ] }, { "question_id": "held-24", "row_indices": [ 50 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.266905083990423 ] }, { "question_id": "held-04", "row_indices": [ 59 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6835189160192385 ] }, { "question_id": "held-33", "row_indices": [ 79 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6411089999892283 ] }, { "question_id": "held-03", "row_indices": [ 58 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.522450457996456 ] }, { "question_id": "held-06", "row_indices": [ 56 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.560305040999083 ] }, { "question_id": "held-17", "row_indices": [ 67 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.574523874995066 ] }, { "question_id": "held-02", "row_indices": [ 42 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7119190829980653 ] }, { "question_id": "held-34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5142207499884535 ] }, { "question_id": "held-10", "row_indices": [ 71 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5808062499854714 ] }, { "question_id": "held-20", "row_indices": [ 45 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6046232919907197 ] }, { "question_id": "held-12", "row_indices": [ 49 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.526121874980163 ] }, { "question_id": "held-07", "row_indices": [ 43 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.51789537500008 ] }, { "question_id": "held-14", "row_indices": [ 68 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5820702920027543 ] }, { "question_id": "held-35", "row_indices": [ 60 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6627202079980634 ] }, { "question_id": "held-26", "row_indices": [ 51 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4839802920178045 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 2, "lower": 0.8666666666666667, "upper": 0.9333333333333333 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 1, "lower": 0.8666666666666667, "upper": 0.9 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1.0, "upper": 1.0 }, "citation_precision": { "passed": 37, "total": 38, "unresolved": 0, "lower": 0.9736842105263158, "upper": 0.9736842105263158 }, "unresolved_questions": 2, "explicit_page_citations": 37, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "qualified": false, "attempt_audit": [ { "question_id": "held-28", "row_indices": [ 86 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.3901990839804057 ] }, { "question_id": "held-11", "row_indices": [ 107 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.4962750420090742 ] }, { "question_id": "held-27", "row_indices": [ 111 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9880023329751566 ] }, { "question_id": "held-38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6150234999950044 ] }, { "question_id": "held-40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5069954580103513 ] }, { "question_id": "held-13", "row_indices": [ 92 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.523295541002881 ] }, { "question_id": "held-30", "row_indices": [ 109 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.5380138749897014 ] }, { "question_id": "held-09", "row_indices": [ 102 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.600192125013564 ] }, { "question_id": "held-36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.637074540980393 ] }, { "question_id": "held-15", "row_indices": [ 114 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5022756250109524 ] }, { "question_id": "held-18", "row_indices": [ 80 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.679031958017731 ] }, { "question_id": "held-32", "row_indices": [ 91 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.55705879200832 ] }, { "question_id": "held-08", "row_indices": [ 116 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6302080410241615 ] }, { "question_id": "held-29", "row_indices": [ 97 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.4467156669998076 ] }, { "question_id": "held-19", "row_indices": [ 108 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5363362909993157 ] }, { "question_id": "held-05", "row_indices": [ 83 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.581306541018421 ] }, { "question_id": "held-25", "row_indices": [ 106 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.7197210419981275 ] }, { "question_id": "held-31", "row_indices": [ 119 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 1.4295172920101322 ] }, { "question_id": "held-37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7382400420028716 ] }, { "question_id": "held-23", "row_indices": [ 85 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.386177709006006 ] }, { "question_id": "held-01", "row_indices": [ 105 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5316628750006203 ] }, { "question_id": "held-16", "row_indices": [ 81 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.7271590000018477 ] }, { "question_id": "held-39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.615898666001158 ] }, { "question_id": "held-21", "row_indices": [ 117 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.9812785419926513 ] }, { "question_id": "held-22", "row_indices": [ 89 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.958056250005029 ] }, { "question_id": "held-24", "row_indices": [ 96 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.272105583018856 ] }, { "question_id": "held-04", "row_indices": [ 87 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.701218416012125 ] }, { "question_id": "held-33", "row_indices": [ 115 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6266131670272443 ] }, { "question_id": "held-03", "row_indices": [ 110 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.491264874988701 ] }, { "question_id": "held-06", "row_indices": [ 82 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5421561659895815 ] }, { "question_id": "held-17", "row_indices": [ 104 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5725615000119433 ] }, { "question_id": "held-02", "row_indices": [ 98 ], "status": "unresolved", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.673435959004564 ] }, { "question_id": "held-34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5154607919976115 ] }, { "question_id": "held-10", "row_indices": [ 112 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.564919042022666 ] }, { "question_id": "held-20", "row_indices": [ 95 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6073113750026096 ] }, { "question_id": "held-12", "row_indices": [ 113 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5247834579786286 ] }, { "question_id": "held-07", "row_indices": [ 94 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.5172420000017155 ] }, { "question_id": "held-14", "row_indices": [ 99 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.564778334024595 ] }, { "question_id": "held-35", "row_indices": [ 93 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 2.6193650840141345 ] }, { "question_id": "held-26", "row_indices": [ 88 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.459474166011205 ] } ] } ], "qualified": false }, "B": { "retained_input_rows": 120, "invalid_rows": [], "repetitions": [ { "repetition": 1, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 22, "total": 30, "unresolved": 8, "lower": 0.7333333333333333, "upper": 1.0 }, "supported_answer_success": { "passed": 0, "total": 30, "unresolved": 0, "lower": 0.0, "upper": 0.0 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 0, "lower": 0.6, "upper": 0.6 }, "citation_precision": { "passed": 0, "total": 44, "unresolved": 0, "lower": 0.0, "upper": 0.0 }, "unresolved_questions": 8, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "qualified": false, "attempt_audit": [ { "question_id": "held-28", "row_indices": [ 8 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.677 ] }, { "question_id": "held-11", "row_indices": [ 29 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.035 ] }, { "question_id": "held-27", "row_indices": [ 38 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 15.11 ] }, { "question_id": "held-38", "row_indices": [ 19 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.17 ] }, { "question_id": "held-40", "row_indices": [ 21 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.062 ] }, { "question_id": "held-13", "row_indices": [ 7 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.622 ] }, { "question_id": "held-30", "row_indices": [ 27 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.415 ] }, { "question_id": "held-09", "row_indices": [ 10 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.702 ] }, { "question_id": "held-36", "row_indices": [ 4 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.338 ] }, { "question_id": "held-15", "row_indices": [ 34 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.538 ] }, { "question_id": "held-18", "row_indices": [ 32 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.802 ] }, { "question_id": "held-32", "row_indices": [ 33 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.171 ] }, { "question_id": "held-08", "row_indices": [ 28 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.031 ] }, { "question_id": "held-29", "row_indices": [ 5 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.1 ] }, { "question_id": "held-19", "row_indices": [ 37 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.895 ] }, { "question_id": "held-05", "row_indices": [ 2 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 14.505 ] }, { "question_id": "held-25", "row_indices": [ 14 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.774 ] }, { "question_id": "held-31", "row_indices": [ 25 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.789 ] }, { "question_id": "held-37", "row_indices": [ 22 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.286 ] }, { "question_id": "held-23", "row_indices": [ 31 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.182 ] }, { "question_id": "held-01", "row_indices": [ 20 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.415 ] }, { "question_id": "held-16", "row_indices": [ 6 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.689 ] }, { "question_id": "held-39", "row_indices": [ 3 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.177 ] }, { "question_id": "held-21", "row_indices": [ 15 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.806 ] }, { "question_id": "held-22", "row_indices": [ 9 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.86 ] }, { "question_id": "held-24", "row_indices": [ 18 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.232 ] }, { "question_id": "held-04", "row_indices": [ 16 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.108 ] }, { "question_id": "held-33", "row_indices": [ 1 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 12.053 ] }, { "question_id": "held-03", "row_indices": [ 35 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 17.891 ] }, { "question_id": "held-06", "row_indices": [ 39 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 37.187 ] }, { "question_id": "held-17", "row_indices": [ 30 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.726 ] }, { "question_id": "held-02", "row_indices": [ 17 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.984 ] }, { "question_id": "held-34", "row_indices": [ 23 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.534 ] }, { "question_id": "held-10", "row_indices": [ 26 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 10.098 ] }, { "question_id": "held-20", "row_indices": [ 24 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.036 ] }, { "question_id": "held-12", "row_indices": [ 36 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.195 ] }, { "question_id": "held-07", "row_indices": [ 0 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.991 ] }, { "question_id": "held-14", "row_indices": [ 13 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.38 ] }, { "question_id": "held-35", "row_indices": [ 11 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.238 ] }, { "question_id": "held-26", "row_indices": [ 12 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.347 ] } ] }, { "repetition": 2, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 22, "total": 30, "unresolved": 8, "lower": 0.7333333333333333, "upper": 1.0 }, "supported_answer_success": { "passed": 0, "total": 30, "unresolved": 0, "lower": 0.0, "upper": 0.0 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 0, "lower": 0.6, "upper": 0.6 }, "citation_precision": { "passed": 0, "total": 44, "unresolved": 0, "lower": 0.0, "upper": 0.0 }, "unresolved_questions": 8, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "qualified": false, "attempt_audit": [ { "question_id": "held-28", "row_indices": [ 46 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.814 ] }, { "question_id": "held-11", "row_indices": [ 63 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.699 ] }, { "question_id": "held-27", "row_indices": [ 47 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.959 ] }, { "question_id": "held-38", "row_indices": [ 76 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.964 ] }, { "question_id": "held-40", "row_indices": [ 44 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.271 ] }, { "question_id": "held-13", "row_indices": [ 77 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.536 ] }, { "question_id": "held-30", "row_indices": [ 41 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.725 ] }, { "question_id": "held-09", "row_indices": [ 66 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.32 ] }, { "question_id": "held-36", "row_indices": [ 57 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.026 ] }, { "question_id": "held-15", "row_indices": [ 61 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.792 ] }, { "question_id": "held-18", "row_indices": [ 65 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.456 ] }, { "question_id": "held-32", "row_indices": [ 55 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.877 ] }, { "question_id": "held-08", "row_indices": [ 73 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.483 ] }, { "question_id": "held-29", "row_indices": [ 64 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.666 ] }, { "question_id": "held-19", "row_indices": [ 54 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.22 ] }, { "question_id": "held-05", "row_indices": [ 70 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.611 ] }, { "question_id": "held-25", "row_indices": [ 74 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.704 ] }, { "question_id": "held-31", "row_indices": [ 53 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.778 ] }, { "question_id": "held-37", "row_indices": [ 78 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.252 ] }, { "question_id": "held-23", "row_indices": [ 40 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.436 ] }, { "question_id": "held-01", "row_indices": [ 48 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.536 ] }, { "question_id": "held-16", "row_indices": [ 62 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.692 ] }, { "question_id": "held-39", "row_indices": [ 72 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.244 ] }, { "question_id": "held-21", "row_indices": [ 69 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.188 ] }, { "question_id": "held-22", "row_indices": [ 52 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.084 ] }, { "question_id": "held-24", "row_indices": [ 50 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 3.895 ] }, { "question_id": "held-04", "row_indices": [ 59 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.388 ] }, { "question_id": "held-33", "row_indices": [ 79 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.0 ] }, { "question_id": "held-03", "row_indices": [ 58 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.894 ] }, { "question_id": "held-06", "row_indices": [ 56 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.056 ] }, { "question_id": "held-17", "row_indices": [ 67 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.428 ] }, { "question_id": "held-02", "row_indices": [ 42 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.473 ] }, { "question_id": "held-34", "row_indices": [ 75 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.753 ] }, { "question_id": "held-10", "row_indices": [ 71 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.775 ] }, { "question_id": "held-20", "row_indices": [ 45 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.23 ] }, { "question_id": "held-12", "row_indices": [ 49 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.223 ] }, { "question_id": "held-07", "row_indices": [ 43 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.25 ] }, { "question_id": "held-14", "row_indices": [ 68 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.861 ] }, { "question_id": "held-35", "row_indices": [ 60 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.732 ] }, { "question_id": "held-26", "row_indices": [ 51 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.807 ] } ] }, { "repetition": 3, "complete_planned_ids": true, "missing_ids": [], "duplicates": [], "grading_receipt_issues": [], "summary": { "attempts": 40, "answer_correctness": { "passed": 22, "total": 30, "unresolved": 8, "lower": 0.7333333333333333, "upper": 1.0 }, "supported_answer_success": { "passed": 0, "total": 30, "unresolved": 0, "lower": 0.0, "upper": 0.0 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 0, "lower": 0.6, "upper": 0.6 }, "citation_precision": { "passed": 0, "total": 44, "unresolved": 0, "lower": 0.0, "upper": 0.0 }, "unresolved_questions": 8, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false }, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "qualified": false, "attempt_audit": [ { "question_id": "held-28", "row_indices": [ 86 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.31 ] }, { "question_id": "held-11", "row_indices": [ 107 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.69 ] }, { "question_id": "held-27", "row_indices": [ 111 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 9.645 ] }, { "question_id": "held-38", "row_indices": [ 101 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.965 ] }, { "question_id": "held-40", "row_indices": [ 118 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.292 ] }, { "question_id": "held-13", "row_indices": [ 92 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.493 ] }, { "question_id": "held-30", "row_indices": [ 109 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.33 ] }, { "question_id": "held-09", "row_indices": [ 102 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.322 ] }, { "question_id": "held-36", "row_indices": [ 103 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.298 ] }, { "question_id": "held-15", "row_indices": [ 114 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.599 ] }, { "question_id": "held-18", "row_indices": [ 80 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.791 ] }, { "question_id": "held-32", "row_indices": [ 91 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.456 ] }, { "question_id": "held-08", "row_indices": [ 116 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.717 ] }, { "question_id": "held-29", "row_indices": [ 97 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.512 ] }, { "question_id": "held-19", "row_indices": [ 108 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.91 ] }, { "question_id": "held-05", "row_indices": [ 83 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 5.958 ] }, { "question_id": "held-25", "row_indices": [ 106 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.596 ] }, { "question_id": "held-31", "row_indices": [ 119 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.28 ] }, { "question_id": "held-37", "row_indices": [ 84 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.951 ] }, { "question_id": "held-23", "row_indices": [ 85 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.295 ] }, { "question_id": "held-01", "row_indices": [ 105 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.681 ] }, { "question_id": "held-16", "row_indices": [ 81 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.73 ] }, { "question_id": "held-39", "row_indices": [ 100 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.713 ] }, { "question_id": "held-21", "row_indices": [ 117 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.485 ] }, { "question_id": "held-22", "row_indices": [ 89 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.632 ] }, { "question_id": "held-24", "row_indices": [ 96 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 11.747 ] }, { "question_id": "held-04", "row_indices": [ 87 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.62 ] }, { "question_id": "held-33", "row_indices": [ 115 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.118 ] }, { "question_id": "held-03", "row_indices": [ 110 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.307 ] }, { "question_id": "held-06", "row_indices": [ 82 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.05 ] }, { "question_id": "held-17", "row_indices": [ 104 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.683 ] }, { "question_id": "held-02", "row_indices": [ 98 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.061 ] }, { "question_id": "held-34", "row_indices": [ 90 ], "status": "pass", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.381 ] }, { "question_id": "held-10", "row_indices": [ 112 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.421 ] }, { "question_id": "held-20", "row_indices": [ 95 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.238 ] }, { "question_id": "held-12", "row_indices": [ 113 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 6.361 ] }, { "question_id": "held-07", "row_indices": [ 94 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 4.693 ] }, { "question_id": "held-14", "row_indices": [ 99 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 8.629 ] }, { "question_id": "held-35", "row_indices": [ 93 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.294 ] }, { "question_id": "held-26", "row_indices": [ 88 ], "status": "fail", "error_rows": [], "timeout_rows": [], "errors": [], "measured_answer_seconds": [ 7.449 ] } ] } ], "qualified": false } }, "paired_effects_A_minus_B": { "complete_question_success": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.75, 0.775 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.625, 0.9 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "supported_answer_success": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.8666666666666667, 0.9 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.7333333333333333, 1.0 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "canonical_fact_correctness": { "unique_questions": 30, "effect_bounds_A_minus_B": [ -0.13333333333333333, 0.2 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.26666666666666666, 0.36666666666666664 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "correct_abstention": { "unique_questions": 10, "effect_bounds_A_minus_B": [ 0.4, 0.4 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.1, 0.7 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "emitted_citation_precision": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.9736842105263158, 0.9736842105263158 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.9090909090909091, 1.0 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." } }, "bootstrap_seed": 271828, "bootstrap_draws": 10000, "comparative_claim_gate": { "eligible_for_review": false, "withhold_reasons": [ "Unreviewed semantic/citation ambiguity remains." ], "automatic_winner": null, "p_values": null, "scope": "Fixed synthetic corpus on the declared single Mac; case-resampling sensitivity does not establish a population or market-wide winner." }, "session_provenance_complete": true, "input_sha256": { "comparison_freeze": "c4c2e30f72178f693a471db74abfc1806e42dd4fab5f6c68d68f6327c4b30d28", "references": "16409377403959d9eaf49186f8375ac8d80e2e2dbcc8241087bdfa79e7df4365", "questions": "76492fc420df3797365e1cc8f89aaf87a024747ab3846c12ca4ce007bb2029d2", "analysis_source": "5412933b74ee67dd5877a53643548948c585fc15c333c4906e5c8e718e66c1bc" } }