{ "development_only": true, "qualificationGranted": false, "unseenComparisonBankInspected": false, "frozenPublishedV13Changed": false, "initial_runner_sha256": "b24613e249e685e2c2cc6cae9f315148d97c6bfef2c013daa19877128fd166c9", "final_runner_sha256": "00d0f68dfa240c7ce937be94ade50172c5051cb243ad50b2740e1b292ea5f65c", "offline_profile_sha256": "e5462ef1daae3785012cb8e4fa05b89105bf4ac4d98be9580a8c357365a4f54f", "grader_and_inference_guard_functions_unchanged": true, "unchanged_function_ast_sha256": { "grade": "26c617eb53af3aa94398025211ac6ae868fcbc16127e7a3713666079c4a8ac8b", "summary": "8cc75a6b7039f06734e2866cab117ca673277d21980b31b92733d43e47f0bcd4", "value_matches": "254cfee1b9a69bfa94f4b6c601c987d58741920df36a596e66011978177337d2", "missing_information": "0cb9bae701d15afff51be6696afd7ee85ed296bf07d7e8a783f486696e6935a9", "normalize_absence": "ef6912c7106638b8fc536da6a7739fd01f48a79e4a00c42d8c9cbb78395406ec", "topic_absence": "bf5d4c792cc5da2564e0a1314c43f7045b04ea8b8726d739fd04e5a87c53351f", "documentary_absence_clauses": "3553d9d4802de1ee71f5068160f58389dbdce39703bb86e787b9027d82b66503", "ask": "05eae8d73f16afa2226ac86b0eff6c2ac7bdb0e5b0fa4a9bd13fcbdfc9ddb9e2" }, "changes": [ "Question clauses outrank internal field labels in passage retrieval.", "Lexical inflection families cover measurement and finance words without answer facts.", "Document identity remains scoped across report/review genre naming.", "Distinctive document-name tokens prioritize matching source pages.", "Explicit qualifier-bearing passages are preferred when present, allowing approved/signed status synonyms.", "Qualifiers stay with the corresponding subquestion rather than spilling into another field." ], "pure_tests": { "exit_code": 0, "log_sha256": "c6b9b1f09add0403cc07e90bfdf272129b1ddfa2f8f7675ea8218862f2516371" }, "retained_files": [ { "path": "kingy-repair-1.jsonl", "sha256": "2914f1ae6b9fc20ead713bc66a3d24acf43112a3f483a9dbe5779241f3384abd", "attempts": 6, "passes": 0, "failures": 6, "model_request_count": 12, "answer_seconds": { "count": 6, "min": 2.9077799579827115, "median": 3.7115721455047606, "max": 4.49531699999352 }, "summary": { "runner_version": "kingy-document-runner/1.4.0", "questions_sha256": "1abe5e67e1242a335e0493e5c72c8ba4518d22cb09e29c492f72f890858c1e12", "references_sha256": "c73ec4c818e4f6aa154a4febf610e2c1fdee1d0edff286013958fd45f1fdd12a", "runner_sha256": "b24613e249e685e2c2cc6cae9f315148d97c6bfef2c013daa19877128fd166c9", "source_changed_during_run": false, "system_prompt_sha256": "cb330d9b009483d2ed91469195ceff86b231fba66feee14ffbdf344153e1ec3a", "parse_seconds": 0.039185707981232554, "index_seconds": 0.001612167019629851, "split": "development", "repetitions": [ { "repetition": 1, "attempts": 2, "answerable_count": 2, "answerable_passes": 0, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 2, "valid_citation_count": 1, "answer_correctness": 0.0, "citation_precision": 0.5, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 0 }, "direct": { "attempts": 1, "passed": 0 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 }, { "repetition": 2, "attempts": 2, "answerable_count": 2, "answerable_passes": 0, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 2, "valid_citation_count": 1, "answer_correctness": 0.0, "citation_precision": 0.5, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 0 }, "direct": { "attempts": 1, "passed": 0 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 }, { "repetition": 3, "attempts": 2, "answerable_count": 2, "answerable_passes": 0, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 2, "valid_citation_count": 1, "answer_correctness": 0.0, "citation_precision": 0.5, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 0 }, "direct": { "attempts": 1, "passed": 0 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 } ], "complete_planned_evaluation": false, "qualified": false } }, { "path": "kingy-repair-2.jsonl", "sha256": "d81a67bf20c11681f4ac524fd8c77357a2b648a961cb11ba31649f236a9268a1", "attempts": 6, "passes": 6, "failures": 0, "model_request_count": 12, "answer_seconds": { "count": 6, "min": 2.6959760840109084, "median": 4.098264083499089, "max": 4.593273417034652 }, "summary": { "runner_version": "kingy-document-runner/1.4.0", "questions_sha256": "1abe5e67e1242a335e0493e5c72c8ba4518d22cb09e29c492f72f890858c1e12", "references_sha256": "c73ec4c818e4f6aa154a4febf610e2c1fdee1d0edff286013958fd45f1fdd12a", "runner_sha256": "c8a7930c185b16567ceefec1049c912b0d2c8c2954d798ad22724e15296726e1", "source_changed_during_run": false, "system_prompt_sha256": "cb330d9b009483d2ed91469195ceff86b231fba66feee14ffbdf344153e1ec3a", "parse_seconds": 0.04120470900670625, "index_seconds": 0.001713167002890259, "split": "development", "repetitions": [ { "repetition": 1, "attempts": 2, "answerable_count": 2, "answerable_passes": 2, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 3, "valid_citation_count": 3, "answer_correctness": 1.0, "citation_precision": 1.0, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 1 }, "direct": { "attempts": 1, "passed": 1 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 }, { "repetition": 2, "attempts": 2, "answerable_count": 2, "answerable_passes": 2, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 3, "valid_citation_count": 3, "answer_correctness": 1.0, "citation_precision": 1.0, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 1 }, "direct": { "attempts": 1, "passed": 1 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 }, { "repetition": 3, "attempts": 2, "answerable_count": 2, "answerable_passes": 2, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 3, "valid_citation_count": 3, "answer_correctness": 1.0, "citation_precision": 1.0, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 1 }, "direct": { "attempts": 1, "passed": 1 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 } ], "complete_planned_evaluation": false, "qualified": false } }, { "path": "kingy-repair-3.jsonl", "sha256": "0a4f915bbcb9c9377525ebffdcf039588024baf4fa5550982fe311666f04d349", "attempts": 6, "passes": 6, "failures": 0, "model_request_count": 12, "answer_seconds": { "count": 6, "min": 2.9620087079820223, "median": 3.7256092919851653, "max": 5.28013720799936 }, "summary": { "runner_version": "kingy-document-runner/1.4.0", "questions_sha256": "1abe5e67e1242a335e0493e5c72c8ba4518d22cb09e29c492f72f890858c1e12", "references_sha256": "c73ec4c818e4f6aa154a4febf610e2c1fdee1d0edff286013958fd45f1fdd12a", "runner_sha256": "00d0f68dfa240c7ce937be94ade50172c5051cb243ad50b2740e1b292ea5f65c", "source_changed_during_run": false, "system_prompt_sha256": "cb330d9b009483d2ed91469195ceff86b231fba66feee14ffbdf344153e1ec3a", "parse_seconds": 0.043701082991901785, "index_seconds": 0.0017875409976113588, "split": "development", "repetitions": [ { "repetition": 1, "attempts": 2, "answerable_count": 2, "answerable_passes": 2, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 3, "valid_citation_count": 3, "answer_correctness": 1.0, "citation_precision": 1.0, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 1 }, "direct": { "attempts": 1, "passed": 1 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 }, { "repetition": 2, "attempts": 2, "answerable_count": 2, "answerable_passes": 2, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 3, "valid_citation_count": 3, "answer_correctness": 1.0, "citation_precision": 1.0, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 1 }, "direct": { "attempts": 1, "passed": 1 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 }, { "repetition": 3, "attempts": 2, "answerable_count": 2, "answerable_passes": 2, "unanswerable_count": 0, "correct_abstentions": 0, "citation_count": 3, "valid_citation_count": 3, "answer_correctness": 1.0, "citation_precision": 1.0, "abstention_accuracy": null, "qualified": false, "by_category": { "cross_document": { "attempts": 1, "passed": 1 }, "direct": { "attempts": 1, "passed": 1 } }, "model_calls": 2, "model_request_count": 4, "guard_only_answers": 0 } ], "complete_planned_evaluation": false, "qualified": false } }, { "path": "regression-17-2.jsonl", "sha256": "a60a9bcfd4af51d8ac11efdebb15da362f077a52e993da9a6e87592064f0c222", "attempts": 17, "passes": 17, "failures": 0, "model_request_count": 34, "answer_seconds": { "count": 17, "min": 2.418795457982924, "median": 2.9926566669892054, "max": 4.760433417017339 }, "summary": { "runner_version": "kingy-document-runner/1.4.0", "questions_sha256": "a69c2f7f8ba63c939d5b226a6a392975af67f5f11d433eecf571ca64a230b469", "references_sha256": "590bbcfb17ec5d3f61323fc40a2a6439d266e62e69a88a489bfc6f008791dccb", "runner_sha256": "00d0f68dfa240c7ce937be94ade50172c5051cb243ad50b2740e1b292ea5f65c", "source_changed_during_run": false, "system_prompt_sha256": "cb330d9b009483d2ed91469195ceff86b231fba66feee14ffbdf344153e1ec3a", "parse_seconds": 0.03935916698537767, "index_seconds": 0.0015818749961908907, "split": "development", "repetitions": [ { "repetition": 1, "attempts": 17, "answerable_count": 13, "answerable_passes": 13, "unanswerable_count": 4, "correct_abstentions": 4, "citation_count": 20, "valid_citation_count": 20, "answer_correctness": 1.0, "citation_precision": 1.0, "abstention_accuracy": 1.0, "qualified": true, "by_category": { "cross_document": { "attempts": 7, "passed": 7 }, "direct": { "attempts": 6, "passed": 6 }, "unanswerable": { "attempts": 4, "passed": 4 } }, "model_calls": 17, "model_request_count": 34, "guard_only_answers": 0 } ], "complete_planned_evaluation": false, "qualified": false } }, { "path": "regression-17.jsonl", "sha256": "5e0eff58b18b7ecc3f33a101e4bf0ff3b6bb25ac8e6bd73e8fd2d6b6df5fff36", "attempts": 17, "passes": 15, "failures": 2, "model_request_count": 34, "answer_seconds": { "count": 17, "min": 2.2564609999826644, "median": 2.9022894999943674, "max": 3.903823333006585 }, "summary": { "runner_version": "kingy-document-runner/1.4.0", "questions_sha256": "a69c2f7f8ba63c939d5b226a6a392975af67f5f11d433eecf571ca64a230b469", "references_sha256": "590bbcfb17ec5d3f61323fc40a2a6439d266e62e69a88a489bfc6f008791dccb", "runner_sha256": "c8a7930c185b16567ceefec1049c912b0d2c8c2954d798ad22724e15296726e1", "source_changed_during_run": false, "system_prompt_sha256": "cb330d9b009483d2ed91469195ceff86b231fba66feee14ffbdf344153e1ec3a", "parse_seconds": 0.0353045419906266, "index_seconds": 0.0014627919881604612, "split": "development", "repetitions": [ { "repetition": 1, "attempts": 17, "answerable_count": 13, "answerable_passes": 11, "unanswerable_count": 4, "correct_abstentions": 4, "citation_count": 18, "valid_citation_count": 17, "answer_correctness": 0.8461538461538461, "citation_precision": 0.9444444444444444, "abstention_accuracy": 1.0, "qualified": false, "by_category": { "cross_document": { "attempts": 7, "passed": 5 }, "direct": { "attempts": 6, "passed": 6 }, "unanswerable": { "attempts": 4, "passed": 4 } }, "model_calls": 17, "model_request_count": 34, "guard_only_answers": 0 } ], "complete_planned_evaluation": false, "qualified": false } }, { "path": "regression-9.jsonl", "sha256": "8d9d3f573363ad70e4d58ed4a7a9ec6f6a800afb80a8390a5c49079b31cc332c", "attempts": 9, "passes": 9, "failures": 0, "model_request_count": 18, "answer_seconds": { "count": 9, "min": 2.7322549159871414, "median": 2.7946781659848057, "max": 3.2021582489833236 }, "summary": { "runner_version": "kingy-document-runner/1.4.0", "questions_sha256": "1a3537084889d8cb7f61f2b40a64f1cbf4b3d62a283b2b81ce4214900d84c103", "references_sha256": "3ec6bd67d8fb393ef269cdfa51ef1f949e21de6ed82de3aba948eb9c186c593e", "runner_sha256": "00d0f68dfa240c7ce937be94ade50172c5051cb243ad50b2740e1b292ea5f65c", "source_changed_during_run": false, "system_prompt_sha256": "cb330d9b009483d2ed91469195ceff86b231fba66feee14ffbdf344153e1ec3a", "parse_seconds": 0.042581790999975055, "index_seconds": 0.0017281250038649887, "split": "development", "repetitions": [ { "repetition": 1, "attempts": 9, "answerable_count": 0, "answerable_passes": 0, "unanswerable_count": 9, "correct_abstentions": 9, "citation_count": 0, "valid_citation_count": 0, "answer_correctness": null, "citation_precision": null, "abstention_accuracy": 1.0, "qualified": false, "by_category": { "unanswerable": { "attempts": 9, "passed": 9 } }, "model_calls": 9, "model_request_count": 18, "guard_only_answers": 0 } ], "complete_planned_evaluation": false, "qualified": false } }, { "path": "source-controls.jsonl", "sha256": "6814efd59fc010a24823204140c4147a8347b08c020e9a6535ed502cb3c8e0b4", "attempts": 30, "passes": 30, "failures": 0, "model_request_count": 60, "answer_seconds": { "count": 30, "min": 1.674753208993934, "median": 1.7648069164861226, "max": 1.9064500829845201 }, "summary": { "runner_sha256": "00d0f68dfa240c7ce937be94ade50172c5051cb243ad50b2740e1b292ea5f65c", "source_changed_during_run": false, "development_only": true, "qualificationGranted": false, "attempts": 30, "model_request_count": 60, "passed": true, "failures": [] } } ], "all_retained_attempts": 91, "all_retained_model_request_count": 182, "pre_model_invocation_errors": [ { "commandSplit": "heldout", "bank": "consumed regression-v12 development split", "error": "Selected question split is empty", "model_requests": 0, "resolution": "Re-ran with the bank existing development split; original question/reference files unchanged." } ], "limitations": [ "Consumed regressions and synthetic controls cannot qualify a release or prove comparative accuracy.", "No model server was started or stopped by this repair task.", "No extra model self-correction call was added; inference remains source selection plus semantic sufficiency checking." ] }