{ "schemaVersion": "kingy-document-three-pair-report/1.0.0", "metadata": { "schemaVersion": "kingy-document-three-pair-report/1.0.0", "studyId": "document-comparison-v15-20261014.1", "scopeDescription": "Ten independently authored synthetic English PDFs with selectable text, 30 pages, 20 direct factual questions, 10 cross-document questions and 10 unanswerable questions. The recorded Mac has an Apple M4 Pro (14 CPU cores, 20 GPU cores), 24 GiB memory and macOS 26.3 build 25D125. Both complete workflows use identical Qwen3-4B Q4_K_M bytes and seeded generation settings. Kingy 1.5 uses llama.cpp b11429; native LM Studio 0.4.26+4 uses engine 2.56.0 and unmodified stock RAG revision 8 with eight retrieved chunks. Three fresh model sessions per workflow follow the frozen A1, B1, B2, A2, A3, B3 order.", "history": { "studyId": "document-comparison-v14-20261014.1", "href": "https://kingy.ai/wp-content/plugins/kingy-local-ai/app/documents/comparisons/document-comparison-v14-20261014.1/report.html" }, "instrumentation": { "amended": true, "phase": "after-A1-before-B1", "beforeFirstNativeHeldout": true, "candidateCorpusPromptsScorerUnchanged": true, "description": "Curtis explicitly authorized AppleScript/System Events. PID/window-guarded native activation and file-picker control plus observed CUA chat controls were disclosed after Kingy A1 and before the first native heldout question. Blank-draft preparation interruptions were preserved and recovered before any prompt submission; no answer was retried. During B3, selecting the original empty template through native search restored a sidebar viewport precondition before calling the unchanged frozen clone, submit and capture helpers. That later metadata-only navigation is separately retained. Candidate, corpus, prompts, scorer, native stock RAG, effective settings and scheduled order remained unchanged. No competitor reliability claim follows from these control interruptions.", "evidenceIds": [ "native-amendment", "control-provenance", "study-materials" ] }, "methodAudit": { "frozenNativeCitationComparisonInvalid": true, "sourceAuditRole": "source-audit" }, "limitations": [ "Kingy 1.5 did not pass the frozen supported-answer lower bound. Supplemental review cannot activate or qualify that candidate; the existing recipe 1.4 remains available with its original evidence.", "Frozen automatic native citation/support/cited-abstention effects are limited by a source-resolver failure on cross-page chunks. Automatic zeros do not mean the native citations lack sources.", "Supplemental source review is post-output, unblinded and from the same Kingy project; it preserves every original response and original score. One native completed-stage condition remains unresolved.", "The source audit credits more factually correct native answers. The retained Kingy outputs more consistently attach supporting passages for their requested claims; this is narrow synthetic-corpus evidence, not a general accuracy winner.", "One physical shared Mac, at most 30 pages and two explicit fields; no OCR, handwriting, long-corpus synthesis, arithmetic or concurrent-serving result. Conditions and swap are retained; OS caches were not flushed.", "Different native runtimes and complete retrieval pipelines. Native question time includes preprocessing/embedding/retrieval. Import and neutral warmups are separate; native cold-start duration was not reliably measured.", "No independent hardware reproduction, novice installation, comparative privacy, energy, cloud-cost or GPU-upgrade result. Native egress enforcement remains unverified.", "Control recovery is disclosed. It does not establish LM Studio reliability. Original v1.4 pilot, incomplete native runs, earlier failures and immutable release files remain separate evidence." ], "artifacts": [ { "id": "paired-results", "label": "Original frozen paired calculations; method limitation disclosed", "sha256": "04a75d372d365808b6437ef48ea8a95c0d41e14beafc3500ecfa64d7c2e91a4b", "role": "paired-results", "href": "evidence/paired-results.json", "bytes": 97888 }, { "id": "summaries", "label": "Original automatic summaries for all six sessions", "sha256": "c877d8e31769d22591ba877b9d43755bd2147df5c8dae87e68cb89d453bb8c63", "role": "summaries", "href": "evidence/summaries.json", "bytes": 53960 }, { "id": "analysis-completion", "label": "Completed frozen analysis and output hashes", "sha256": "34e8a336118aab528a6b5b93e7c343dd14b5058414d5235e558cf3e544fb7761", "role": "analysis-completion", "href": "evidence/analysis-completion.json", "bytes": 1131 }, { "id": "session-provenance", "label": "All six original-session integrity checks", "sha256": "9cb5331525a640ba7292dc115f0941d8c2298ac3fede7889a5fce340fa80d058", "role": "session-provenance", "href": "evidence/session-provenance.json", "bytes": 42443 }, { "id": "kingy-raw", "label": "All 120 original Kingy responses and frozen grades", "sha256": "6b0a0907ef5cb53614a8684b206ca2670ed66fe10220dd200515103af65dc71a", "role": "kingy-raw", "href": "evidence/kingy-raw.jsonl", "bytes": 1277112 }, { "id": "native-raw", "label": "All 120 original native responses and frozen grades", "sha256": "39995ba54b09f6be82cb7b7e0a5c87a242c505f8fd055f153dd09f51bea733df", "role": "native-raw", "href": "evidence/native-raw.jsonl", "bytes": 21036970 }, { "id": "protocol", "label": "Original frozen paired protocol", "sha256": "0dc0bd1e0533a73fc279cca744889a41df9591d6fd4c24d6ddf7a76dd42ec32b", "role": "protocol", "href": "evidence/protocol.md", "bytes": 7081 }, { "id": "scoring", "label": "Original frozen common scoring rules", "sha256": "4f64b8eaf4744cc714c90b5bf562e6ffbe37a5ddc121f85a2cb3f6d48939901a", "role": "scoring", "href": "evidence/scoring.md", "bytes": 8686 }, { "id": "freeze", "label": "Before-measurement complete study freeze", "sha256": "c4c2e30f72178f693a471db74abfc1806e42dd4fab5f6c68d68f6327c4b30d28", "role": "freeze", "href": "evidence/freeze.json", "bytes": 5364 }, { "id": "source-audit", "label": "Separate unblinded source review of all 240 originals", "sha256": "b84204892f04a376b12dafc8f244652b5801010cbab7ab9770881b44bb9c5211", "role": "source-audit", "href": "evidence/source-audit.json", "bytes": 887966 }, { "id": "source-audit-verification", "label": "Source-review provenance and unchanged original-result hashes", "sha256": "6185c9b34f42a660581e402cb1c1718e4eeaf24b6ff05d10dea2214f2ebb5634", "href": "evidence/source-audit-verification.json", "bytes": 509 }, { "id": "source-audit-code", "label": "Executed post-output source-review code; not a new primary scorer", "sha256": "fc5408c57dc64d0d97e747afa8808573afdff91df85415f2a09070ab4b64f6ce", "href": "evidence/source-audit-code.txt", "bytes": 9722 }, { "id": "qualification-decision", "label": "Negative candidate qualification and packaging decision", "sha256": "5ca7a6b53f0e99e4a3d9215451fba52698a5fa0c498f288da332972bca648261", "href": "evidence/qualification-decision.json", "bytes": 457 }, { "id": "native-settings", "label": "Frozen native effective settings verified on every measured answer", "sha256": "6c5c1df8f2412149991ccbff5206a90f5dd3f70e54ad15ec43ce2f453992c2ff", "role": "native-settings", "href": "evidence/native-settings.json", "bytes": 11178 }, { "id": "native-runtime", "label": "Installed native engine and embedding artifact hashes", "sha256": "d2f3a8e469022f4a1dd2b78a7ac685d21baa53afa8b6c334e9b190455a354d76", "role": "native-runtime", "href": "evidence/native-runtime.json", "bytes": 3121 }, { "id": "native-rag", "label": "Installed unmodified stock RAG source hashes", "sha256": "c46211b91224f40b122b8140db0d2dd1b12f164ffbbe6047050d4da9f850a0dc", "role": "native-rag", "href": "evidence/native-rag.json", "bytes": 1268 }, { "id": "runtime-recheck", "label": "Post-measurement 34 installed artifact hash checks", "sha256": "8be94e4fb7ab5b8396ce8ede5dc1e6a7548207157083b7bab3acb1b01f9d6d52", "href": "evidence/runtime-recheck.json", "bytes": 1158 }, { "id": "history-manifest", "label": "Original immutable v1.4 manifest", "sha256": "fc28d7f64e75a37768586ded40807195f55aac7d1337d27ecf7a700fbd63a4e6", "role": "history-manifest", "href": "evidence/history-manifest.json", "bytes": 3822 }, { "id": "measurement-integrity", "label": "All original native snapshots and 39 frozen-input checks before reference access", "sha256": "41af8d96b45617e1c6975d0be70c6e3161e547ab925874acbf949c205e39267b", "href": "evidence/measurement-integrity.json", "bytes": 1355 }, { "id": "native-amendment", "label": "Disclosed original control amendment after A1 and before native B1", "sha256": "054f5960092d2a72bb94d460b99e57f6c38e483ebf408359674e97e43d4f6a67", "href": "evidence/native-amendment.json", "bytes": 2777 }, { "id": "development-repairs", "label": "Consumed-case repair controls; separate from unseen evaluation", "sha256": "0b44ade3c1dcd246c7072a10a366699b1108ab343fbd4ec6fb3edc9b8d2d0c1d", "href": "evidence/development-repairs.json", "bytes": 5008 }, { "id": "kingy-settings", "role": "kingy-settings", "label": "Actual pinned Kingy configuration and full-answer timing boundary", "sha256": "1db05507bc7484f5ee1c148b14357d03682ad0940aaf2122ef60fa000f10a0ee", "href": "evidence/kingy-settings.json", "bytes": 1774 }, { "id": "control-provenance", "role": "control-provenance", "label": "Actual native control recovery and instrumentation provenance", "sha256": "dfc662653c41014a87c253da12ef0fd49e199b90a11c6c291daa2576dd8b7eda", "href": "evidence/control-provenance.json", "bytes": 7985 }, { "id": "conditions", "label": "Actual shared-Mac conditions and neutral warmups", "sha256": "070372dba6aab33a2307a3a68b3f711f93e16e4113e09f68f4ab3f5ec89b8b4f", "href": "evidence/conditions.json", "bytes": 82049 }, { "id": "study-materials", "label": "Audit bundle: candidate, corpus, original records/logs and controls; not a qualified recipe", "sha256": "2c3919ab0f7d4d3b8dffcbdf596cb541b8fc6376968b590ec2ad88cac6b06cd5", "href": "evidence/study-materials.zip", "bytes": 9575526 } ] }, "analysisCompletedAt": "2026-10-14T08:34:36.027744+00:00", "uniqueQuestionClusters": 40, "repetitions": 3, "attemptsPerWorkflow": 120, "summaries": { "A1": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 2, "lower": 0.8666666666666667, "upper": 0.9333333333333333 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 1, "lower": 0.8666666666666667, "upper": 0.9 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1, "upper": 1 }, "citation_precision": { "passed": 37, "total": 38, "unresolved": 0, "lower": 0.9736842105263158, "upper": 0.9736842105263158 }, "unresolved_questions": 2, "explicit_page_citations": 37, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "latency": { "n": 40, "unavailable": 0, "p50": 2.61784297900158, "p95": 3.9339249999902677, "max": 4.096474207995925, "errors_retained": 0, "timeouts_retained": 0, "boundary": null }, "answer_model_request_count": 79, "original_session_receipt": { "original_summary": { "executor_version": "kingy-comparison-session/1.0.0", "system": "A", "repetition": 1, "status": "complete", "qualificationGranted": false, "complete_query_deadline_seconds": 90, "generation_seed": 271828, "client_offline_profile": "offline.sb", "client_offline_profile_sha256": "e5462ef1daae3785012cb8e4fa05b89105bf4ac4d98be9580a8c357365a4f54f", "timing_boundary": "runner.ask entry through its complete returned answer, source resolution and sufficiency check; excludes prior PDF import/index, cold start, warmup and later grading.", "cache_policy": "cache_prompt false; no claim that OS file caches were flushed.", "freeze_sha256": "c4c2e30f72178f693a471db74abfc1806e42dd4fab5f6c68d68f6327c4b30d28", "schedule_sha256": "fe1c5a3f3efb992c345dc609dbc2e4fe1cd2980b5e66c5eccd89c9bd3d770f42", "questions_sha256": "76492fc420df3797365e1cc8f89aaf87a024747ab3846c12ca4ce007bb2029d2", "executor_sha256": "867eb5347c47ba098bd38c321cb9b30963e7d24f1e10a72591648993ed9db614", "runner_sha256": "d5e3c68878af3926c8844e7406a5f9f2666c133eabe624bc34836a85161f17b2", "model_sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5", "corpus_pdf_sha256": { "doc-01.pdf": "1d360b8e51d8a57c1a366c502a22800b375668ef7b16a9f690ca89cb0df1fe8a", "doc-02.pdf": "8e791cfcb4a8269645901bef23a65b55d1a915e3ee50daff285595c005bdde14", "doc-03.pdf": "a3026b5ec89e8048aade43cc9041d1b47a2a6c218bec69ea61fcb4928aded921", "doc-04.pdf": "171c36640f813254e2f00a840bfb3af7afd132759a7918b44b0a101abd790fd0", "doc-05.pdf": "bada3f5e96cd20db3e60df2855ca32536499e0ad794ca0e754110bb536818642", "doc-06.pdf": "c9e93f31f3a9846764fb57faa6c213f473a6771bba5ff5f90acf8fa504c852b2", "doc-07.pdf": "1fcb01d0e73fcee53275215fd320179d1aad807d3dbd55b30d35b78992544232", "doc-08.pdf": "d0ce85c99f55be103e8f514115744196eb5ceb6190ac589c079841e280571114", "doc-09.pdf": "f8d56fd31a7017fef9e8f1feee71b4fe3dee5bcc3104e0c089b0c754a33f5fcd", "doc-10.pdf": "6a8c5888b5fb930261312c04f7a127a397235afdf4b60468892afd466972c736" }, "planned_question_ids": [ "held-07", "held-33", "held-05", "held-39", "held-36", "held-29", "held-16", "held-13", "held-28", "held-22", "held-09", "held-35", "held-26", "held-14", "held-25", "held-21", "held-04", "held-02", "held-24", "held-38", "held-01", "held-40", "held-37", "held-34", "held-20", "held-31", "held-10", "held-30", "held-08", "held-11", "held-17", "held-23", "held-18", "held-32", "held-15", "held-03", "held-12", "held-19", "held-27", "held-06" ], "session_order": [ "A1", "B1", "B2", "A2", "A3", "B3" ], "artifact_verification_seconds": 2.081889624998439, "runtime": { "name": "llama.cpp", "binary_sha256": "51afbf4ea6485b9ada19e99af97181f12baca42ef704b1e291c26a9ac0644eae", "version": { "command": [ "/opt/homebrew/Cellar/llama.cpp/0.6.0/bin/llama-server", "--version" ], "exit_code": 0, "stdout": "", "stderr": "version: 0.6.0 (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000334 for Darwin arm64" } }, "before": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 52%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 17408.00M used = 15949.50M free = 1458.50M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 68Gi 15% 453k 716M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "task_owned_server_pid": 36446, "cold_process_to_health_seconds": 2.0370149160153233, "ready_model_identity": { "models": [ { "name": "kingy-qwen3-4b", "model": "kingy-qwen3-4b", "modified_at": "", "size": "", "digest": "", "type": "model", "description": "", "tags": [ "" ], "capabilities": [ "completion" ], "parameters": "", "details": { "parent_model": "", "format": "gguf", "family": "", "families": [ "" ], "parameter_size": "", "quantization_level": "" } } ], "object": "list", "data": [ { "id": "kingy-qwen3-4b", "aliases": [ "kingy-qwen3-4b" ], "tags": [], "object": "model", "architecture": { "input_modalities": [ "text" ], "output_modalities": [ "text" ] }, "created": 1791948102, "owned_by": "llamacpp", "meta": { "vocab_type": 2, "n_vocab": 151936, "n_ctx": 8192, "n_ctx_train": 40960, "n_embd": 2560, "n_params": 4022468096, "size": 2491323904, "ftype": "Q4_K - Medium" } } ] }, "warmup_model_request_count": 1, "warmup_seconds": 0.07289979100460187, "parse_seconds": 0.046252166997874156, "index_seconds": 0.0011787080147769302, "loaded_pdf_pages": 30, "retained_attempts": 40, "planned_attempts": 40, "unexecuted_question_ids": [], "unexecuted_attempt_count": 0, "model_request_count": 79, "session_total_model_request_count": 80, "attempt_errors": 0, "timeouts": 0, "after": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 50%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 18432.00M used = 17095.81M free = 1336.19M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 67Gi 15% 453k 705M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "session_wall_seconds": 117.45725708297687, "frozen_inputs_unchanged": true }, "integrity_issues": [], "source_sha256": { "session-plan.json": "6fba21d5cdaa1d4444c2cb644faf0b9c96aff6531c143408b16a565d4033c598", "session-summary.json": "70c5469588b2dd1cc6056386682e8ea69e517431b6e595216b1b395d81405f12", "attempts.jsonl": "516f7cbf0b7625fdb5e938a32c8863228b8e1e1c34a2c77a660751cf8aa4cfb6" }, "complete_40_original_rows": true }, "provenance_verified": true }, "A2": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 2, "lower": 0.8666666666666667, "upper": 0.9333333333333333 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 1, "lower": 0.8666666666666667, "upper": 0.9 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1, "upper": 1 }, "citation_precision": { "passed": 37, "total": 38, "unresolved": 0, "lower": 0.9736842105263158, "upper": 0.9736842105263158 }, "unresolved_questions": 2, "explicit_page_citations": 37, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "latency": { "n": 40, "unavailable": 0, "p50": 2.617779354011873, "p95": 3.9865453749953303, "max": 4.003020333999302, "errors_retained": 0, "timeouts_retained": 0, "boundary": null }, "answer_model_request_count": 79, "original_session_receipt": { "original_summary": { "executor_version": "kingy-comparison-session/1.0.0", "system": "A", "repetition": 2, "status": "complete", "qualificationGranted": false, "complete_query_deadline_seconds": 90, "generation_seed": 271828, "client_offline_profile": "offline.sb", "client_offline_profile_sha256": "e5462ef1daae3785012cb8e4fa05b89105bf4ac4d98be9580a8c357365a4f54f", "timing_boundary": "runner.ask entry through its complete returned answer, source resolution and sufficiency check; excludes prior PDF import/index, cold start, warmup and later grading.", "cache_policy": "cache_prompt false; no claim that OS file caches were flushed.", "freeze_sha256": "c4c2e30f72178f693a471db74abfc1806e42dd4fab5f6c68d68f6327c4b30d28", "schedule_sha256": "fe1c5a3f3efb992c345dc609dbc2e4fe1cd2980b5e66c5eccd89c9bd3d770f42", "questions_sha256": "76492fc420df3797365e1cc8f89aaf87a024747ab3846c12ca4ce007bb2029d2", "executor_sha256": "867eb5347c47ba098bd38c321cb9b30963e7d24f1e10a72591648993ed9db614", "runner_sha256": "d5e3c68878af3926c8844e7406a5f9f2666c133eabe624bc34836a85161f17b2", "model_sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5", "corpus_pdf_sha256": { "doc-01.pdf": "1d360b8e51d8a57c1a366c502a22800b375668ef7b16a9f690ca89cb0df1fe8a", "doc-02.pdf": "8e791cfcb4a8269645901bef23a65b55d1a915e3ee50daff285595c005bdde14", "doc-03.pdf": "a3026b5ec89e8048aade43cc9041d1b47a2a6c218bec69ea61fcb4928aded921", "doc-04.pdf": "171c36640f813254e2f00a840bfb3af7afd132759a7918b44b0a101abd790fd0", "doc-05.pdf": "bada3f5e96cd20db3e60df2855ca32536499e0ad794ca0e754110bb536818642", "doc-06.pdf": "c9e93f31f3a9846764fb57faa6c213f473a6771bba5ff5f90acf8fa504c852b2", "doc-07.pdf": "1fcb01d0e73fcee53275215fd320179d1aad807d3dbd55b30d35b78992544232", "doc-08.pdf": "d0ce85c99f55be103e8f514115744196eb5ceb6190ac589c079841e280571114", "doc-09.pdf": "f8d56fd31a7017fef9e8f1feee71b4fe3dee5bcc3104e0c089b0c754a33f5fcd", "doc-10.pdf": "6a8c5888b5fb930261312c04f7a127a397235afdf4b60468892afd466972c736" }, "planned_question_ids": [ "held-23", "held-30", "held-02", "held-07", "held-40", "held-20", "held-28", "held-27", "held-01", "held-12", "held-24", "held-26", "held-22", "held-31", "held-19", "held-32", "held-06", "held-36", "held-03", "held-04", "held-35", "held-15", "held-16", "held-11", "held-29", "held-18", "held-09", "held-17", "held-14", "held-21", "held-05", "held-10", "held-39", "held-08", "held-25", "held-34", "held-38", "held-13", "held-37", "held-33" ], "session_order": [ "A1", "B1", "B2", "A2", "A3", "B3" ], "artifact_verification_seconds": 2.187174458988011, "runtime": { "name": "llama.cpp", "binary_sha256": "51afbf4ea6485b9ada19e99af97181f12baca42ef704b1e291c26a9ac0644eae", "version": { "command": [ "/opt/homebrew/Cellar/llama.cpp/0.6.0/bin/llama-server", "--version" ], "exit_code": 0, "stdout": "", "stderr": "version: 0.6.0 (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000334 for Darwin arm64" } }, "before": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 78%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 10240.00M used = 9304.00M free = 936.00M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 72Gi 14% 453k 753M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t44%; charging; 3:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "task_owned_server_pid": 53158, "cold_process_to_health_seconds": 1.783840957999928, "ready_model_identity": { "models": [ { "name": "kingy-qwen3-4b", "model": "kingy-qwen3-4b", "modified_at": "", "size": "", "digest": "", "type": "model", "description": "", "tags": [ "" ], "capabilities": [ "completion" ], "parameters": "", "details": { "parent_model": "", "format": "gguf", "family": "", "families": [ "" ], "parameter_size": "", "quantization_level": "" } } ], "object": "list", "data": [ { "id": "kingy-qwen3-4b", "aliases": [ "kingy-qwen3-4b" ], "tags": [], "object": "model", "architecture": { "input_modalities": [ "text" ], "output_modalities": [ "text" ] }, "created": 1791964368, "owned_by": "llamacpp", "meta": { "vocab_type": 2, "n_vocab": 151936, "n_ctx": 8192, "n_ctx_train": 40960, "n_embd": 2560, "n_params": 4022468096, "size": 2491323904, "ftype": "Q4_K - Medium" } } ] }, "warmup_model_request_count": 1, "warmup_seconds": 0.08193491699057631, "parse_seconds": 0.05187637501512654, "index_seconds": 0.0012360000109765679, "loaded_pdf_pages": 30, "retained_attempts": 40, "planned_attempts": 40, "unexecuted_question_ids": [], "unexecuted_attempt_count": 0, "model_request_count": 79, "session_total_model_request_count": 80, "attempt_errors": 0, "timeouts": 0, "after": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 79%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 10240.00M used = 8768.00M free = 1472.00M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 72Gi 14% 453k 753M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t44%; charging; (no estimate) present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "session_wall_seconds": 117.47648024998489, "frozen_inputs_unchanged": true }, "integrity_issues": [], "source_sha256": { "session-plan.json": "9630876cbc2c4c843447ad7f82be6acac32a26f89ea75edc6db60fc26a9a5bfd", "session-summary.json": "abaf4048cc6876150cf580ac49a307d6aa56c450d854f9d80042c32b036e349e", "attempts.jsonl": "fa38555d6b71ce808cd00514d7d1d01f666331290ca37e82b40c1be33c5ebd4b" }, "complete_40_original_rows": true }, "provenance_verified": true }, "A3": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 2, "lower": 0.8666666666666667, "upper": 0.9333333333333333 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 1, "lower": 0.8666666666666667, "upper": 0.9 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1, "upper": 1 }, "citation_precision": { "passed": 37, "total": 38, "unresolved": 0, "lower": 0.9736842105263158, "upper": 0.9736842105263158 }, "unresolved_questions": 2, "explicit_page_citations": 37, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": true, "abstention_accuracy": true }, "latency": { "n": 40, "unavailable": 0, "p50": 2.615461082998081, "p95": 3.958056250005029, "max": 3.9880023329751566, "errors_retained": 0, "timeouts_retained": 0, "boundary": null }, "answer_model_request_count": 79, "original_session_receipt": { "original_summary": { "executor_version": "kingy-comparison-session/1.0.0", "system": "A", "repetition": 3, "status": "complete", "qualificationGranted": false, "complete_query_deadline_seconds": 90, "generation_seed": 271828, "client_offline_profile": "offline.sb", "client_offline_profile_sha256": "e5462ef1daae3785012cb8e4fa05b89105bf4ac4d98be9580a8c357365a4f54f", "timing_boundary": "runner.ask entry through its complete returned answer, source resolution and sufficiency check; excludes prior PDF import/index, cold start, warmup and later grading.", "cache_policy": "cache_prompt false; no claim that OS file caches were flushed.", "freeze_sha256": "c4c2e30f72178f693a471db74abfc1806e42dd4fab5f6c68d68f6327c4b30d28", "schedule_sha256": "fe1c5a3f3efb992c345dc609dbc2e4fe1cd2980b5e66c5eccd89c9bd3d770f42", "questions_sha256": "76492fc420df3797365e1cc8f89aaf87a024747ab3846c12ca4ce007bb2029d2", "executor_sha256": "867eb5347c47ba098bd38c321cb9b30963e7d24f1e10a72591648993ed9db614", "runner_sha256": "d5e3c68878af3926c8844e7406a5f9f2666c133eabe624bc34836a85161f17b2", "model_sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5", "corpus_pdf_sha256": { "doc-01.pdf": "1d360b8e51d8a57c1a366c502a22800b375668ef7b16a9f690ca89cb0df1fe8a", "doc-02.pdf": "8e791cfcb4a8269645901bef23a65b55d1a915e3ee50daff285595c005bdde14", "doc-03.pdf": "a3026b5ec89e8048aade43cc9041d1b47a2a6c218bec69ea61fcb4928aded921", "doc-04.pdf": "171c36640f813254e2f00a840bfb3af7afd132759a7918b44b0a101abd790fd0", "doc-05.pdf": "bada3f5e96cd20db3e60df2855ca32536499e0ad794ca0e754110bb536818642", "doc-06.pdf": "c9e93f31f3a9846764fb57faa6c213f473a6771bba5ff5f90acf8fa504c852b2", "doc-07.pdf": "1fcb01d0e73fcee53275215fd320179d1aad807d3dbd55b30d35b78992544232", "doc-08.pdf": "d0ce85c99f55be103e8f514115744196eb5ceb6190ac589c079841e280571114", "doc-09.pdf": "f8d56fd31a7017fef9e8f1feee71b4fe3dee5bcc3104e0c089b0c754a33f5fcd", "doc-10.pdf": "6a8c5888b5fb930261312c04f7a127a397235afdf4b60468892afd466972c736" }, "planned_question_ids": [ "held-18", "held-16", "held-06", "held-05", "held-37", "held-23", "held-28", "held-04", "held-26", "held-22", "held-34", "held-32", "held-13", "held-35", "held-07", "held-20", "held-24", "held-29", "held-02", "held-14", "held-39", "held-38", "held-09", "held-36", "held-17", "held-01", "held-25", "held-11", "held-19", "held-30", "held-03", "held-27", "held-10", "held-12", "held-15", "held-33", "held-08", "held-21", "held-40", "held-31" ], "session_order": [ "A1", "B1", "B2", "A2", "A3", "B3" ], "artifact_verification_seconds": 1.8421307500102557, "runtime": { "name": "llama.cpp", "binary_sha256": "51afbf4ea6485b9ada19e99af97181f12baca42ef704b1e291c26a9ac0644eae", "version": { "command": [ "/opt/homebrew/Cellar/llama.cpp/0.6.0/bin/llama-server", "--version" ], "exit_code": 0, "stdout": "", "stderr": "version: 0.6.0 (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000334 for Darwin arm64" } }, "before": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 79%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 10240.00M used = 8672.00M free = 1568.00M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 72Gi 14% 453k 755M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t44%; charging; (no estimate) present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "task_owned_server_pid": 54671, "cold_process_to_health_seconds": 1.5292013750004116, "ready_model_identity": { "models": [ { "name": "kingy-qwen3-4b", "model": "kingy-qwen3-4b", "modified_at": "", "size": "", "digest": "", "type": "model", "description": "", "tags": [ "" ], "capabilities": [ "completion" ], "parameters": "", "details": { "parent_model": "", "format": "gguf", "family": "", "families": [ "" ], "parameter_size": "", "quantization_level": "" } } ], "object": "list", "data": [ { "id": "kingy-qwen3-4b", "aliases": [ "kingy-qwen3-4b" ], "tags": [], "object": "model", "architecture": { "input_modalities": [ "text" ], "output_modalities": [ "text" ] }, "created": 1791964501, "owned_by": "llamacpp", "meta": { "vocab_type": 2, "n_vocab": 151936, "n_ctx": 8192, "n_ctx_train": 40960, "n_embd": 2560, "n_params": 4022468096, "size": 2491323904, "ftype": "Q4_K - Medium" } } ] }, "warmup_model_request_count": 1, "warmup_seconds": 0.06480879199807532, "parse_seconds": 0.03830933399149217, "index_seconds": 0.0011352500005159527, "loaded_pdf_pages": 30, "retained_attempts": 40, "planned_attempts": 40, "unexecuted_question_ids": [], "unexecuted_attempt_count": 0, "model_request_count": 79, "session_total_model_request_count": 80, "attempt_errors": 0, "timeouts": 0, "after": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 78%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 10240.00M used = 8536.00M free = 1704.00M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 72Gi 14% 453k 755M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t44%; charging; (no estimate) present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "session_wall_seconds": 116.39186295800027, "frozen_inputs_unchanged": true }, "integrity_issues": [], "source_sha256": { "session-plan.json": "c6dc51ba6a321123be01d8aa95bf24e8c73b538d007dc7eb8d5d06c2f768d6cf", "session-summary.json": "c952af18253329a30487a4d7e7c241ceacb64f69ed4304b24613347aca01c4ce", "attempts.jsonl": "8b3d7915bf20352220a6696afc03cd567957da5103c729afe255393a29c17f90" }, "complete_40_original_rows": true }, "provenance_verified": true }, "B1": { "attempts": 40, "answer_correctness": { "passed": 22, "total": 30, "unresolved": 8, "lower": 0.7333333333333333, "upper": 1 }, "supported_answer_success": { "passed": 0, "total": 30, "unresolved": 0, "lower": 0, "upper": 0 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 0, "lower": 0.6, "upper": 0.6 }, "citation_precision": { "passed": 0, "total": 44, "unresolved": 0, "lower": 0, "upper": 0 }, "unresolved_questions": 8, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "latency": { "n": 40, "unavailable": 0, "p50": 8.79, "p95": 15.11, "max": 37.187, "errors_retained": 0, "timeouts_retained": 0, "boundary": "Native app submission to documented logger final-answer event; includes native preprocessing/retrieval/embedding/inference; excludes later title generation and automation/export." }, "answer_model_request_count": 40, "original_session_receipt": { "session_path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v15/sessions/B1", "logger": { "path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v15/sessions/B1/model-io.jsonl", "sha256": "2aecebea961c8cf69658139b1392cfe32d6e2e48a42efcdd7de79d2843dcff17", "parsed_events": 80, "malformed_lines": [] }, "integrity_issues": [], "source_sha256": { "session-plan.json": "ee2643914458e0e4171b5e9fd771d614b03427e4794e9f25f30ecdb1609eed4b", "native-manifest.jsonl": "ba733750a96e5d966891cb3e44d5828d66b1dc113c5e2ce4ced94c14d5d1b381", "pending.json": "38e0b9de817f645c4bec37c0d4a3e58baecccb040f5718dc069a72c7385a0bed" }, "retained_manifest_rows": 40, "unmanifested_pending_attempt": false, "complete_40_original_manifest_rows": true, "original_files_modified_by_analysis": false }, "provenance_verified": true }, "B2": { "attempts": 40, "answer_correctness": { "passed": 22, "total": 30, "unresolved": 8, "lower": 0.7333333333333333, "upper": 1 }, "supported_answer_success": { "passed": 0, "total": 30, "unresolved": 0, "lower": 0, "upper": 0 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 0, "lower": 0.6, "upper": 0.6 }, "citation_precision": { "passed": 0, "total": 44, "unresolved": 0, "lower": 0, "upper": 0 }, "unresolved_questions": 8, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "latency": { "n": 40, "unavailable": 0, "p50": 8.536, "p95": 9.725, "max": 9.861, "errors_retained": 0, "timeouts_retained": 0, "boundary": "Native app submission to documented logger final-answer event; includes native preprocessing/retrieval/embedding/inference; excludes later title generation and automation/export." }, "answer_model_request_count": 40, "original_session_receipt": { "session_path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v15/sessions/B2", "logger": { "path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v15/sessions/B2/model-io.jsonl", "sha256": "ef3acb2bcb9b51bd0d671f9612a0009c6519a498535fbc477cc25206e2e53b8b", "parsed_events": 80, "malformed_lines": [] }, "integrity_issues": [], "source_sha256": { "session-plan.json": "f6f1295cda923a32846fc848c19f3f55b811052205d132885b6e042d7cea1139", "native-manifest.jsonl": "4f48cae060562c4538bf2cfe462bb5d42adbc45297776a48454308b3e99b8a3a", "pending.json": "38e0b9de817f645c4bec37c0d4a3e58baecccb040f5718dc069a72c7385a0bed" }, "retained_manifest_rows": 40, "unmanifested_pending_attempt": false, "complete_40_original_manifest_rows": true, "original_files_modified_by_analysis": false }, "provenance_verified": true }, "B3": { "attempts": 40, "answer_correctness": { "passed": 22, "total": 30, "unresolved": 8, "lower": 0.7333333333333333, "upper": 1 }, "supported_answer_success": { "passed": 0, "total": 30, "unresolved": 0, "lower": 0, "upper": 0 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 0, "lower": 0.6, "upper": 0.6 }, "citation_precision": { "passed": 0, "total": 44, "unresolved": 0, "lower": 0, "upper": 0 }, "unresolved_questions": 8, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "latency": { "n": 40, "unavailable": 0, "p50": 7.3454999999999995, "p95": 8.91, "max": 11.747, "errors_retained": 0, "timeouts_retained": 0, "boundary": "Native app submission to documented logger final-answer event; includes native preprocessing/retrieval/embedding/inference; excludes later title generation and automation/export." }, "answer_model_request_count": 40, "original_session_receipt": { "session_path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v15/sessions/B3", "logger": { "path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v15/sessions/B3/model-io.jsonl", "sha256": "f727c58a7c19fd2744ee20ff855c6e8e58b2f1a050847da7d1a1da0da7f2ea60", "parsed_events": 80, "malformed_lines": [] }, "integrity_issues": [], "source_sha256": { "session-plan.json": "192789edd74ceb39598b48c7addf11f2dffa91ab39868f8df4c34fd72f46bda1", "native-manifest.jsonl": "df7c1381241a8cb1bb9e26309668e442d342063f41cab75d601835770257eb19", "pending.json": "38e0b9de817f645c4bec37c0d4a3e58baecccb040f5718dc069a72c7385a0bed" }, "retained_manifest_rows": 40, "unmanifested_pending_attempt": false, "complete_40_original_manifest_rows": true, "original_files_modified_by_analysis": false }, "provenance_verified": true } }, "pairedEffects": { "complete_question_success": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.75, 0.775 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.625, 0.9 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "supported_answer_success": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.8666666666666667, 0.9 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.7333333333333333, 1 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "canonical_fact_correctness": { "unique_questions": 30, "effect_bounds_A_minus_B": [ -0.13333333333333333, 0.2 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.26666666666666666, 0.36666666666666664 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "correct_abstention": { "unique_questions": 10, "effect_bounds_A_minus_B": [ 0.4, 0.4 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.1, 0.7 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "emitted_citation_precision": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.9736842105263158, 0.9736842105263158 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0.9090909090909091, 1 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." } }, "automaticWinner": null }