{ "schemaVersion": "kingy-document-three-pair-report/1.0.0", "metadata": { "schemaVersion": "kingy-document-three-pair-report/1.0.0", "studyId": "document-comparison-v16-20261014.1", "scopeDescription": "Kingy 1.5.1 did not qualify and was not launched; the active download remains workflow 1.4. Ten separately authored English selectable-text synthetic PDFs, 30 pages, 30 answerable and 10 unknown questions, at most two requested fields. Apple M4 Pro with 24 GiB, macOS 26.3 build 25D125. Identical Qwen3-4B Q4_K_M weights and seeded generation; Kingy 1.5.1 / llama.cpp b11429 versus native LM Studio 0.4.26+4, engine 2.56.0 and unmodified stock RAG revision 8 with eight retrieved chunks. Three fresh model sessions each in frozen A1, B1, B2, A2, A3, B3 order.", "history": { "studyId": "document-comparison-v14-20261014.1", "href": "https://kingy.ai/wp-content/plugins/kingy-local-ai/app/documents/comparisons/document-comparison-v14-20261014.1/report.html" }, "instrumentation": { "amended": true, "phase": "after-A1-before-B1", "beforeFirstNativeHeldout": true, "candidateCorpusPromptsScorerUnchanged": true, "description": "Closed modal left CUA coordinate input unavailable while accessibility reads remained valid. Public System Events focuses the two screenshot-declared points and performs guarded Search/owned Rename paste after exact accessibility text selection; clipboard restoration waits for cleared and exact-value acknowledgement. Named-chat More options replaces right-click with the identical native menu. Native factual drafts use the public accessibility field setter, with empty owned chat/ten identical PDFs/exact saved prompt/model/unique selected-name guards retained before one Send. The task-owned privileged file worker runs only the pinned public System Events script; no model APIs or native-record edits. Pre-submission control errors and the rejected early worker restart remain private setup evidence and are not model or product reliability results.", "evidenceIds": [ "native-amendment", "control-provenance", "study-materials" ] }, "limitations": [ "Kingy 1.5.1 failed all three runs: 26/30 supported answers (86.7%) and 38/42 valid emitted citations (90.5%), below the frozen 90% and 95% targets. All 10/10 unknown questions passed in each run. Workflow 1.4 remains the active recipe; no new 1.5.1 installation or launch is claimed.", "The three old Kingy failures pass development regression, and the repaired cross-page mapper resolves all 132 consumed native cross-page citation instances. Those are repair controls, not fresh qualification results. Four new Kingy cases failed repeatedly; the new corpus and original grades were not edited after exposure.", "LM Studio has 13 unresolved factual judgments, 11 unresolved supported-answer judgments and four unresolved unknown judgments per run. Conservative automatic matching does not resolve valid prose, conditional meaning or every citation. No accuracy, source-quality or refusal advantage is asserted from those unresolved grades.", "Descriptive complete-answer time was lower for Kingy in all 120 pairs, including all 42 paired attempts where both systems had confirmed passing grades. All-attempt medians were 2.88 seconds for Kingy and 6.30 seconds for native LM Studio. All-attempt timing includes incorrect responses. This is observed latency on this fixed corpus and shared Mac, not model throughput, novice setup speed or a population claim.", "Kingy emits explicit PDF page citations. Native answers use citation IDs linked to retrieval cards; the scorer maps unique exact source quotes across pages for credit. This difference in page presentation was observed, not tested as a user-comprehension advantage.", "Separate-agent synthetic authorship is not human gold validation or independent hardware reproduction. Forty unique question clusters repeated three times are not 120 independent questions. The shared-machine conditions and swap are retained; OS caches were not flushed.", "Different native runtimes and retrieval pipelines were compared as complete workflows. Native timing includes preprocessing, embedding, retrieval and inference. PDF import, startup, neutral warmup and GUI preparation/export are excluded; native cold-start time was not reliably measured.", "No OCR, handwriting, long-corpus synthesis, arithmetic, concurrent serving, cloud quality/cost, GPU upgrade, energy, comparative privacy or novice-installation result. Native egress enforcement remains unverified.", "Original workflow 1.4 and failed 1.5.0 reports remain immutable historical evidence. This new study does not rewrite their scores, method limitations, failed candidates or released files." ], "artifacts": [ { "id": "paired-results", "label": "Frozen paired calculations, unresolved bounds and cluster sensitivity", "sha256": "80cf4bb52c1bcdd7313c398aba3d3e61cbf2fd85929f811f20801b742b3cf5ae", "role": "paired-results", "href": "evidence/paired-results.json", "bytes": 99047 }, { "id": "summaries", "label": "Frozen summaries for all six original sessions", "sha256": "4a16828d13c1e97f01cf485e7edc64791dcc7bf3841f8069b67a4aeb3e684f92", "role": "summaries", "href": "evidence/summaries.json", "bytes": 54430 }, { "id": "analysis-completion", "label": "Completed analysis and exact output hashes", "sha256": "0351587588904ce5aa0bfd543de55b351dac4f7ee3504ebafd4580aa15563abc", "role": "analysis-completion", "href": "evidence/analysis-completion.json", "bytes": 1131 }, { "id": "session-provenance", "label": "All six original-session integrity checks", "sha256": "f46d8fdd28b3b55a94bd3e23e1920200536b13c57e58852123715055ef2c9a0a", "role": "session-provenance", "href": "evidence/session-provenance.json", "bytes": 42717 }, { "id": "kingy-raw", "label": "All 120 original Kingy responses and unchanged frozen grades", "sha256": "ce90e99ada1f925f40bbe7632a5ca738e86b9b9f243c259f588350dc15b3435e", "role": "kingy-raw", "href": "evidence/kingy-raw.jsonl", "bytes": 1261500 }, { "id": "native-raw", "label": "All 120 original native LM Studio responses and unchanged frozen grades", "sha256": "6f01abb4a2861dff23ec9ba78ff86a966dc9195552bc7a64e94ed0d9a12a15e7", "role": "native-raw", "href": "evidence/native-raw.jsonl", "bytes": 21270335 }, { "id": "protocol", "label": "Before-output frozen paired protocol", "sha256": "b432d414bc6c8c912510db168229b80cce472d2cb14c93bb7a82071d72a3604b", "role": "protocol", "href": "evidence/protocol.md", "bytes": 8153 }, { "id": "scoring", "label": "Common scorer 1.2.0 and conservative scoring rules", "sha256": "06fa5f5b4c0ff16ac615cd662d560fa7148a6e40525bac94a195df744173435f", "role": "scoring", "href": "evidence/scoring.md", "bytes": 9466 }, { "id": "freeze", "label": "Complete study freeze before inference", "sha256": "b0dde55a2f92f156f588d5df510832b2ea874e3da41e072ab096564269356d43", "role": "freeze", "href": "evidence/freeze.json", "bytes": 6212 }, { "id": "method-freeze", "label": "Candidate and method freeze before root access to the fresh bank", "sha256": "2ac778674ffdde98983c3f818721bda4c1d1bd5c9eb1af7eb4c1e5d3b85c939a", "href": "evidence/method-freeze.json", "bytes": 4890 }, { "id": "qualification-decision", "label": "Negative release decision; workflow 1.4 stays active", "sha256": "c23c0fd87635aeb9d213cdb3ff38c2b08553ccb8a3d55c6aac2877d19ae06c2c", "href": "evidence/qualification-decision.json", "bytes": 2468 }, { "id": "fresh-failures", "label": "All four repeated fresh Kingy failure cases, without retuning", "sha256": "e6520628a3fe0304acf1f5583f976cc0dbbc1e4c9d0644a6c599d2515437f03a", "href": "evidence/fresh-failures.json", "bytes": 9286 }, { "id": "descriptive-latency", "label": "Descriptive timing of all paired attempts and the disclosed joint-pass subset", "sha256": "90c915ec697c7ee955403a474f3ddb179c93524a07b92b8dc8db6e6ee9ecd6dc", "href": "evidence/descriptive-latency.json", "bytes": 2773 }, { "id": "native-settings", "label": "Frozen effective native settings checked on every measured answer", "sha256": "d28305b7df7214eb6e4797591b90991ab6add825980bbc38a91ae10958a7668d", "role": "native-settings", "href": "evidence/native-settings.json", "bytes": 11338 }, { "id": "native-runtime", "label": "Installed native engine and local embedding hashes, rechecked in this study", "sha256": "d2f3a8e469022f4a1dd2b78a7ac685d21baa53afa8b6c334e9b190455a354d76", "role": "native-runtime", "href": "evidence/native-runtime.json", "bytes": 3121 }, { "id": "native-rag", "label": "Unmodified installed stock RAG source hashes, rechecked in this study", "sha256": "c46211b91224f40b122b8140db0d2dd1b12f164ffbbe6047050d4da9f850a0dc", "role": "native-rag", "href": "evidence/native-rag.json", "bytes": 1268 }, { "id": "runtime-recheck", "label": "After-measurement verification of 34 installed artifact hashes", "sha256": "0408c0858fa34ae857513514ecfea35f9bc04f50294006468611abba39065d93", "href": "evidence/runtime-recheck.json", "bytes": 11646 }, { "id": "history-manifest", "label": "Original immutable workflow 1.4 comparison manifest", "sha256": "fc28d7f64e75a37768586ded40807195f55aac7d1337d27ecf7a700fbd63a4e6", "role": "history-manifest", "href": "evidence/history-manifest.json", "bytes": 3822 }, { "id": "v15-history-manifest", "label": "Original immutable failed 1.5.0 comparison manifest", "sha256": "f62295ede008d7312b9dc967d7c7394fd552a3736558b04286f69b075827ed5e", "href": "evidence/v15-history-manifest.json", "bytes": 4793 }, { "id": "history-preservation", "label": "226 earlier candidate, method and public evidence files preserved", "sha256": "727be257eed6f581370a6d5f53c7c6b185bbf2898fb684cfb281e2ea9f75d6f6", "href": "evidence/history-preservation.json", "bytes": 35889 }, { "id": "measurement-integrity", "label": "All 240 originals sealed before root reference access", "sha256": "5a9c3f9ad355ecc4e2de559d8864b5c577100629b2ed799db598369d59bc744c", "href": "evidence/measurement-integrity.json", "bytes": 17668 }, { "id": "native-amendment", "label": "Disclosed GUI transport amendment before the first native held-out question", "sha256": "c15be6c13604dfea6dc6e1112bbc5dc1ebe696f99acba7c9ef88fae08ad90cb0", "href": "evidence/native-amendment.json", "bytes": 2087 }, { "id": "development-repairs", "label": "Three consumed-case fixes and cross-page controls; not unseen qualification", "sha256": "4594882a6b11ce9d8c52f53581f80b40b3d2db0a62a689a2c85342478bb2d517", "href": "evidence/development-repairs.json", "bytes": 625 }, { "id": "corpus-authorship", "label": "Separate agent authorship and before-output corpus freeze", "sha256": "ed05d20bad33d760ce7249563525b3b165b24fc24dfec055f4ce7711eb013efd", "href": "evidence/corpus-authorship.json", "bytes": 2693 }, { "id": "input-provenance", "label": "Exact author-pack to execution-pack copy provenance", "sha256": "3be03c9a3f64e2c30e13bc9e2060d075263dbfade4a56ce3192c44feb0d0016a", "href": "evidence/input-provenance.json", "bytes": 1651 }, { "id": "kingy-settings", "label": "Actual Kingy settings and complete-answer timing boundary", "sha256": "9f927e9832082795b070c037c60b918950c6fa19f6aa5b4cb4c713c718db2292", "role": "kingy-settings", "href": "evidence/kingy-settings.json", "bytes": 1767 }, { "id": "control-provenance", "label": "Actual control amendment and original-attempt provenance", "sha256": "236c7e9268b329d36f9c678bc95457657902c5d9fecf3b0d31213cefabc3ec36", "role": "control-provenance", "href": "evidence/control-provenance.json", "bytes": 1580 }, { "id": "conditions", "label": "Recorded shared-Mac conditions and neutral warmups", "sha256": "2695c9cf973ccc2b1d041dbe49db3460ac0792c33677373bb5925d43383b08f9", "href": "evidence/conditions.json", "bytes": 63390 }, { "id": "study-materials", "label": "Audit bundle: exact tested candidate, frozen method, fresh corpus and all originals; not a qualified recipe", "sha256": "96488e5a4445fcb9d2820bd09a7c72d535728c2b5b0e0516bce5f630eae29849", "href": "evidence/study-materials.zip", "bytes": 10041927 } ] }, "analysisCompletedAt": "2026-10-14T16:47:52.685807+00:00", "uniqueQuestionClusters": 40, "repetitions": 3, "attemptsPerWorkflow": 120, "summaries": { "A1": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 3, "lower": 0.8666666666666667, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 0, "lower": 0.8666666666666667, "upper": 0.8666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1, "upper": 1 }, "citation_precision": { "passed": 38, "total": 42, "unresolved": 0, "lower": 0.9047619047619048, "upper": 0.9047619047619048 }, "unresolved_questions": 3, "explicit_page_citations": 38, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "latency": { "n": 40, "unavailable": 0, "p50": 3.4909559169900604, "p95": 5.010730540991062, "max": 5.4041547079978045, "errors_retained": 0, "timeouts_retained": 0, "boundary": null }, "answer_model_request_count": 77, "original_session_receipt": { "original_summary": { "executor_version": "kingy-comparison-session/1.0.0", "system": "A", "repetition": 1, "status": "complete", "qualificationGranted": false, "complete_query_deadline_seconds": 90, "generation_seed": 271828, "client_offline_profile": "offline.sb", "client_offline_profile_sha256": "e5462ef1daae3785012cb8e4fa05b89105bf4ac4d98be9580a8c357365a4f54f", "timing_boundary": "runner.ask entry through its complete returned answer, source resolution and sufficiency check; excludes prior PDF import/index, cold start, warmup and later grading.", "cache_policy": "cache_prompt false; no claim that OS file caches were flushed.", "freeze_sha256": "b0dde55a2f92f156f588d5df510832b2ea874e3da41e072ab096564269356d43", "schedule_sha256": "c091813a67fc28330171b15a0f735e40be6bb68f092c3e4afecd0cfc46c4aedb", "questions_sha256": "2c09fe76f96bfcfc59d06f3517c119d071b8555031200baefc1cd5d92f4d4486", "executor_sha256": "32a6c505904db80943cc747290f13476a9228805978e7ff3846a467a35953fd2", "runner_sha256": "abc2334bfb7b4727eee31f61f11da50512ac30ee2585d9974232ae2231276ef8", "model_sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5", "corpus_pdf_sha256": { "D01.pdf": "d3bc33ed0779da036c9e0ee668f8f7b7d0e08a04a0d3284c084b28a829f1eddf", "D02.pdf": "af1509878e48b6eff5b63e80d8646f9a8282116906d6d01f92d46059390bd0bd", "D03.pdf": "27fdbaa48ba3cc3f549e7cff0964e5007781abcaf64f46ba51092b70230fefd8", "D04.pdf": "0141d841f7f7dc20a419d5778da8cf7630b353f99a609b74a0934fefb333b327", "D05.pdf": "ca0b2c4070a1c17b8b07f968e795a36d73214bc6189b02612a1561e57e2b8447", "D06.pdf": "5c78ad081151899e120deaad556ef47b1c4331b8265b6e24d7e193827611f541", "D07.pdf": "9cce9d2ad80c9f764b58ba7d4b7dc5ed873eaf3f6d266e460605116beeeceb8e", "D08.pdf": "4c3c3ae7f21e0d2e1e9333a4399d2087f4ccc19344a43130bf6956d96bc52f94", "D09.pdf": "046e7515fe62f99fc580b695214c308d4c14c9625953904772814bd98864e6a2", "D10.pdf": "bb517485abfdae2f3c26d02833b87dc950e120760b795c75f0dadc4be8e65919" }, "planned_question_ids": [ "heldout-07", "heldout-33", "heldout-05", "heldout-39", "heldout-36", "heldout-29", "heldout-16", "heldout-13", "heldout-28", "heldout-22", "heldout-09", "heldout-35", "heldout-26", "heldout-14", "heldout-25", "heldout-21", "heldout-04", "heldout-02", "heldout-24", "heldout-38", "heldout-01", "heldout-40", "heldout-37", "heldout-34", "heldout-20", "heldout-31", "heldout-10", "heldout-30", "heldout-08", "heldout-11", "heldout-17", "heldout-23", "heldout-18", "heldout-32", "heldout-15", "heldout-03", "heldout-12", "heldout-19", "heldout-27", "heldout-06" ], "session_order": [ "A1", "B1", "B2", "A2", "A3", "B3" ], "artifact_verification_seconds": 1.8676444579905365, "runtime": { "name": "llama.cpp", "binary_sha256": "51afbf4ea6485b9ada19e99af97181f12baca42ef704b1e291c26a9ac0644eae", "version": { "command": [ "/opt/homebrew/Cellar/llama.cpp/0.6.0/bin/llama-server", "--version" ], "exit_code": 0, "stdout": "", "stderr": "version: 0.6.0 (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000334 for Darwin arm64" } }, "before": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 77%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 12288.00M used = 10897.69M free = 1390.31M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 66Gi 15% 453k 690M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "task_owned_server_pid": 70720, "cold_process_to_health_seconds": 1.8046461670019198, "ready_model_identity": { "models": [ { "name": "kingy-qwen3-4b", "model": "kingy-qwen3-4b", "modified_at": "", "size": "", "digest": "", "type": "model", "description": "", "tags": [ "" ], "capabilities": [ "completion" ], "parameters": "", "details": { "parent_model": "", "format": "gguf", "family": "", "families": [ "" ], "parameter_size": "", "quantization_level": "" } } ], "object": "list", "data": [ { "id": "kingy-qwen3-4b", "aliases": [ "kingy-qwen3-4b" ], "tags": [], "object": "model", "architecture": { "input_modalities": [ "text" ], "output_modalities": [ "text" ] }, "created": 1791986234, "owned_by": "llamacpp", "meta": { "vocab_type": 2, "n_vocab": 151936, "n_ctx": 8192, "n_ctx_train": 40960, "n_embd": 2560, "n_params": 4022468096, "size": 2491323904, "ftype": "Q4_K - Medium" } } ] }, "warmup_model_request_count": 1, "warmup_seconds": 0.08230595799977891, "parse_seconds": 0.03730483300751075, "index_seconds": 0.001503916981164366, "loaded_pdf_pages": 30, "retained_attempts": 40, "planned_attempts": 40, "unexecuted_question_ids": [], "unexecuted_attempt_count": 0, "model_request_count": 77, "session_total_model_request_count": 78, "attempt_errors": 0, "timeouts": 0, "after": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 76%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 13312.00M used = 11978.00M free = 1334.00M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 64Gi 16% 453k 675M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "session_wall_seconds": 147.06838687500567, "frozen_inputs_unchanged": true }, "integrity_issues": [], "source_sha256": { "session-plan.json": "037a85907c20e12eb01507d022268f948742152d1d829798bff4ed36f7e18a4c", "session-summary.json": "4e21d72691e62ed9e414f67662ffaf64f495274ef5c05af7f384ccd9db96faf8", "attempts.jsonl": "491f72553985584a2644e50bba0c93d612854154375b168a7f09d46aa0dca2d8" }, "complete_40_original_rows": true }, "provenance_verified": true }, "A2": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 3, "lower": 0.8666666666666667, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 0, "lower": 0.8666666666666667, "upper": 0.8666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1, "upper": 1 }, "citation_precision": { "passed": 38, "total": 42, "unresolved": 0, "lower": 0.9047619047619048, "upper": 0.9047619047619048 }, "unresolved_questions": 3, "explicit_page_citations": 38, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "latency": { "n": 40, "unavailable": 0, "p50": 2.4642572915181518, "p95": 3.7909144170116633, "max": 3.931772625015583, "errors_retained": 0, "timeouts_retained": 0, "boundary": null }, "answer_model_request_count": 77, "original_session_receipt": { "original_summary": { "executor_version": "kingy-comparison-session/1.0.0", "system": "A", "repetition": 2, "status": "complete", "qualificationGranted": false, "complete_query_deadline_seconds": 90, "generation_seed": 271828, "client_offline_profile": "offline.sb", "client_offline_profile_sha256": "e5462ef1daae3785012cb8e4fa05b89105bf4ac4d98be9580a8c357365a4f54f", "timing_boundary": "runner.ask entry through its complete returned answer, source resolution and sufficiency check; excludes prior PDF import/index, cold start, warmup and later grading.", "cache_policy": "cache_prompt false; no claim that OS file caches were flushed.", "freeze_sha256": "b0dde55a2f92f156f588d5df510832b2ea874e3da41e072ab096564269356d43", "schedule_sha256": "c091813a67fc28330171b15a0f735e40be6bb68f092c3e4afecd0cfc46c4aedb", "questions_sha256": "2c09fe76f96bfcfc59d06f3517c119d071b8555031200baefc1cd5d92f4d4486", "executor_sha256": "32a6c505904db80943cc747290f13476a9228805978e7ff3846a467a35953fd2", "runner_sha256": "abc2334bfb7b4727eee31f61f11da50512ac30ee2585d9974232ae2231276ef8", "model_sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5", "corpus_pdf_sha256": { "D01.pdf": "d3bc33ed0779da036c9e0ee668f8f7b7d0e08a04a0d3284c084b28a829f1eddf", "D02.pdf": "af1509878e48b6eff5b63e80d8646f9a8282116906d6d01f92d46059390bd0bd", "D03.pdf": "27fdbaa48ba3cc3f549e7cff0964e5007781abcaf64f46ba51092b70230fefd8", "D04.pdf": "0141d841f7f7dc20a419d5778da8cf7630b353f99a609b74a0934fefb333b327", "D05.pdf": "ca0b2c4070a1c17b8b07f968e795a36d73214bc6189b02612a1561e57e2b8447", "D06.pdf": "5c78ad081151899e120deaad556ef47b1c4331b8265b6e24d7e193827611f541", "D07.pdf": "9cce9d2ad80c9f764b58ba7d4b7dc5ed873eaf3f6d266e460605116beeeceb8e", "D08.pdf": "4c3c3ae7f21e0d2e1e9333a4399d2087f4ccc19344a43130bf6956d96bc52f94", "D09.pdf": "046e7515fe62f99fc580b695214c308d4c14c9625953904772814bd98864e6a2", "D10.pdf": "bb517485abfdae2f3c26d02833b87dc950e120760b795c75f0dadc4be8e65919" }, "planned_question_ids": [ "heldout-23", "heldout-30", "heldout-02", "heldout-07", "heldout-40", "heldout-20", "heldout-28", "heldout-27", "heldout-01", "heldout-12", "heldout-24", "heldout-26", "heldout-22", "heldout-31", "heldout-19", "heldout-32", "heldout-06", "heldout-36", "heldout-03", "heldout-04", "heldout-35", "heldout-15", "heldout-16", "heldout-11", "heldout-29", "heldout-18", "heldout-09", "heldout-17", "heldout-14", "heldout-21", "heldout-05", "heldout-10", "heldout-39", "heldout-08", "heldout-25", "heldout-34", "heldout-38", "heldout-13", "heldout-37", "heldout-33" ], "session_order": [ "A1", "B1", "B2", "A2", "A3", "B3" ], "artifact_verification_seconds": 2.126286499988055, "runtime": { "name": "llama.cpp", "binary_sha256": "51afbf4ea6485b9ada19e99af97181f12baca42ef704b1e291c26a9ac0644eae", "version": { "command": [ "/opt/homebrew/Cellar/llama.cpp/0.6.0/bin/llama-server", "--version" ], "exit_code": 0, "stdout": "", "stderr": "version: 0.6.0 (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000334 for Darwin arm64" } }, "before": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 73%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 11264.00M used = 10100.44M free = 1163.56M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 62Gi 16% 453k 655M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "task_owned_server_pid": 31373, "cold_process_to_health_seconds": 1.7789460830099415, "ready_model_identity": { "models": [ { "name": "kingy-qwen3-4b", "model": "kingy-qwen3-4b", "modified_at": "", "size": "", "digest": "", "type": "model", "description": "", "tags": [ "" ], "capabilities": [ "completion" ], "parameters": "", "details": { "parent_model": "", "format": "gguf", "family": "", "families": [ "" ], "parameter_size": "", "quantization_level": "" } } ], "object": "list", "data": [ { "id": "kingy-qwen3-4b", "aliases": [ "kingy-qwen3-4b" ], "tags": [], "object": "model", "architecture": { "input_modalities": [ "text" ], "output_modalities": [ "text" ] }, "created": 1791994790, "owned_by": "llamacpp", "meta": { "vocab_type": 2, "n_vocab": 151936, "n_ctx": 8192, "n_ctx_train": 40960, "n_embd": 2560, "n_params": 4022468096, "size": 2491323904, "ftype": "Q4_K - Medium" } } ] }, "warmup_model_request_count": 1, "warmup_seconds": 0.0725610830122605, "parse_seconds": 0.04199304198846221, "index_seconds": 0.0014008749858476222, "loaded_pdf_pages": 30, "retained_attempts": 40, "planned_attempts": 40, "unexecuted_question_ids": [], "unexecuted_attempt_count": 0, "model_request_count": 77, "session_total_model_request_count": 78, "attempt_errors": 0, "timeouts": 0, "after": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 76%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 11264.00M used = 9996.44M free = 1267.56M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 62Gi 16% 453k 655M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "session_wall_seconds": 111.70388641598402, "frozen_inputs_unchanged": true }, "integrity_issues": [], "source_sha256": { "session-plan.json": "0ab8dd4c512664d5ddd8702b9904cf844b9113c996421c661e201136225829e5", "session-summary.json": "a2dbb3eda9fd08d1dcaf0b6a538e2d1d8688d1a4fbaa6e16f7f880d0f7583204", "attempts.jsonl": "af6865ba20fc0694aa9e7453d670c9e800c5fc7235c4f6ed124a0b47a1384482" }, "complete_40_original_rows": true }, "provenance_verified": true }, "A3": { "attempts": 40, "answer_correctness": { "passed": 26, "total": 30, "unresolved": 3, "lower": 0.8666666666666667, "upper": 0.9666666666666667 }, "supported_answer_success": { "passed": 26, "total": 30, "unresolved": 0, "lower": 0.8666666666666667, "upper": 0.8666666666666667 }, "abstention_accuracy": { "passed": 10, "total": 10, "unresolved": 0, "lower": 1, "upper": 1 }, "citation_precision": { "passed": 38, "total": 42, "unresolved": 0, "lower": 0.9047619047619048, "upper": 0.9047619047619048 }, "unresolved_questions": 3, "explicit_page_citations": 38, "uniquely_quote_mapped_citations": 0, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": true }, "latency": { "n": 40, "unavailable": 0, "p50": 2.4685694164945744, "p95": 3.803498416004004, "max": 3.9323652909952216, "errors_retained": 0, "timeouts_retained": 0, "boundary": null }, "answer_model_request_count": 77, "original_session_receipt": { "original_summary": { "executor_version": "kingy-comparison-session/1.0.0", "system": "A", "repetition": 3, "status": "complete", "qualificationGranted": false, "complete_query_deadline_seconds": 90, "generation_seed": 271828, "client_offline_profile": "offline.sb", "client_offline_profile_sha256": "e5462ef1daae3785012cb8e4fa05b89105bf4ac4d98be9580a8c357365a4f54f", "timing_boundary": "runner.ask entry through its complete returned answer, source resolution and sufficiency check; excludes prior PDF import/index, cold start, warmup and later grading.", "cache_policy": "cache_prompt false; no claim that OS file caches were flushed.", "freeze_sha256": "b0dde55a2f92f156f588d5df510832b2ea874e3da41e072ab096564269356d43", "schedule_sha256": "c091813a67fc28330171b15a0f735e40be6bb68f092c3e4afecd0cfc46c4aedb", "questions_sha256": "2c09fe76f96bfcfc59d06f3517c119d071b8555031200baefc1cd5d92f4d4486", "executor_sha256": "32a6c505904db80943cc747290f13476a9228805978e7ff3846a467a35953fd2", "runner_sha256": "abc2334bfb7b4727eee31f61f11da50512ac30ee2585d9974232ae2231276ef8", "model_sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5", "corpus_pdf_sha256": { "D01.pdf": "d3bc33ed0779da036c9e0ee668f8f7b7d0e08a04a0d3284c084b28a829f1eddf", "D02.pdf": "af1509878e48b6eff5b63e80d8646f9a8282116906d6d01f92d46059390bd0bd", "D03.pdf": "27fdbaa48ba3cc3f549e7cff0964e5007781abcaf64f46ba51092b70230fefd8", "D04.pdf": "0141d841f7f7dc20a419d5778da8cf7630b353f99a609b74a0934fefb333b327", "D05.pdf": "ca0b2c4070a1c17b8b07f968e795a36d73214bc6189b02612a1561e57e2b8447", "D06.pdf": "5c78ad081151899e120deaad556ef47b1c4331b8265b6e24d7e193827611f541", "D07.pdf": "9cce9d2ad80c9f764b58ba7d4b7dc5ed873eaf3f6d266e460605116beeeceb8e", "D08.pdf": "4c3c3ae7f21e0d2e1e9333a4399d2087f4ccc19344a43130bf6956d96bc52f94", "D09.pdf": "046e7515fe62f99fc580b695214c308d4c14c9625953904772814bd98864e6a2", "D10.pdf": "bb517485abfdae2f3c26d02833b87dc950e120760b795c75f0dadc4be8e65919" }, "planned_question_ids": [ "heldout-18", "heldout-16", "heldout-06", "heldout-05", "heldout-37", "heldout-23", "heldout-28", "heldout-04", "heldout-26", "heldout-22", "heldout-34", "heldout-32", "heldout-13", "heldout-35", "heldout-07", "heldout-20", "heldout-24", "heldout-29", "heldout-02", "heldout-14", "heldout-39", "heldout-38", "heldout-09", "heldout-36", "heldout-17", "heldout-01", "heldout-25", "heldout-11", "heldout-19", "heldout-30", "heldout-03", "heldout-27", "heldout-10", "heldout-12", "heldout-15", "heldout-33", "heldout-08", "heldout-21", "heldout-40", "heldout-31" ], "session_order": [ "A1", "B1", "B2", "A2", "A3", "B3" ], "artifact_verification_seconds": 1.8626966669980902, "runtime": { "name": "llama.cpp", "binary_sha256": "51afbf4ea6485b9ada19e99af97181f12baca42ef704b1e291c26a9ac0644eae", "version": { "command": [ "/opt/homebrew/Cellar/llama.cpp/0.6.0/bin/llama-server", "--version" ], "exit_code": 0, "stdout": "", "stderr": "version: 0.6.0 (build 11429, commit d81235049)\nbuilt with AppleClang 21.0.0.21000334 for Darwin arm64" } }, "before": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 78%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 11264.00M used = 9988.44M free = 1275.56M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 62Gi 16% 453k 655M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "task_owned_server_pid": 31718, "cold_process_to_health_seconds": 1.5339827090210747, "ready_model_identity": { "models": [ { "name": "kingy-qwen3-4b", "model": "kingy-qwen3-4b", "modified_at": "", "size": "", "digest": "", "type": "model", "description": "", "tags": [ "" ], "capabilities": [ "completion" ], "parameters": "", "details": { "parent_model": "", "format": "gguf", "family": "", "families": [ "" ], "parameter_size": "", "quantization_level": "" } } ], "object": "list", "data": [ { "id": "kingy-qwen3-4b", "aliases": [ "kingy-qwen3-4b" ], "tags": [], "object": "model", "architecture": { "input_modalities": [ "text" ], "output_modalities": [ "text" ] }, "created": 1791994913, "owned_by": "llamacpp", "meta": { "vocab_type": 2, "n_vocab": 151936, "n_ctx": 8192, "n_ctx_train": 40960, "n_embd": 2560, "n_params": 4022468096, "size": 2491323904, "ftype": "Q4_K - Medium" } } ] }, "warmup_model_request_count": 1, "warmup_seconds": 0.06468349997885525, "parse_seconds": 0.03854408298502676, "index_seconds": 0.0013869999966118485, "loaded_pdf_pages": 30, "retained_attempts": 40, "planned_attempts": 40, "unexecuted_question_ids": [], "unexecuted_attempt_count": 0, "model_request_count": 77, "session_total_model_request_count": 78, "attempt_errors": 0, "timeouts": 0, "after": { "os": { "command": [ "sw_vers" ], "exit_code": 0, "stdout": "ProductName:\t\tmacOS\nProductVersion:\t\t26.3\nBuildVersion:\t\t25D125", "stderr": "" }, "hardware": { "command": [ "sysctl", "hw.model", "hw.memsize", "hw.physicalcpu", "hw.logicalcpu" ], "exit_code": 0, "stdout": "hw.model: Mac16,7\nhw.memsize: 25769803776\nhw.physicalcpu: 14\nhw.logicalcpu: 14", "stderr": "" }, "memory_pressure": { "command": [ "memory_pressure", "-Q" ], "exit_code": 0, "stdout": "The system has 25769803776 (1572864 pages with a page size of 16384).\nSystem-wide memory free percentage: 78%", "stderr": "" }, "swap": { "command": [ "sysctl", "-n", "vm.swapusage" ], "exit_code": 0, "stdout": "total = 11264.00M used = 9972.44M free = 1291.56M (encrypted)", "stderr": "" }, "disk": { "command": [ "df", "-h", "/" ], "exit_code": 0, "stdout": "Filesystem Size Used Avail Capacity iused ifree %iused Mounted on\n/dev/disk3s1s1 460Gi 11Gi 62Gi 16% 453k 655M 0% /", "stderr": "" }, "ac_power": { "command": [ "pmset", "-g", "batt" ], "exit_code": 0, "stdout": "Now drawing from 'AC Power'\n -InternalBattery-0 (id=22216803)\t100%; charged; 0:00 remaining present: true", "stderr": "" }, "power_mode": { "command": [ "pmset", "-g", "custom" ], "exit_code": 0, "stdout": "Battery Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 0\n networkoversleep 0\n sleep 0\n lessbright 1\n tcpkeepalive 1\n disksleep 0\n SleepServices 0\nAC Power:\n Sleep On Power Button 1\n powermode 0\n standby 0\n ttyskeepawake 1\n hibernatemode 3\n powernap 0\n hibernatefile /var/vm/sleepimage\n displaysleep 0\n womp 1\n networkoversleep 0\n sleep 0\n tcpkeepalive 1\n disksleep 0\n SleepServices 0", "stderr": "" }, "thermal": { "command": [ "pmset", "-g", "therm" ], "exit_code": 0, "stdout": "Note: No thermal warning level has been recorded\nNote: No performance warning level has been recorded\nNote: No CPU power status has been recorded", "stderr": "" }, "chip_inventory": [ { "chip_type": "Apple M4 Pro", "physical_memory": "24 GB", "number_processors": "proc 14:10:4", "machine_model": "Mac16,7", "machine_name": "MacBook Pro" } ] }, "session_wall_seconds": 111.27523666698835, "frozen_inputs_unchanged": true }, "integrity_issues": [], "source_sha256": { "session-plan.json": "1b6220d42c4de0474f226fe91f1b34d4eeb6b211ff14d1eb02886cf84be1fd7b", "session-summary.json": "3f1494cb868ffde59018f2e156c37211ab65ea1e53ee4b834ead0814e8de5b2e", "attempts.jsonl": "5ba5863fcbad4d898c64860d3a0945b3f669debca9a373d7178cfc90d62908bc" }, "complete_40_original_rows": true }, "provenance_verified": true }, "B1": { "attempts": 40, "answer_correctness": { "passed": 10, "total": 30, "unresolved": 13, "lower": 0.3333333333333333, "upper": 0.7666666666666667 }, "supported_answer_success": { "passed": 9, "total": 30, "unresolved": 11, "lower": 0.3, "upper": 0.6666666666666666 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 4, "lower": 0.6, "upper": 1 }, "citation_precision": { "passed": 38, "total": 48, "unresolved": 4, "lower": 0.7916666666666666, "upper": 0.875 }, "unresolved_questions": 17, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 38, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "latency": { "n": 40, "unavailable": 0, "p50": 6.445, "p95": 8.467, "max": 8.862, "errors_retained": 0, "timeouts_retained": 0, "boundary": "Native app submission to documented logger final-answer event; includes native preprocessing/retrieval/embedding/inference; excludes later title generation and automation/export." }, "answer_model_request_count": 40, "original_session_receipt": { "session_path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v16/sessions/B1", "logger": { "path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v16/sessions/B1/model-io.jsonl", "sha256": "155d20a8eb5e2debb65d4a4fe249f35c6883fc9d7f3d3a94c01d1b4046d4edb9", "parsed_events": 80, "malformed_lines": [] }, "integrity_issues": [], "source_sha256": { "session-plan.json": "8645d7e341f224b151d965422c945c7631c47cde0e1231d747590f5d66968a6a", "native-manifest.jsonl": "bd5e6773169e4f9da7895da59af4bcef261b74429bd9a9e2c13d3fc9e15d5df4", "pending.json": "38e0b9de817f645c4bec37c0d4a3e58baecccb040f5718dc069a72c7385a0bed" }, "retained_manifest_rows": 40, "unmanifested_pending_attempt": false, "complete_40_original_manifest_rows": true, "original_files_modified_by_analysis": false }, "provenance_verified": true }, "B2": { "attempts": 40, "answer_correctness": { "passed": 10, "total": 30, "unresolved": 13, "lower": 0.3333333333333333, "upper": 0.7666666666666667 }, "supported_answer_success": { "passed": 9, "total": 30, "unresolved": 11, "lower": 0.3, "upper": 0.6666666666666666 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 4, "lower": 0.6, "upper": 1 }, "citation_precision": { "passed": 38, "total": 48, "unresolved": 4, "lower": 0.7916666666666666, "upper": 0.875 }, "unresolved_questions": 17, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 38, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "latency": { "n": 40, "unavailable": 0, "p50": 6.2165, "p95": 7.408, "max": 7.703, "errors_retained": 0, "timeouts_retained": 0, "boundary": "Native app submission to documented logger final-answer event; includes native preprocessing/retrieval/embedding/inference; excludes later title generation and automation/export." }, "answer_model_request_count": 40, "original_session_receipt": { "session_path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v16/sessions/B2", "logger": { "path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v16/sessions/B2/model-io.jsonl", "sha256": "78e8d4e480ae9eb51d1f268a92c4ec497836c22186e0f11e6df1ebba31aebe90", "parsed_events": 80, "malformed_lines": [] }, "integrity_issues": [], "source_sha256": { "session-plan.json": "bfdc3cc6ca69a33529809b52851846424643e8bc6f9f8990a391b4462505eedf", "native-manifest.jsonl": "20e9e498d410f46e375e5989d6731e11989638c18868405e25f88af756d08228", "pending.json": "38e0b9de817f645c4bec37c0d4a3e58baecccb040f5718dc069a72c7385a0bed" }, "retained_manifest_rows": 40, "unmanifested_pending_attempt": false, "complete_40_original_manifest_rows": true, "original_files_modified_by_analysis": false }, "provenance_verified": true }, "B3": { "attempts": 40, "answer_correctness": { "passed": 10, "total": 30, "unresolved": 13, "lower": 0.3333333333333333, "upper": 0.7666666666666667 }, "supported_answer_success": { "passed": 9, "total": 30, "unresolved": 11, "lower": 0.3, "upper": 0.6666666666666666 }, "abstention_accuracy": { "passed": 6, "total": 10, "unresolved": 4, "lower": 0.6, "upper": 1 }, "citation_precision": { "passed": 38, "total": 48, "unresolved": 4, "lower": 0.7916666666666666, "upper": 0.875 }, "unresolved_questions": 17, "explicit_page_citations": 0, "uniquely_quote_mapped_citations": 38, "all_semantic_judgments_resolved": false, "qualified": false, "complete_planned_ids": true, "thresholds": { "supported_answer_success": 0.9, "citation_precision": 0.95, "abstention_accuracy": 0.9 }, "thresholds_passed_on_lower_bounds": { "supported_answer_success": false, "citation_precision": false, "abstention_accuracy": false }, "latency": { "n": 40, "unavailable": 0, "p50": 6.0075, "p95": 7.515, "max": 7.91, "errors_retained": 0, "timeouts_retained": 0, "boundary": "Native app submission to documented logger final-answer event; includes native preprocessing/retrieval/embedding/inference; excludes later title generation and automation/export." }, "answer_model_request_count": 40, "original_session_receipt": { "session_path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v16/sessions/B3", "logger": { "path": "/Users/curtispyke/Documents/ChatGPT/kingy Wiki/implementation/2026-10-13/local-ai-guide/evidence/comparison-v16/sessions/B3/model-io.jsonl", "sha256": "b6d53eb63bbf8c622c6967430cafe0f3b1fb029d78b1a028776fa353f3e7d8c3", "parsed_events": 80, "malformed_lines": [] }, "integrity_issues": [], "source_sha256": { "session-plan.json": "f8b11655bcda729fef32759f5c18fcca931f1dc6fb8a81f0242722551d118586", "native-manifest.jsonl": "e8426aee6f02dc6e2883e15bef728e2f1aaa4e9a2f2f6017eb33d0a8a455cb4d", "pending.json": "38e0b9de817f645c4bec37c0d4a3e58baecccb040f5718dc069a72c7385a0bed" }, "retained_manifest_rows": 40, "unmanifested_pending_attempt": false, "complete_40_original_manifest_rows": true, "original_files_modified_by_analysis": false }, "provenance_verified": true } }, "pairedEffects": { "complete_question_success": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.15, 0.525 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.025, 0.7 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "supported_answer_success": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.2, 0.5666666666666667 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0, 0.7666666666666667 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "canonical_fact_correctness": { "unique_questions": 30, "effect_bounds_A_minus_B": [ 0.1, 0.6333333333333333 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.1, 0.8 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "correct_abstention": { "unique_questions": 10, "effect_bounds_A_minus_B": [ 0, 0.4 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ 0, 0.7 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." }, "emitted_citation_precision": { "unique_questions": 40, "effect_bounds_A_minus_B": [ 0.029761904761904767, 0.11309523809523814 ], "cluster_bootstrap_95pct_sensitivity_envelope": [ -0.132449790045528, 0.29166666666666663 ], "bootstrap_draws": 10000, "usable_bootstrap_draws": 10000, "interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample." } }, "automaticWinner": null }