{
"schemaVersion": "kingy-document-paired-report/1.1.0",
"studyId": "document-comparison-v14-20261014.1",
"measuredAt": "2026-10-14T00:40:47.967298+00:00",
"scope": {
"uniqueQuestions": 40,
"repetitions": 1,
"plannedRepetitions": 3,
"completedPairedRepetitions": 1,
"status": "incomplete-paired-pilot",
"answerableQuestions": 30,
"unanswerableQuestions": 10,
"description": "Ten independently authored synthetic English PDFs,30 pages;20 direct factual,10 cross-document and10 unanswerable questions. Corpus and method frozen before first question access; no post-test source tuning."
},
"incompleteStudy": {
"reason": "The computer-control service crashed during B2 and later returned inconsistent accessibility and screenshot states. Seven B2 responses are retained. A replacement run stopped after neutral warmup, before any document question. B3 was not executed. Causation and LM Studio product reliability remain unknown. Native stop and publication scope were recorded before answer-key access.",
"evidenceId": "stop-decision",
"stoppedBeforeAnswerKeyAccess": true
},
"metricDefinitions": {
"factCorrectness": "Canonical required-value checks across30 answerable questions, separately from citations. The frozen matcher can undercredit semantically correct concise native answers; supplementary source review must be consulted. Unresolved cases remain in denominator with explicit bounds.",
"supportCorrectness": "Complete required facts plus actually emitted citations across30 answerable questions. Every requested field must be covered. Errors, missing essential fields, unsupported assertions and timeouts within complete runs fail; unresolved cases retain their denominator. Literal-wording limitations are disclosed.",
"citationCorrectness": "Grounded emitted citation instances that support the associated required claim divided by all emitted citations, including unsupported or unresolved claims. Displayed native source cards count only when claimed in answer text. A uniquely quote-mapped native citation can earn full credit; an explicit page label is separately reported.",
"abstention": "Correctly declining each of10 unanswerable questions, with no invented facts. Valid native prose is allowed; exact labelledUNKNOWN is not the only possible decline. Supplementary source review addresses explanatory prose and unresolved assertions."
},
"arms": [
{
"id": "kingy",
"name": "Kingy workflow 1.4",
"configuration": {
"machine": {
"chip": "M4 Pro",
"cpuCores": 14,
"gpuCores": 20,
"memoryGiB": 24,
"os": "macOS 26.3 build 25D125",
"power": "AC, automatic power mode",
"environment": "Shared machine; conditions and swap retained. Other active jobs were not stopped."
},
"model": {
"id": "Qwen3-4B",
"quantization": "Q4_K_M",
"bytes": 2497280256,
"sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5"
},
"context": 8192,
"maxOutputTokens": 512,
"temperature": 0,
"generationSeed": 271828,
"completeQueryDeadlineSeconds": 90,
"thinking": false,
"PDFs": "same ten byte-identical PDFs,30 pages",
"firstMessageOnly": true,
"workflow": "1.4.0",
"runtime": "llama.cpp0.6.0b11429 d81235049",
"retrieval": "BM25 pages plus generic field/clause scoring; source-ID selection then source-sufficiency inference; no embedding service",
"requestsPerQuestion": 2,
"runnerSha256": "00d0f68dfa240c7ce937be94ade50172c5051cb243ad50b2740e1b292ea5f65c",
"load": "CPU threads 10, fullMetal, flashattention, F16 KV, parallel 1",
"timer": {
"n": 40,
"p50": 4.496676228998695,
"p95": 6.2484013339853846,
"max": 8.307981291989563,
"boundary": "Kingy ask entry through complete returned answer including retrieval, selection and sufficiency check; excludes prior PDF import/index, cold start and warmup."
},
"warmup": "Neutral READY, no documents/questions before measurement",
"coldStartAndPDFIndex": "Recorded separately, excluded from answer timer"
},
"configurationEvidenceId": "conditions",
"rawEvidenceId": "kingy-raw",
"repetitions": [
{
"id": "A1",
"attempts": {
"planned": 40,
"recorded": 40,
"missing": 0,
"errors": 0,
"timeouts": 0
},
"factCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 1,
"bounds": [
0.9333333333333333,
0.9666666666666667
]
},
"supportCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 0,
"bounds": [
0.9333333333333333,
0.9333333333333333
]
},
"citationCorrectness": {
"passed": 37,
"total": 38,
"unresolved": 0,
"bounds": [
0.9736842105263158,
0.9736842105263158
]
},
"abstention": {
"passed": 10,
"total": 10,
"unresolved": 0,
"bounds": [
1,
1
]
}
}
],
"failures": [
{
"id": "v14-heldout-cross-02",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "kingy-raw"
},
{
"id": "v14-heldout-cross-10",
"description": "Frozen primary status fail; factual status unresolved; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "kingy-raw"
}
]
},
{
"id": "native",
"name": "Native LM Studio PDF chat",
"configuration": {
"machine": {
"chip": "M4 Pro",
"cpuCores": 14,
"gpuCores": 20,
"memoryGiB": 24,
"os": "macOS 26.3 build 25D125",
"power": "AC, automatic power mode",
"environment": "Shared machine; conditions and swap retained. Other active jobs were not stopped."
},
"model": {
"id": "Qwen3-4B",
"quantization": "Q4_K_M",
"bytes": 2497280256,
"sha256": "7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5"
},
"context": 8192,
"maxOutputTokens": 512,
"temperature": 0,
"generationSeed": 271828,
"completeQueryDeadlineSeconds": 90,
"thinking": false,
"PDFs": "same ten byte-identical PDFs,30 pages",
"firstMessageOnly": true,
"app": "LM Studio 0.4.26+4 official notarized installer",
"engine": "llama.cpp-mac-arm64-apple-metal-advsimd2.56.0;0.6.0-dev1commit8bb3cf0",
"retrieval": "native PDF attachments and unmodified installed rag-v1 revision 8",
"retrievalLimit": 8,
"affinityThreshold": 0.5,
"embedding": "bundled Nomic Embed Textv1.5Q4_K_M,84,106,624bytes",
"requestsPerAnswer": 1,
"actualEffectiveSettings": {
"load_config": {
"fields": [
{
"key": "llm.load.llama.autoFit",
"value": false
},
{
"key": "llm.load.mlx.autoFit",
"value": false
},
{
"key": "llm.load.llama.cpuThreadPoolSize",
"value": 10
},
{
"key": "llm.load.contextLength",
"value": 8192
},
{
"key": "llm.load.llama.flashAttention",
"value": true
},
{
"key": "llm.load.promptTemplate",
"value": {
"type": "jinja",
"jinjaPromptTemplate": {
"template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for index in range(ns.last_query_index, -1, -1) %}\n {%- set message = messages[index] %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not('' in message.content and '' in message.content) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in message.content %}\n {%- set content = message.content.split('')[-1].lstrip('\\n') %}\n {%- set reasoning_content = message.content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}"
}
}
},
{
"key": "llm.load.llama.tryDirectIO",
"value": false
},
{
"key": "llm.load.llama.acceleration.offloadRatio",
"value": 1
},
{
"key": "llm.load.yuzu.autoFit",
"value": false
},
{
"key": "llm.load.vllm.autoFit",
"value": false
},
{
"key": "llm.load.numParallelSessions",
"value": 1
},
{
"key": "llm.load.seed",
"value": {
"checked": true,
"value": 271828
}
}
]
},
"prediction_config": {
"fields": [
{
"key": "llm.prediction.promptTemplate",
"value": {
"type": "jinja",
"jinjaPromptTemplate": {
"template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for index in range(ns.last_query_index, -1, -1) %}\n {%- set message = messages[index] %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not('' in message.content and '' in message.content) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in message.content %}\n {%- set content = message.content.split('')[-1].lstrip('\\n') %}\n {%- set reasoning_content = message.content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}"
},
"stopStrings": []
}
},
{
"key": "llm.prediction.llama.cpuThreads",
"value": 10
},
{
"key": "llm.prediction.repeatPenalty",
"value": {
"checked": true,
"value": 1
}
},
{
"key": "llm.prediction.topPSampling",
"value": {
"checked": true,
"value": 1
}
},
{
"key": "llm.prediction.minPSampling",
"value": {
"checked": true,
"value": 0
}
},
{
"key": "llm.prediction.temperature",
"value": 0
},
{
"key": "llm.prediction.maxPredictedTokens",
"value": {
"checked": true,
"value": 512
}
},
{
"key": "llm.prediction.reasoning.enableThinking",
"value": false
},
{
"key": "llm.prediction.contextOverflowPolicy",
"value": "stopAtLimit"
},
{
"key": "llm.prediction.tools",
"value": {
"type": "none"
}
}
]
},
"plugin_configs": {
"lmstudio/rag-v1": {
"preset": "",
"config": {
"fields": [
{
"key": "retrievalLimit",
"value": 8
}
]
}
}
},
"plugins": [
"lmstudio/rag-v1"
],
"system_prompt": ""
},
"load": "CPU threads 10, fullMetal, flashattention, F16 KV, parallel 1, seeded GUIreload",
"timer": {
"n": 40,
"p50": 13.6965,
"p95": 20.771,
"max": 26.497,
"boundary": "Native app submission to documented logger final-answer event; includes native preprocessing/retrieval/embedding/inference; excludes later title generation and automation/export."
},
"warmup": "Neutral READY through official SDK before document questions",
"extraTitleOutputs": "Retained separately; no best-response selection",
"runtimeEgressEnforcement": "unverified; no comparative privacy claim"
},
"configurationEvidenceId": "native-settings",
"rawEvidenceId": "native-raw",
"repetitions": [
{
"id": "B1",
"attempts": {
"planned": 40,
"recorded": 40,
"missing": 0,
"errors": 0,
"timeouts": 0
},
"factCorrectness": {
"passed": 16,
"total": 30,
"unresolved": 6,
"bounds": [
0.5333333333333333,
0.7333333333333333
]
},
"supportCorrectness": {
"passed": 16,
"total": 30,
"unresolved": 2,
"bounds": [
0.5333333333333333,
0.6
]
},
"citationCorrectness": {
"passed": 35,
"total": 40,
"unresolved": 1,
"bounds": [
0.875,
0.9
]
},
"abstention": {
"passed": 9,
"total": 10,
"unresolved": 1,
"bounds": [
0.9,
1
]
}
}
],
"failures": [
{
"id": "v14-heldout-unknown-09",
"description": "Frozen primary status unresolved; factual status None; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-cross-09",
"description": "Frozen primary status unresolved; factual status unresolved; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-16",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-cross-02",
"description": "Frozen primary status fail; factual status unresolved; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-cross-06",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-cross-05",
"description": "Frozen primary status fail; factual status unresolved; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-04",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-cross-04",
"description": "Frozen primary status fail; factual status unresolved; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-01",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-20",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-cross-10",
"description": "Frozen primary status fail; factual status unresolved; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-08",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-11",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-17",
"description": "Frozen primary status fail; factual status fail; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
},
{
"id": "v14-heldout-doc-06",
"description": "Frozen primary status unresolved; factual status unresolved; source-based review may expose concise-answer false negatives. See full answer, field and citation checks.",
"evidenceId": "native-raw"
}
]
}
],
"interruptedNative": {
"armId": "native",
"repetitionId": "B2",
"recordedAttempts": 7,
"excludedFromPairedMetrics": true,
"qualityClassification": "instrumentation-interruption",
"reason": "All seven completed original outputs and supplemental grades are downloadable.33planned questions were not executed. These seven are not pooled into A1/B1, and the unrun questions are not scored as native quality failures.",
"evidenceId": "native-interrupted"
},
"uncertainty": {
"method": "Frozen 10,000-draw whole-question paired case-resampling sensitivity with arithmetic unresolved lower/upper bounds.",
"unit": "40 matched unique questions;3 repetitions planned, one native pair complete. A2/A3 are Kingy repeatability checks only.",
"notes": [
"Case-resampling envelopes are sensitivity analyses on this fixed corpus, not population confidence intervals or p-values.",
"Repeated attempts on one Mac are not independent hardware reproduction or120independent cases.",
"The frozen primary automatic metric has concise-answer false negatives. It cannot alone establish an accuracy winner; supplementary source review is separate and not blinded.",
"All raw responses, failures, uncertain judgments, interruptions and timing boundaries remain public."
],
"pairedEffects": {
"complete_question_success": {
"unique_questions": 40,
"effect_bounds_A_minus_B": [
0.25,
0.325
],
"cluster_bootstrap_95pct_sensitivity_envelope": [
0.125,
0.475
],
"bootstrap_draws": 10000,
"usable_bootstrap_draws": 10000,
"interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample."
},
"supported_answer_success": {
"unique_questions": 30,
"effect_bounds_A_minus_B": [
0.3333333333333333,
0.4
],
"cluster_bootstrap_95pct_sensitivity_envelope": [
0.16666666666666666,
0.5666666666666667
],
"bootstrap_draws": 10000,
"usable_bootstrap_draws": 10000,
"interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample."
},
"canonical_fact_correctness": {
"unique_questions": 30,
"effect_bounds_A_minus_B": [
0.2,
0.43333333333333335
],
"cluster_bootstrap_95pct_sensitivity_envelope": [
0,
0.6
],
"bootstrap_draws": 10000,
"usable_bootstrap_draws": 10000,
"interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample."
},
"correct_abstention": {
"unique_questions": 10,
"effect_bounds_A_minus_B": [
0,
0.1
],
"cluster_bootstrap_95pct_sensitivity_envelope": [
0,
0.3
],
"bootstrap_draws": 10000,
"usable_bootstrap_draws": 10000,
"interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample."
},
"emitted_citation_precision": {
"unique_questions": 40,
"effect_bounds_A_minus_B": [
0.0736842105263158,
0.09868421052631582
],
"cluster_bootstrap_95pct_sensitivity_envelope": [
0,
0.18928841991341988
],
"bootstrap_draws": 10000,
"usable_bootstrap_draws": 10000,
"interpretation": "Paired whole-question case-resampling sensitivity on this fixed corpus; not a p-value, population confidence claim or independent-repeat sample."
}
}
},
"qualification": {
"armId": "kingy",
"includedInPairedMetrics": false,
"protocolEvidenceId": "scoring",
"rawEvidenceId": "kingy-qualification",
"assessmentEvidenceId": "summaries",
"repetitions": [
{
"id": "A1",
"attempts": {
"planned": 40,
"recorded": 40,
"missing": 0,
"errors": 0,
"timeouts": 0
},
"factCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 1,
"bounds": [
0.9333333333333333,
0.9666666666666667
]
},
"supportCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 0,
"bounds": [
0.9333333333333333,
0.9333333333333333
]
},
"citationCorrectness": {
"passed": 37,
"total": 38,
"unresolved": 0,
"bounds": [
0.9736842105263158,
0.9736842105263158
]
},
"abstention": {
"passed": 10,
"total": 10,
"unresolved": 0,
"bounds": [
1,
1
]
}
},
{
"id": "A2",
"attempts": {
"planned": 40,
"recorded": 40,
"missing": 0,
"errors": 0,
"timeouts": 0
},
"factCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 1,
"bounds": [
0.9333333333333333,
0.9666666666666667
]
},
"supportCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 0,
"bounds": [
0.9333333333333333,
0.9333333333333333
]
},
"citationCorrectness": {
"passed": 37,
"total": 38,
"unresolved": 0,
"bounds": [
0.9736842105263158,
0.9736842105263158
]
},
"abstention": {
"passed": 10,
"total": 10,
"unresolved": 0,
"bounds": [
1,
1
]
}
},
{
"id": "A3",
"attempts": {
"planned": 40,
"recorded": 40,
"missing": 0,
"errors": 0,
"timeouts": 0
},
"factCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 1,
"bounds": [
0.9333333333333333,
0.9666666666666667
]
},
"supportCorrectness": {
"passed": 28,
"total": 30,
"unresolved": 0,
"bounds": [
0.9333333333333333,
0.9333333333333333
]
},
"citationCorrectness": {
"passed": 37,
"total": 38,
"unresolved": 0,
"bounds": [
0.9736842105263158,
0.9736842105263158
]
},
"abstention": {
"passed": 10,
"total": 10,
"unresolved": 0,
"bounds": [
1,
1
]
}
}
]
},
"findings": [
{
"claim": "The two consumed v1.3 failure cases passed all six targeted development attempts after the repair.",
"scope": "These are repair regressions, not fresh held-out accuracy evidence. Two different cross-document failures remain in the new corpus.",
"evidenceIds": [
"repairs",
"kingy-qualification"
]
},
{
"claim": "The source-based review found 28/30 fully supported Kingy answers and 25–26/30 native answers in this completed pair.",
"scope": "A descriptive result on this fixed corpus with a supplementary, unblinded review. The automatic16 vs 28gap materially overstates the difference. This does not prove general or repeatable accuracy superiority.",
"evidenceIds": [
"source-review",
"kingy-raw",
"native-raw"
]
},
{
"claim": "Kingy returned complete answers sooner on all 40questions in the completed A1/B1 pair: median 4.50 seconds versus 13.70 seconds.",
"scope": "Descriptive post-submission workflow time on this shared Mac and exact configurations. Includes incorrect/unresolved answers. Native time includes its preprocessing/embedding; no model-throughput, novice speed or repeatability advantage is established.",
"evidenceIds": [
"latency",
"conditions"
]
},
{
"claim": "Kingy returned explicit document/page/quote citations. Native citations were credited by mapping their source quotes to PDFs.",
"scope": "The frozen common rubric recorded 37 supported explicit-page citations for Kingy and 35 supported quote-mapped native citations, plus one unresolved native citation. This establishes the recorded output format, not general citation superiority.",
"evidenceIds": [
"kingy-raw",
"native-raw",
"summaries"
]
},
{
"claim": "The fresh ZIP passed four installation smoke tests and denied tested outbound IPv4, IPv6 and wrong-loopback-port connections; local health and authentication behaved as declared.",
"scope": "Owned Mac, declared prerequisites already installed, existing model reused after byte verification. No novice-user or comparative privacy claim.",
"evidenceIds": [
"installation",
"recipe"
]
}
],
"limitations": [
"One complete paired repetition out of three planned; full native repeatability remains unmeasured.",
"Forty synthetic English document questions,30 pages, one physical shared Mac, at most two factual fields; no real private-document/user study.",
"Same model bytes and generation limits, different native runtimes and complete retrieval pipelines. LM Studio retrieval limit 8 was selected before the fresh bank using consumed controls; default 3 was not used as an artificially weak baseline.",
"Concise native answers can be penalized by literal canonical matching even when the source supports their meaning. Do not equate frozen automatic counts with an unbiased semantic accuracy ranking.",
"No measured comparative ease of installation, general privacy, monetary cost, hardware-upgrade benefit, cloud quality or model throughput.",
"No direct page labels in the captured native citation output is an observed format property of this version/configuration, not a claim that LM Studio never exposes source navigation.",
"Immutable v1.3 results and all earlier failed candidates are preserved. New tuning would require a new unseen corpus."
],
"protocolEvidenceId": "protocol",
"scoringEvidenceId": "scoring",
"artifacts": [
{
"id": "kingy-raw",
"label": "A1 complete Kingy responses and frozen grades",
"sha256": "761e172bb61a30c75c44a4af709d335cbe9190fa57ecb39892deccb7d17f50bf",
"href": "evidence/kingy-raw.jsonl",
"bytes": 453152
},
{
"id": "native-raw",
"label": "B1 complete native responses and frozen grades",
"sha256": "151a723458075d295158b8bc6d16bc65b3915b4b91f503ccd751132bf0283ec1",
"href": "evidence/native-raw.jsonl",
"bytes": 7530196
},
{
"id": "native-interrupted",
"label": "All seven interrupted native responses and grades, excluded from paired comparison",
"sha256": "5e0562d5617c851ca93291e515636849ff91f9a3211efb3e83bbf1a561464bcf",
"href": "evidence/native-interrupted.jsonl",
"bytes": 1321657
},
{
"id": "protocol",
"label": "Original frozen three-pair comparison plan",
"sha256": "da6e147a013780c1b901d97b209443f5881a98291d0bce96133a2ed2056b4534",
"href": "evidence/protocol.md",
"bytes": 8832
},
{
"id": "scoring",
"label": "Frozen common scoring rules",
"sha256": "0b65c8e0bb5dec8e5cce33aff926d7ed7585e42ee230ccad7d30401b422b58af",
"href": "evidence/scoring.md",
"bytes": 12662
},
{
"id": "freeze",
"label": "Before-bank study freeze",
"sha256": "02c45667fa4c99f91cde61788350f0669a66d2c78cac3986f9bf0d0e0bc3e1e3",
"href": "evidence/freeze.json",
"bytes": 14521
},
{
"id": "stop-decision",
"label": "Before-grading native stop and scope decision",
"sha256": "7f705bbda0fb605a93231dd3b88670f9fc81e7047a2f15406e07d926dc12991d",
"href": "evidence/stop-decision.json",
"bytes": 1426
},
{
"id": "paired",
"label": "Frozen A1/B1 analysis and unresolved bounds",
"sha256": "28c0eb1df26333457da7eee621aa2797d5d7a2f355ae11d3a797dd91868c7cc6",
"href": "evidence/paired.json",
"bytes": 36098
},
{
"id": "summaries",
"label": "All retained run summaries",
"sha256": "f25014f8cf0382e9ce1add347a4dc2377947595b7e3534352b834c0b00d60d03",
"href": "evidence/summaries.json",
"bytes": 5648
},
{
"id": "latency",
"label": "All40 paired answer-time observations and boundaries",
"sha256": "f16be6c10f2b8791df5bff32dc6b1c4176bfad9c64b0dabe159bae5c85937709",
"href": "evidence/latency.json",
"bytes": 8037
},
{
"id": "installation",
"label": "Fresh ZIP installation, four self-tests and offline checks",
"sha256": "dbf89af7ebf1e0413a11f42542b4992906e2f8d1ebe7aaa3cb9f05b79ff023e3",
"href": "evidence/installation.json",
"bytes": 2182
},
{
"id": "repairs",
"label": "Six targeted repair passes and retained development audit",
"sha256": "4cb3c8c107636f313b030abe1881f5b5f52c966b7fe62d5620b1f6c19d6b1c12",
"href": "evidence/repairs.json",
"bytes": 20105
},
{
"id": "native-settings",
"label": "Actual native effective load, prediction and retrieval settings",
"sha256": "a714dcdcbce3575c87810d33d29cd340d065a9a180358a43d282612a02527782",
"href": "evidence/native-settings.json",
"bytes": 11338
},
{
"id": "native-runtime",
"label": "Installed native engine and binary hashes",
"sha256": "d2f3a8e469022f4a1dd2b78a7ac685d21baa53afa8b6c334e9b190455a354d76",
"href": "evidence/native-runtime.json",
"bytes": 3121
},
{
"id": "native-rag",
"label": "Installed stock RAG and embedding hashes",
"sha256": "c46211b91224f40b122b8140db0d2dd1b12f164ffbbe6047050d4da9f850a0dc",
"href": "evidence/native-rag.json",
"bytes": 1268
},
{
"id": "grading-access",
"label": "Frozen inputs and first-reference grading receipt",
"sha256": "5749bd5a9afca3b98fcc7594fdd8f792b11e3efe272d5b4da682755696e48a99",
"href": "evidence/grading-access.json",
"bytes": 584
},
{
"id": "kingy-qualification",
"label": "Kingy A1/A2/A3 complete raw responses and common grades",
"sha256": "8115eca4806a1e65a7a0d36101abe053e7821369d94afaef9418aeb782b767e8",
"href": "evidence/kingy-qualification.jsonl",
"bytes": 1359504
},
{
"id": "methods",
"label": "Byte-verified frozen scoring, capture, executor, UI helpers and controls",
"sha256": "7c7f8c9ecdf6c69a42fec851e90ef762eb2e2e95827e076489525764f6384b58",
"href": "evidence/methods.zip",
"bytes": 52080
},
{
"id": "recipe",
"label": "Workflow1.4, public corpus, references and installation smoke tests",
"sha256": "9882cc00670d07ee6aa6fd03645d20a1e8697c68a5ceafa2f7040d4724068edb",
"href": "evidence/recipe.zip",
"bytes": 8405507
},
{
"id": "conditions",
"label": "Recorded machine, session setup and environment conditions",
"sha256": "34694ab485d7ca140c12d05632bb570660dfd9b50d0e736749db5a93817b1847",
"href": "evidence/conditions.json",
"bytes": 50879
},
{
"id": "source-review",
"label": "Supplementary source-based audit, all 17 case judgments and matcher false negatives",
"sha256": "bc7b192b517a381fe3f6291f448784a0dc35dcd3c92bb31fd0d9ed2d59a82283",
"href": "evidence/source-review.json",
"bytes": 128905
}
],
"semanticReview": {
"evidenceId": "source-review",
"method": "Separate evidence review of all 17 A1/B1 non-pass or unresolved cases against source PDFs and required facts. Conducted after outputs, not blinded to system, by a review agent in the same project. It is not independent hardware reproduction or a replacement frozen scorer.",
"reason": "The frozen literal canonical matcher undercredited seven semantically correct concise native answers and two unresolved native supported answers. The frozen scores remain unchanged below. The supplementary review resolves an explanatory native absence correctly, and identifies Kingy's fact-unresolved exposure-dose answer as a definite wrong date. One native scope-amendment citation remains ambiguous.",
"rows": [
{
"workflow": "Kingy A1",
"supported": "28/30",
"factual": "28/30",
"abstention": "10/10",
"citations": "37/38"
},
{
"workflow": "Native LM Studio B1",
"supported": "25–26/30 (one source-support ambiguity)",
"factual": "26/30",
"abstention": "10/10",
"citations": "36–37/40 (one source-support ambiguity)"
}
]
}
}