"""Build portable visible evidence; retain every historical attempt separately.""" from pathlib import Path import json,shutil,hashlib,html,re B=Path(__file__).resolve().parents[1];DEST=B/'evidence';DEST.mkdir(exist_ok=True) def save(p,d):p.parent.mkdir(parents=True,exist_ok=True);p.write_text(json.dumps(d,indent=2,ensure_ascii=False)) def scrub(v): if isinstance(v,dict):return {k:scrub(x)for k,x in v.items()if k not in {'reasoning','reasoning_details','reasoning_content','analysis','authorization','Authorization','key','api_key'}} if isinstance(v,list):return [scrub(x)for x in v] return v def export_run(src,dest): dest.mkdir(parents=True,exist_ok=True) for name in ['result.json','conversation.json']: p=src/name if p.exists():save(dest/('visible-transcript.json'if name=='conversation.json'else name),scrub(json.loads(p.read_text()))) for stage in ['initial','revision']: if(src/stage).exists():shutil.copytree(src/stage,dest/stage,dirs_exist_ok=True) def index_folder(folder,title): files=[p for p in folder.rglob('*')if p.is_file()and p.name!='index.html'] (folder/'index.html').write_text(''+html.escape(title)+'

'+html.escape(title)+'

Initial and revised files are separate. Visible transcripts retain tool calls and responses and omit private reasoning. Historical attempts remain linked.

') shutil.copytree(B/'kit/candidate',DEST/'inputs',dirs_exist_ok=True) shutil.copytree(B/'kit/operator/answer-keys',DEST/'answer-keys',dirs_exist_ok=True) for name in ['TRANSPARENCY.md','METHOD-CORRECTIONS.md']: if(B/name).exists():shutil.copy(B/name,DEST/name) for name in ['RUBRIC.md','REVISION-PROMPT.txt','EXECUTION-SETTINGS.md','EXPANSION.md','input-manifest.json','run-log.csv']: if(B/'kit/operator'/name).exists():shutil.copy(B/'kit/operator'/name,DEST/name) for p in (B/'reconciliation').glob('*'): if p.is_file()and p.suffix in ['.md','.json','.csv']:shutil.copy(p,DEST/'reconciliation'/p.name) (DEST/'method').mkdir(exist_ok=True) for p in (B/'build').glob('*'): if p.suffix in ['.py','.mjs']and p.name not in ['build_workbook.mjs']:shutil.copy(p,DEST/'method'/p.name) shutil.copy(B/'completion/executing-runner-source.py',DEST/'method/completion-runner.py') (DEST/'access').mkdir(exist_ok=True) for p in (B/'kit/operator/access').glob('*.json'): if p.name not in ['account-status.json','credits-status.json']:shutil.copy(p,DEST/'access'/p.name) if(B/'completion/accounting-current.json').exists():shutil.copy(B/'completion/accounting-current.json',DEST/'completion-accounting.json') if(B/'completion/integrity-report.json').exists():shutil.copy(B/'completion/integrity-report.json',DEST/'integrity-report.json') for name in ['review-audit.json','review-final-note.md']: if(B/'completion'/name).exists():shutil.copy(B/'completion'/name,DEST/name) reviewfiles=list(B.glob('independent-review-*.json'))+[B/'completion/reviews.json'] reviews={} if(B/'original-grades-validation.json').exists():shutil.copy(B/'original-grades-validation.json',DEST/'original-grades-validation.json') for p in reviewfiles: if p.exists(): shutil.copy(p,DEST/('completion-reviews.json'if p.parent.name=='completion'else p.name)) for r in json.loads(p.read_text()).get('reviews',[]):reviews[(r['run_id'],r['stage'])]=r entries={};receipts={};runs=set() for batch in ['execution','execution-added','completion']: root=B/batch if not(root/'ledger.json').exists():continue ledger=json.loads((root/'ledger.json').read_text());receipts.update({batch+':'+k:v for k,v in ledger['calls'].items()}) for run in sorted((root/'runs').glob('*')): if not run.is_dir():continue rid=run.name;runs.add(rid);target=DEST/'runs'/rid;target.mkdir(parents=True,exist_ok=True) archive=target/'attempts'/batch;export_run(run,archive) result=json.loads((run/'result.json').read_text())if(run/'result.json').exists()else{'stages':{}} for stage,info in result.get('stages',{}).items(): # A completion attempt is the latest declared attempt, even when unsuccessful. if(run/stage).exists(): if(target/stage).exists():shutil.rmtree(target/stage) shutil.copytree(run/stage,target/stage) pattern=re.compile('^'+re.escape(rid)+r'-(?:retry\d+-)?'+stage+'-') matched={k:v for k,v in ledger['calls'].items()if pattern.match(k)} # Latest attempt group is the selected result; older groups remain in receipts. groups={} for k,v in matched.items(): prefix=k.split('-'+stage+'-',1)[0];groups.setdefault(prefix,[]).append(v) selected=max(groups,key=lambda k:max(x.get('started','')for x in groups[k]))if groups else None costs=groups.get(selected,[]);unknown=any('actual'not in v for v in costs) row={'run_id':rid,'batch':batch,'stage':stage,'status':info['status'],'elapsed_min':info.get('elapsed_min'),'selected_attempt_prefix':selected,'actual_cost_usd':None if unknown else sum(float(v.get('actual',0))for v in costs),'known_receipts_usd':sum(float(v.get('actual',0))for v in costs),'cost_status':'unknown calls; subtotal only'if unknown else'receipted','unsettled_reserve_usd':sum(float(v['reserved'])for v in costs if'actual'not in v)} review=reviews.get((rid,stage)) if review and info['status']in {'complete','turn_limit','output_truncated','timeout'}: hashes=review.get('reviewed_file_sha256',review.get('reviewed_sha256',{})) hash_ok=bool(hashes)and all((run/stage/'output'/name).is_file()and hashlib.sha256((run/stage/'output'/name).read_bytes()).hexdigest()==h for name,h in hashes.items()) if not hashes and info['status']=='output_truncated' and not(run/stage/'output').exists():hash_ok=True if hash_ok:row.update(score=review.get('total'),outcome=review.get('outcome'),critical_failure=review.get('critical_failure'),reviewer='unblinded AI review',review_hashes_match=True) entries[(rid,stage)]=row # Per-stage transcript is needed when only the revision came from completion. if(run/'conversation.json').exists(): messages=scrub(json.loads((run/'conversation.json').read_text())) boundary=next((i for i,m in enumerate(messages[2:],2)if m.get('role')=='user'),len(messages)) if stage=='initial':messages=messages[:boundary] save(target/(stage+'-visible-transcript.json'),messages) own={k:v for k,v in ledger['calls'].items()if k.startswith(rid+'-')};save(archive/'cost-receipts.json',own) responses=[] for raw in sorted((root/'raw').glob(rid+'-*.json')): data=json.loads(raw.read_text());responses.append({'file':raw.name,**{k:data.get(k)for k in ['id','model','provider','usage']},'finish_reasons':[x.get('finish_reason')for x in data.get('choices',[])]}) save(archive/'response-metadata.json',responses) save(archive/'request-receipts.json',[json.loads(p.read_text())for p in sorted((root/'requests').glob(rid+'-*.json'))]) index_folder(archive,rid+' — '+batch) for attempt in (root/'attempts').glob('*/*'): if attempt.is_dir(): dest=DEST/'interrupted-attempts'/attempt.parent.name/attempt.name for f in attempt.rglob('*'): if f.is_file()and'workspace'not in f.relative_to(attempt).parts: t=dest/f.relative_to(attempt);t.parent.mkdir(parents=True,exist_ok=True) if f.suffix=='.json':save(t,scrub(json.loads(f.read_text()))) else:shutil.copy(f,t) for rid in runs: target=DEST/'runs'/rid own={k:v for k,v in receipts.items()if k.split(':',1)[1].startswith(rid+'-')};save(target/'cost-receipts.json',own) selected={stage:row for (r,stage),row in entries.items()if r==rid};save(target/'result.json',selected) index_folder(target,rid) entries=list(entries.values());save(DEST/'observed-results.json',entries);save(DEST/'cost-ledger.json',receipts) rows=[] for t in range(1,11): for rep in [1,2]: for model in ['qwen4','astra','sol','opus','fable']: rid=f'T{t:02}-R{rep}-{model}' for stage in ['initial','revision']: e=next((x for x in entries if x['run_id']==rid and x['stage']==stage),None) if not e:rows.append(f'{rid}{stage}Not run');continue mins=f'{e["elapsed_min"]:.2f}'if e['elapsed_min']is not None else'not recorded' trace=f'runs/{rid}/{stage}-visible-transcript.json';trace_link=f' · Cumulative trace through stage'if(DEST/trace).exists()else'' final=f'runs/{rid}/{stage}/final.txt';final_link=f' · Final'if(DEST/final).exists()else'' cost_text=f'{e["actual_cost_usd"]:.5f}'if e['actual_cost_usd']is not None else f'unknown; known subtotal {e["known_receipts_usd"]:.5f}' rows.append(f'{rid}{stage}{html.escape(e["status"])}{mins}{cost_text}All files and attempts{final_link}{trace_link} · Cost
AI review: {e.get("score","ungraded")}') page='Kingy model test evidence

Ten-task model comparison: evidence index

Every planned stage is shown, including failures. Synthetic fixtures, original and revised files, prior attempts, visible tool traces, independent unblinded AI grades and receipts are retained. A completed API response is not a passing grade. Times and charges in this table belong to the selected attempt; receipts include earlier attempts too. Extract the download before opening these links.

Rubric · Input hashes · Results · All receipts · Reconciliation · Method corrections

'+''.join(rows)+'
RunStageStatusMinutesUSD selected attemptEvidence

Inputs and reviews

') print(json.dumps({'saved_stages':len(entries),'complete':sum(x['status']=='complete'for x in entries),'receipt_total':sum(float(v.get('actual',0))for v in receipts.values())}))