"""Build portable visible evidence; retain every historical attempt separately."""
from pathlib import Path
import json,shutil,hashlib,html,re
B=Path(__file__).resolve().parents[1];DEST=B/'evidence';DEST.mkdir(exist_ok=True)
def save(p,d):p.parent.mkdir(parents=True,exist_ok=True);p.write_text(json.dumps(d,indent=2,ensure_ascii=False))
def scrub(v):
if isinstance(v,dict):return {k:scrub(x)for k,x in v.items()if k not in {'reasoning','reasoning_details','reasoning_content','analysis','authorization','Authorization','key','api_key'}}
if isinstance(v,list):return [scrub(x)for x in v]
return v
def export_run(src,dest):
dest.mkdir(parents=True,exist_ok=True)
for name in ['result.json','conversation.json']:
p=src/name
if p.exists():save(dest/('visible-transcript.json'if name=='conversation.json'else name),scrub(json.loads(p.read_text())))
for stage in ['initial','revision']:
if(src/stage).exists():shutil.copytree(src/stage,dest/stage,dirs_exist_ok=True)
def index_folder(folder,title):
files=[p for p in folder.rglob('*')if p.is_file()and p.name!='index.html']
(folder/'index.html').write_text('
'+html.escape(title)+''+html.escape(title)+'
Initial and revised files are separate. Visible transcripts retain tool calls and responses and omit private reasoning. Historical attempts remain linked.
')
shutil.copytree(B/'kit/candidate',DEST/'inputs',dirs_exist_ok=True)
shutil.copytree(B/'kit/operator/answer-keys',DEST/'answer-keys',dirs_exist_ok=True)
for name in ['TRANSPARENCY.md','METHOD-CORRECTIONS.md']:
if(B/name).exists():shutil.copy(B/name,DEST/name)
for name in ['RUBRIC.md','REVISION-PROMPT.txt','EXECUTION-SETTINGS.md','EXPANSION.md','input-manifest.json','run-log.csv']:
if(B/'kit/operator'/name).exists():shutil.copy(B/'kit/operator'/name,DEST/name)
for p in (B/'reconciliation').glob('*'):
if p.is_file()and p.suffix in ['.md','.json','.csv']:shutil.copy(p,DEST/'reconciliation'/p.name)
(DEST/'method').mkdir(exist_ok=True)
for p in (B/'build').glob('*'):
if p.suffix in ['.py','.mjs']and p.name not in ['build_workbook.mjs']:shutil.copy(p,DEST/'method'/p.name)
shutil.copy(B/'completion/executing-runner-source.py',DEST/'method/completion-runner.py')
(DEST/'access').mkdir(exist_ok=True)
for p in (B/'kit/operator/access').glob('*.json'):
if p.name not in ['account-status.json','credits-status.json']:shutil.copy(p,DEST/'access'/p.name)
if(B/'completion/accounting-current.json').exists():shutil.copy(B/'completion/accounting-current.json',DEST/'completion-accounting.json')
if(B/'completion/integrity-report.json').exists():shutil.copy(B/'completion/integrity-report.json',DEST/'integrity-report.json')
for name in ['review-audit.json','review-final-note.md']:
if(B/'completion'/name).exists():shutil.copy(B/'completion'/name,DEST/name)
reviewfiles=list(B.glob('independent-review-*.json'))+[B/'completion/reviews.json']
reviews={}
if(B/'original-grades-validation.json').exists():shutil.copy(B/'original-grades-validation.json',DEST/'original-grades-validation.json')
for p in reviewfiles:
if p.exists():
shutil.copy(p,DEST/('completion-reviews.json'if p.parent.name=='completion'else p.name))
for r in json.loads(p.read_text()).get('reviews',[]):reviews[(r['run_id'],r['stage'])]=r
entries={};receipts={};runs=set()
for batch in ['execution','execution-added','completion']:
root=B/batch
if not(root/'ledger.json').exists():continue
ledger=json.loads((root/'ledger.json').read_text());receipts.update({batch+':'+k:v for k,v in ledger['calls'].items()})
for run in sorted((root/'runs').glob('*')):
if not run.is_dir():continue
rid=run.name;runs.add(rid);target=DEST/'runs'/rid;target.mkdir(parents=True,exist_ok=True)
archive=target/'attempts'/batch;export_run(run,archive)
result=json.loads((run/'result.json').read_text())if(run/'result.json').exists()else{'stages':{}}
for stage,info in result.get('stages',{}).items():
# A completion attempt is the latest declared attempt, even when unsuccessful.
if(run/stage).exists():
if(target/stage).exists():shutil.rmtree(target/stage)
shutil.copytree(run/stage,target/stage)
pattern=re.compile('^'+re.escape(rid)+r'-(?:retry\d+-)?'+stage+'-')
matched={k:v for k,v in ledger['calls'].items()if pattern.match(k)}
# Latest attempt group is the selected result; older groups remain in receipts.
groups={}
for k,v in matched.items():
prefix=k.split('-'+stage+'-',1)[0];groups.setdefault(prefix,[]).append(v)
selected=max(groups,key=lambda k:max(x.get('started','')for x in groups[k]))if groups else None
costs=groups.get(selected,[]);unknown=any('actual'not in v for v in costs)
row={'run_id':rid,'batch':batch,'stage':stage,'status':info['status'],'elapsed_min':info.get('elapsed_min'),'selected_attempt_prefix':selected,'actual_cost_usd':None if unknown else sum(float(v.get('actual',0))for v in costs),'known_receipts_usd':sum(float(v.get('actual',0))for v in costs),'cost_status':'unknown calls; subtotal only'if unknown else'receipted','unsettled_reserve_usd':sum(float(v['reserved'])for v in costs if'actual'not in v)}
review=reviews.get((rid,stage))
if review and info['status']in {'complete','turn_limit','output_truncated','timeout'}:
hashes=review.get('reviewed_file_sha256',review.get('reviewed_sha256',{}))
hash_ok=bool(hashes)and all((run/stage/'output'/name).is_file()and hashlib.sha256((run/stage/'output'/name).read_bytes()).hexdigest()==h for name,h in hashes.items())
if not hashes and info['status']=='output_truncated' and not(run/stage/'output').exists():hash_ok=True
if hash_ok:row.update(score=review.get('total'),outcome=review.get('outcome'),critical_failure=review.get('critical_failure'),reviewer='unblinded AI review',review_hashes_match=True)
entries[(rid,stage)]=row
# Per-stage transcript is needed when only the revision came from completion.
if(run/'conversation.json').exists():
messages=scrub(json.loads((run/'conversation.json').read_text()))
boundary=next((i for i,m in enumerate(messages[2:],2)if m.get('role')=='user'),len(messages))
if stage=='initial':messages=messages[:boundary]
save(target/(stage+'-visible-transcript.json'),messages)
own={k:v for k,v in ledger['calls'].items()if k.startswith(rid+'-')};save(archive/'cost-receipts.json',own)
responses=[]
for raw in sorted((root/'raw').glob(rid+'-*.json')):
data=json.loads(raw.read_text());responses.append({'file':raw.name,**{k:data.get(k)for k in ['id','model','provider','usage']},'finish_reasons':[x.get('finish_reason')for x in data.get('choices',[])]})
save(archive/'response-metadata.json',responses)
save(archive/'request-receipts.json',[json.loads(p.read_text())for p in sorted((root/'requests').glob(rid+'-*.json'))])
index_folder(archive,rid+' — '+batch)
for attempt in (root/'attempts').glob('*/*'):
if attempt.is_dir():
dest=DEST/'interrupted-attempts'/attempt.parent.name/attempt.name
for f in attempt.rglob('*'):
if f.is_file()and'workspace'not in f.relative_to(attempt).parts:
t=dest/f.relative_to(attempt);t.parent.mkdir(parents=True,exist_ok=True)
if f.suffix=='.json':save(t,scrub(json.loads(f.read_text())))
else:shutil.copy(f,t)
for rid in runs:
target=DEST/'runs'/rid
own={k:v for k,v in receipts.items()if k.split(':',1)[1].startswith(rid+'-')};save(target/'cost-receipts.json',own)
selected={stage:row for (r,stage),row in entries.items()if r==rid};save(target/'result.json',selected)
index_folder(target,rid)
entries=list(entries.values());save(DEST/'observed-results.json',entries);save(DEST/'cost-ledger.json',receipts)
rows=[]
for t in range(1,11):
for rep in [1,2]:
for model in ['qwen4','astra','sol','opus','fable']:
rid=f'T{t:02}-R{rep}-{model}'
for stage in ['initial','revision']:
e=next((x for x in entries if x['run_id']==rid and x['stage']==stage),None)
if not e:rows.append(f'| {rid} | {stage} | Not run | | | |
');continue
mins=f'{e["elapsed_min"]:.2f}'if e['elapsed_min']is not None else'not recorded'
trace=f'runs/{rid}/{stage}-visible-transcript.json';trace_link=f' · Cumulative trace through stage'if(DEST/trace).exists()else''
final=f'runs/{rid}/{stage}/final.txt';final_link=f' · Final'if(DEST/final).exists()else''
cost_text=f'{e["actual_cost_usd"]:.5f}'if e['actual_cost_usd']is not None else f'unknown; known subtotal {e["known_receipts_usd"]:.5f}'
rows.append(f'| {rid} | {stage} | {html.escape(e["status"])} | {mins} | {cost_text} | All files and attempts{final_link}{trace_link} · Cost AI review: {e.get("score","ungraded")} |
')
page='Kingy model test evidenceTen-task model comparison: evidence index
Every planned stage is shown, including failures. Synthetic fixtures, original and revised files, prior attempts, visible tool traces, independent unblinded AI grades and receipts are retained. A completed API response is not a passing grade. Times and charges in this table belong to the selected attempt; receipts include earlier attempts too. Extract the download before opening these links.
Rubric · Input hashes · Results · All receipts · Reconciliation · Method corrections
| Run | Stage | Status | Minutes | USD selected attempt | Evidence |
'+''.join(rows)+'
Inputs and reviews
'
for t in range(1,11):page+=f'- Task {t:02}: Prompt · Key
'
for p in sorted(DEST.glob('*review*.json')):page+=f'- {p.name}
'
(DEST/'index.html').write_text(page+'
')
print(json.dumps({'saved_stages':len(entries),'complete':sum(x['status']=='complete'for x in entries),'receipt_total':sum(float(v.get('actual',0))for v in receipts.values())}))