"""Integrity and completeness checks for publication artifacts; no API calls.""" from pathlib import Path from collections import Counter import hashlib,json,re B=Path(__file__).resolve().parents[1];E=B/'evidence' rows=json.loads((E/'observed-results.json').read_text());reviews={} for p in list(B.glob('independent-review-*.json'))+[B/'completion/reviews.json']: if p.exists(): for r in json.loads(p.read_text()).get('reviews',[]):reviews[r['run_id'],r['stage']]=r keys=[(r['run_id'],r['stage'])for r in rows] if len(keys)!=len(set(keys)):raise RuntimeError('Duplicate result rows') terminal={'complete','turn_limit','output_truncated','timeout'};errors=[];missing=[];ungraded=[];hashes=0 for task in range(1,11): for rep in [1,2]: for model in ['qwen4','astra','sol','opus','fable']: rid=f'T{task:02}-R{rep}-{model}' for stage in ['initial','revision']: row=next((r for r in rows if r['run_id']==rid and r['stage']==stage),None) if not row or row['status']not in terminal:missing.append([rid,stage,row['status']if row else'no result']);continue review=reviews.get((rid,stage)) if not review:ungraded.append([rid,stage]);continue scores=review['scores'];total=sum(scores) if review.get('critical_failure')not in ['Yes','No']:errors.append([rid,stage,'invalid critical flag']) for c in review.get('criterion_evidence',[]): idx=c.get('criterion') if isinstance(idx,int)and 'score'in c and c['score']!=scores[idx-1]:errors.append([rid,stage,'criterion evidence score mismatch',idx]) if len(scores)!=5 or total!=review['total']or any(not isinstance(x,int)or x<0 or x>4 for x in scores):errors.append([rid,stage,'score arithmetic']) outcome='failed'if review['critical_failure']=='Yes'or total<12 else'major repair'if total<16 or min(scores)<3 else'usable'if total==20 else'minor edits' if outcome!=review['outcome']:errors.append([rid,stage,'outcome',outcome,review['outcome']]) recorded=review.get('reviewed_file_sha256',review.get('reviewed_sha256',{})) if not recorded and not(row['status']=='output_truncated'and not(E/'runs'/rid/stage/'output').exists()):errors.append([rid,stage,'no artifact hashes']) if not row.get('review_hashes_match'):errors.append([rid,stage,'grade not bound to selected artifacts']) for name,h in recorded.items(): out=E/'runs'/rid/stage/'output'/name if not out.exists()or hashlib.sha256(out.read_bytes()).hexdigest()!=h:errors.append([rid,stage,'hash',name]) hashes+=1 for name,h in review.get('external_operator_evidence_sha256',review.get('operator_evidence_sha256',{})).items(): candidates=[E/'runs'/rid/stage/name, B/row['batch']/'runs'/rid/stage/name, B/name] if not any(q.is_file()and hashlib.sha256(q.read_bytes()).hexdigest()==h for q in candidates):errors.append([rid,stage,'operator evidence hash',name]) links=[] for p in E.rglob('*.html'): for href in re.findall(r'href="([^"]+)"',p.read_text()): if not href.startswith(('http','#'))and not(p.parent/href).exists():links.append([str(p.relative_to(E)),href]) # Check the guide's own local targets and worksheet evidence folders too. for target in re.findall(r'\[[^\]]+\]\(([^)]+)\)',(B/'article-working.md').read_text()): if target.startswith(('http','#')):continue if target=='candidate-inputs.zip':continue if not(B/target.split('#')[0]).exists():links.append(['article-working.md',target]) for row in rows: if not(E/'runs'/row['run_id']/'index.html').exists():links.append(['workbook',row['run_id']]) # Verify public JSON traces omit private reasoning and credential fields. for p in E.rglob('*transcript.json'): def walk(v): if isinstance(v,dict): for k,x in v.items(): if k in {'reasoning','reasoning_details','reasoning_content','authorization','Authorization','api_key'}:errors.append([str(p),'private field',k]) walk(x) elif isinstance(v,list): for x in v:walk(x) walk(json.loads(p.read_text())) # Verify all final responses are real terminal outputs, not filter/empty returns. finish_issues=[] for p in (B/'completion/raw').glob('*.json'): if p.name.endswith('-partial.json'):continue d=json.loads(p.read_text());c=d['choices'][0] if c.get('finish_reason')not in {'stop','tool_calls','length'}:finish_issues.append([p.name,c.get('finish_reason')]) if c.get('finish_reason')=='stop'and not(c['message'].get('content')or'').strip():finish_issues.append([p.name,'empty stop']) # Every newly started initial must receive the same first system/user packet. input_packet_issues=[] for p in (B/'completion/runs').glob('*/conversation.json'): rid=p.parent.name;task=rid[1:3] prior=next((x for root in ['execution','execution-added'] for x in (B/root/'runs').glob(f'T{task}-*/conversation.json')),None) if prior: new=json.loads(p.read_text())[:2];old=json.loads(prior.read_text())[:2] if new!=old:input_packet_issues.append(rid) errors.extend([rid,'initial input packet differs from original task']for rid in input_packet_issues) report={'planned_stages':200,'saved_stages':len(rows),'terminal_outcomes':sum(r['status']in terminal for r in rows),'execution_statuses':dict(Counter(r['status']for r in rows)),'missing_outcomes':missing,'ungraded_outcomes':ungraded,'verified_review_hashes':hashes,'errors':errors,'broken_evidence_links':links,'unexpected_finish_reasons':finish_issues,'complete':not(missing or ungraded or errors or links or finish_issues)} (B/'completion/integrity-report.json').write_text(json.dumps(report,indent=2));print(json.dumps({k:v for k,v in report.items()if k not in ['missing_outcomes','ungraded_outcomes']},indent=2))