"""Independent deterministic output checks. These are evidence, not a prose judge.""" from pathlib import Path import csv,json,re,hashlib,datetime B=Path(__file__).resolve().parents[1];R=B/'execution';K=B/'kit' def csvrows(p):return list(csv.DictReader(p.open())) def audit(folder,stage): out=folder/stage/'output';checks=[] def add(name,ok,detail):checks.append({'check':name,'pass':None if ok is None else bool(ok),'detail':detail}) tid=folder.name[:3] expected={'T01':['cleaned-sales.csv','cleaned-refunds.csv','cleaned-ads.csv','exceptions.csv','summary.md'],'T02':['recommendation.md','comparison.csv'],'T03':['site/index.html','site/server.py','site/app.js','site/style.css','handoff.md'],'T04':['action-plan.csv','decision-log.md','update-email.txt'],'T05':['article.md','metadata.json','editor-notes.md'],'T06':['diagnosis.md'],'T07':['triage.csv','replies.md'],'T08':['schedule.csv','risk-note.md','invitation-draft.txt'],'T09':['invoices.json','review-note.md'],'T10':['requirements.csv','handoff.md']}[tid] for f in expected:add('file:'+f,(out/f).is_file(),f) for p in out.rglob('*.csv'): try: rows=csvrows(p);add('csv_parse:'+p.name,bool(rows),{'rows':len(rows),'columns':list(rows[0])if rows else []}) except Exception as e:add('csv_parse:'+p.name,False,str(e)) for p in out.rglob('*.json'): try:json.loads(p.read_text());add('json_parse:'+p.name,True,'valid JSON') except Exception as e:add('json_parse:'+p.name,False,str(e)) limits={'T02':{'recommendation.md':650},'T04':{'update-email.txt':220},'T06':{'diagnosis.md':500},'T08':{'risk-note.md':500},'T09':{'review-note.md':400},'T10':{'handoff.md':900}} for name,maxwords in limits.get(tid,{}).items(): if(out/name).exists(): n=len(re.findall(r'\b[\w]+(?:[’\x27-][\w]+)*\b',(out/name).read_text()));add('word_limit:'+name,True if n<=maxwords else None,{'words':n,'max':maxwords,'method':'regex words; human judgment around boundary'}) if tid=='T01': # Identify included rows from each model's own explicit status column, not row position. for name,idkey,expected_ids in [('cleaned-sales.csv','order_id',{'N1001','N1002','N1003','N1005','N1006','N1008','N1010','N1011','N1012'}),('cleaned-refunds.csv','refund_id',{'RF01','RF02'}),('cleaned-ads.csv','ad_id',{'AD01','AD02','AD03','AD04','AD05'})]: p=out/name if not p.exists():continue rows=csvrows(p);fields=list(rows[0])if rows else[];flag=next((x for x in fields if x.lower()in ['included','included_in_october','include_in_october','included_october','include','is_included','applied']),None) if flag: included=[x for x in rows if x[flag].lower()in ['yes','true','1','included']];actual={x.get(idkey)for x in included};add('included_ids:'+name,actual==expected_ids,{'flag':flag,'actual':sorted(str(x)for x in actual),'expected':sorted(expected_ids)}) else:checks.append({'check':'included_ids:'+name,'pass':None,'detail':{'manual_review':'No recognized inclusion field; included-only output is allowed','fields':fields}}) p=out/'summary.md' if p.exists(): txt=p.read_text();add('provisional_word_present',bool(re.search('provisional',txt,re.I)),'keyword only; inspect actual claims manually') if tid=='T05': p=out/'metadata.json' if p.exists(): try: d=json.loads(p.read_text());add('headline_length',50<=len(d.get('headline',''))<=70,len(d.get('headline','')));add('description_length',140<=len(d.get('search_description',''))<=160,len(d.get('search_description','')));add('excerpt_length',len(d.get('excerpt','').split())<=45,len(d.get('excerpt','').split())) except Exception:pass if(out/'article.md').exists(): n=len(re.findall(r'\b[\w]+(?:[’\x27-][\w]+)*\b',(out/'article.md').read_text()));add('article_word_count',600<=n<=800,n) report={'run':folder.name,'stage':stage,'checks':checks,'note':'Deterministic checks only. Requires source-based manual scoring; keyword hits do not prove factual accuracy.'} save=folder/stage/'operator-audit.json';save.write_text(json.dumps(report,indent=2));return report if __name__=='__main__': reports=[] for root in [R,B/'execution-added',B/'completion']: for f in sorted((root/'runs').glob('*')): if not(f/'result.json').exists():continue result=json.loads((f/'result.json').read_text()) for stage in result['stages']:reports.append(audit(f,stage)) print(json.dumps([{'run':r['run'],'stage':r['stage'],'failed_checks':[x['check']for x in r['checks']if x['pass'] is False],'manual_checks':[x['check']for x in r['checks']if x['pass'] is None]}for r in reports],indent=2))