"""Generate the study guide from inspectable evidence, never invented results.""" from pathlib import Path from statistics import mean,median from collections import Counter import json,re B=Path(__file__).resolve().parents[1];E=B/'evidence';rows=json.loads((E/'observed-results.json').read_text());lookup={(r['run_id'],r['stage']):r for r in rows} reviews={} for p in list(B.glob('independent-review-*.json'))+[B/'completion/reviews.json']: if p.exists(): for r in json.loads(p.read_text()).get('reviews',[]):reviews[r['run_id'],r['stage']]=r names=['qwen4','astra','sol','opus','fable'];labels={'qwen4':'Qwen 4-Max','astra':'GPT-6 Astra','sol':'GPT-6.1 Sol','opus':'Claude Opus 5.5','fable':'Claude Fable 5.2'} tasks=json.loads((B/'build/task-descriptions.json').read_text());costs=json.loads((E/'cost-ledger.json').read_text());account=json.loads((B/'completion/accounting-current.json').read_text()) terminal={'complete','turn_limit','output_truncated','timeout'} finished=sum(r['status']in terminal for r in rows);complete=sum(r['status']=='complete'for r in rows);graded=sum(k in reviews for k,r in lookup.items()if r['status']in terminal) def current_review(rid,st): r=lookup.get((rid,st)) return reviews.get((rid,st))if r and r['status']in terminal and r.get('review_hashes_match') else None def usable(r):return r and sum(r['scores'])>=16 and min(r['scores'])>=3 and r['critical_failure']=='No' def outcome(r): if r['critical_failure']=='Yes'or sum(r['scores'])<12:return 'failed' if sum(r['scores'])<16 or min(r['scores'])<3:return 'major repair' return 'usable'if sum(r['scores'])==20 else'minor edits' def clean(s): s=re.sub(r'(?<=\d)(?=(?:orders|people|accounts|points|seats|copies)\b)', ' ', s) s=re.sub(r'\b(at|to|in|and|of|the)(?=\d)',r'\1 ',s) return s text=f'''# Qwen 4 versus Astra, Sol, Opus and Fable: ten practical jobs, every result exposed **Study status:** {finished}/200 stages have reached a reported outcome; {complete} returned a final response and {graded} have a source-based grade. {'The full matrix is evaluated.' if finished==200 and graded==200 else 'Execution and review are still in progress; do not present this version as the final comparison.'} We gave five models the same work: reconcile a messy month of sales, choose a laptop, repair a small website, recover project decisions, write from a source packet, diagnose an export problem, triage customer tickets, schedule a meeting, check invoice images and extract current requirements from a long archive. Each task was run twice. Every initial submission then received the same self-review request, without the answer key. The question was practical: **what can someone use, what needs fixing, and what did it cost to get there?** This is an inspection of returned work, not a claim that a model can run a company. Every business, customer, offer, invoice and software screenshot in the test is a constructed fixture. The tasks resemble real work while giving us known facts to check. [Open all 200 result rows and their files](evidence/index.html). [Download the exact candidate inputs](candidate-inputs.zip). [Read the grading rubric](evidence/RUBRIC.md). The downloadable guide-and-evidence bundle keeps these links together: extract it before opening the guide or evidence index. ## The comparison at a glance A result meets our usability threshold only if it scores at least 16/20, scores at least 3/4 on **each** criterion, and has no defined critical failure. “Minor edits” meets that threshold; “major repair” does not. Five strong totals cannot compensate for one missing essential deliverable. | Model | Initial usable / graded | Initial mean /20 | Revised usable / graded | Revised mean /20 | Median initial minutes | Receipted study cost, USD | |---|---:|---:|---:|---:|---:|---:| ''' for n in names: initial=[current_review(f'T{t:02}-R{rep}-{n}','initial')for t in range(1,11)for rep in [1,2]];initial=[r for r in initial if r] revised=[current_review(f'T{t:02}-R{rep}-{n}','revision')for t in range(1,11)for rep in [1,2]];revised=[r for r in revised if r] times=[r['elapsed_min']for r in rows if r['run_id'].endswith('-'+n)and r['stage']=='initial'and r['status']in terminal and r['elapsed_min']is not None] cost=sum(float(v.get('actual',0))for k,v in costs.items()if '-'+n+'-'in k) def avg(a):return f'{mean(r["total"]for r in a):.2f}'if a else'—' text+=f'| {labels[n]} | {sum(usable(r)for r in initial)}/{len(initial)} | {avg(initial)} | {sum(usable(r)for r in revised)}/{len(revised)} | {avg(revised)} | {median(times):.2f} | {cost:.4f} |\n' text+='\n### What the paired revisions changed\n\n' for n in names: pairs=[(current_review(f'T{t:02}-R{rep}-{n}','initial'),current_review(f'T{t:02}-R{rep}-{n}','revision'))for t in range(1,11)for rep in [1,2]] pairs=[(a,b)for a,b in pairs if a and b] if pairs: improved=sum(b['total']>a['total']for a,b in pairs);worse=sum(b['total']