"""Resume only untouched jobs; preserve interrupted jobs as separate attempts.""" import sys,json,shutil,concurrent.futures from pathlib import Path sys.argv.append('--added-tasks') import run_suite as s R=s.R jobs=[] for rep in [1,2]: for task in range(7,11): names=list(s.MODELS);shift=(task-1+(rep-1)*2)%5 for model in names[shift:]+names[:shift]: tag={'Qwen 4':'qwen4','Astra':'astra','Sol':'sol','Opus':'opus','Fable':'fable'}[model];rid=f'T{task:02}-R{rep}-{tag}';p=R/'runs'/rid if not p.exists():jobs.append((model,task,rep));continue result=json.loads((p/'result.json').read_text()) if(p/'result.json').exists()else None if result and result.get('stages',{}).get('initial',{}).get('status')=='stopped: Global accounting halt': dest=R/'attempts'/'never-dispatched'/rid;dest.parent.mkdir(parents=True,exist_ok=True);shutil.move(str(p),dest);jobs.append((model,task,rep)) elif rid=='T10-R1-sol' and result and 'Connection reset' in result.get('stages',{}).get('initial',{}).get('status',''): dest=R/'attempts'/'connection-reset'/rid;dest.parent.mkdir(parents=True,exist_ok=True);shutil.move(str(p),dest) # Preserve old request identities, costs and unknown reserve. New calls use retry ID. jobs.append((model,task,rep)) original_call=s.call def call(model,messages,ident): if ident.startswith('T10-R1-sol-'):ident=ident.replace('T10-R1-sol-','T10-R1-sol-retry1-',1) return original_call(model,messages,ident) s.call=call s.save(R/'resume-note.json',{'reason':'Connection reset on Sol T10; reserve retained. Untouched jobs resume; Sol gets one fresh infrastructure retry. No model-output failure reruns.','jobs':len(jobs),'time':s.now()}) with concurrent.futures.ThreadPoolExecutor(max_workers=3)as pool: fs=[pool.submit(s.run,*job)for job in jobs] for f in concurrent.futures.as_completed(fs): try:f.result() except Exception as e:print('RESUME ERROR',str(e),flush=True)