from pathlib import Path
import csv, json, textwrap, hashlib, zipfile
from decimal import Decimal
BASE = Path(__file__).resolve().parents[1]
KIT = BASE / 'kit'
C = KIT / 'candidate'
O = KIT / 'operator'
def put(path, text):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(textwrap.dedent(text).strip() + '\n')
def table(path, headers, rows):
path.parent.mkdir(parents=True, exist_ok=True)
with path.open('w', newline='') as f:
w = csv.writer(f); w.writerow(headers); w.writerows(rows)
put(KIT/'START-HERE.md', '''
# Qwen 4 real-world work test kit — version 1.0
Prepared November 15, 2026 (America/Vancouver). No model runs have been performed.
All business records, people, products, companies, prices and source documents in
the exercises are synthetic. The tasks simulate real work; they are not actual
Kingy financial records or product recommendations. Task 6 images are staged UI
screenshots, not evidence from a commercial product.
## Run one assignment
1. Read operator/PROTOCOL.md. Choose the exact endpoint/version for each model
family: Qwen 4, Astra, Sol, Opus, Fable. No endpoint is preselected or claimed
available. Record access and settings before running. This package makes no API calls.
2. Create a fresh isolated workspace containing ONLY candidate/COMMON.md and ONE
candidate/task-NN/ directory. Keep operator/, build/, grading files, other models'
outputs and answer keys outside its filesystem and context. A zip alone is not
a security boundary if the agent can browse the host.
3. Attach COMMON.md and that task's PROMPT.md plus its inputs. Paste the prompt
exactly; use the same files for every model. Do not expose any answer key.
4. Save the full response, generated files, timing and tool trace. Stop the timer
at completion or at the common 30-minute limit. Grade the original before revision.
5. Send operator/REVISION-PROMPT.txt verbatim in the same session, allowing 15 more
minutes. Save and grade separately. Do not supply hidden answers or custom hints.
6. Use operator/grading.xlsx (Results, Rubric, Setup) and run-log.csv. Repeat in a
fresh workspace/session for repetition 2. Planned count: 6 tasks × 5 models × 2
repetitions = 60 initial runs, with up to 60 revision stages.
## What to open
- candidate/task-01: messy sales, refunds and ad exports.
- candidate/task-02: fixed purchase research packet with conflicting offers.
- candidate/task-03: runnable broken website (Python standard library, no npm install).
- candidate/task-04: emails, notes and a change request.
- candidate/task-05: fictional AI-release source packet and editorial brief.
- candidate/task-06: three UI screenshots and short product help.
- operator/answer-keys/: private ground truth for each assignment.
- operator/RUBRIC.md: exact scoring and critical failures.
- operator/run-order.csv: deterministic, rotated run order.
## Packages
candidate-inputs.zip contains only model-facing material. full-operator-kit.zip
contains the complete kit including answers and grading files; never give it to a model.
Task 2 is a controlled research-synthesis test. Optional live-web research is described
separately and must not be pooled with its scores.
''')
put(C/'COMMON.md', '''
# Shared assignment instructions
Complete the assignment using the supplied files. All entities and source packets
are fictional test fixtures. Treat the exercise's dates, policies and source versions
as authoritative within this scenario, not as claims about real products.
Use only the supplied inputs, local execution tools and installed software. No web
research, paid services, API calls, external uploads or package downloads. Cite input
filenames and section headings or record IDs for factual claims. Do not fabricate
missing evidence, actions, test results or access. State any limitation precisely.
You have 30 minutes for the initial submission. Make reasonable progress without
asking questions; record assumptions where the inputs do not resolve a point.
Write final deliverables into an output/ directory (or provide clearly named file
attachments if your interface cannot write files). Preserve raw input files. For task
3, modify a working copy of site/ and include that copy in output/.
End with a concise handoff listing output filenames, checks actually performed,
remaining limitations and estimated minutes a human would need to finish. Do not
treat your estimate as a measured result. Do not include hidden chain-of-thought;
provide only useful decisions, calculations and evidence.
''')
prompts = [
'''# Task 1 — Reconcile a small business's October results
Northline Workshops needs October 2026 results in CAD. Read inputs/policy.md and the
three raw CSVs. Produce cleaned-sales.csv, cleaned-refunds.csv, cleaned-ads.csv,
exceptions.csv and summary.md in output/. Preserve source row IDs in cleaned files
and explain exclusions in exceptions.csv. Normalize valid dates to YYYY-MM-DD and
money to two decimal places; preserve unresolved values without guessing.
Report paid order count, gross sales, applied refunds, net revenue, advertising spend,
contribution after advertising and net revenue per paid order. Show Workshop, Course
and combined totals. Reconcile included and excluded records. Give a practical
recommendation, distinguishing this contribution measure from business profit.
''',
'''# Task 2 — Recommend a laptop from a conflicting source packet
Choose a laptop for a Vancouver freelance editor using ONLY inputs/. Budget is CAD
2,200 including the scenario's 12% sales tax and shipping. Hard requirements: new,
at least 32 GB RAM and 1 TB SSD, Windows 11, discrete GPU with at least 8 GB VRAM,
and confirmed delivery by October 23, 2026. The purchase date is October 19, 2026.
The editor uses Premiere Pro for 4K footage and accepts a heavier machine.
Produce recommendation.md (up to 650 words) and comparison.csv. Compare every
listed offer, calculate landed totals, identify hard failures/unknowns, cite exact
filenames and sections, and recommend one qualifying offer plus a fallback approach.
Do not invent benchmark results or treat listing errors as verified specifications.
''',
'''# Task 3 — Repair a workshop signup website
Read inputs/brief.md. Repair the working copy of site/ so the lead form stores valid
signups through the supplied server, the price calculation matches the brief, and the
page works on mobile. Keep the existing content and visual identity. Do not add a
framework, dependencies, external services or tracking.
Deliver output/site/ with all files needed to run it, plus output/handoff.md describing
the fixes and checks you actually performed. Include evidence for desktop and mobile,
valid/invalid form submissions and the supplied calculator examples. Do not claim a
browser check if you did not perform one.
''',
'''# Task 4 — Turn conflicting project documents into an action plan
You are coordinating the Field Notes webinar as of October 20, 2026, 10:00 Pacific.
Read all inputs. Produce action-plan.csv with columns action_id,action,owner,due,
status,dependencies,source; a decision-log.md; and an update-email.txt of at most
220 words addressed to the project team. Draft only; do not send anything.
Resolve superseded instructions using explicit approvals and dates. Separate agreed
decisions from proposals. Include unresolved ownership, scheduling and consent issues
without inventing answers. Give a realistic sequence of next steps.
''',
'''# Task 5 — Write an evidence-led AI release article
Prepare an editorial package about the fictional LumenDesk 2.0 launch using inputs/.
Deliver article.md (600–800 words excluding metadata), metadata.json (headline,
excerpt, search_description) and editor-notes.md. Read the editorial brief and voice
sample. Attribute performance claims, cite filenames/sections, reconcile corrections,
and distinguish announced access from future plans. Do not claim personal testing.
The article should help a small business decide whether to try the product now.
Keep unsupported source claims out of the article and explain them in editor notes.
Use a headline of 50–70 characters, excerpt at most 45 words and search description
of 140–160 characters. No publication, images or external research is required.
''',
'''# Task 6 — Diagnose an export failure from screenshots
Read inputs/operator-message.txt, inputs/product-help.md and the three PNG screenshots.
Produce diagnosis.md of at most 500 words. Identify the immediate blocker and any
secondary warning. Give numbered steps using the exact visible control labels and
explain what changes after the immediate fix. Separate visible evidence from anything
that still needs checking. Include a short reply suitable to paste to the operator.
You are interpreting screenshots, not operating the application. Do not claim to have
clicked, changed, exported, inspected a hidden panel or verified a successful export.
'''
]
for n,p in enumerate(prompts,1): put(C/f'task-{n:02}'/'PROMPT.md',p)
# Task 1: realistic raw exports with small, independently checkable ledger.
t=C/'task-01'/'inputs'
put(t/'policy.md', '''
# Northline Workshops accounting note — October 2026
All amounts are CAD, tax excluded. Reporting period: October 1–31 inclusive.
Revenue is paid order gross in this exercise, less settled refunds whose settlement
date falls in October; every refund here relates to an October order. Do not subtract
pending refunds. Contribution after advertising = net revenue − ad spend. No COGS,
payment fees, salaries or overhead are supplied, so do not call this net profit.
Order ID identifies an order. Exact duplicate rows can be collapsed. For conflicting
rows with the same order ID, retain the latest updated_at record; do not count both.
Only status paid is revenue; cancelled and pending are excluded. Refund ID and ad ID
are similarly unique. A settled refund requires a valid paid order and a nonmissing
amount; otherwise flag it and exclude it from the calculable total.
Dates in slash form are DD/MM/YYYY; ISO dates and named English month dates are also
valid. Empty values are unknown, never zero. Currency strings can include CAD or $.
Ad channel and product names should ignore surrounding spaces and letter case.
Unmapped products must be flagged rather than assigned. Workshop and Course are the
only products. October ads are all attributable to the product named in each row.
Refunds cannot exceed paid gross; flag if encountered. Missing refund amounts mean
the resulting net revenue is provisional even if a calculable total can be given.
''')
sales=[
['S01','N1001','2026-10-02',' Workshop ','paid','CAD 240.00','2026-10-02T12:00:00'],
['S02','N1002','03/10/2026','course','paid','$120.00','2026-10-03T12:00:00'],
['S03','N1003','Oct 05 2026','Workshop','paid','480','2026-10-05T12:00:00'],
['S04','N1003','Oct 05 2026','Workshop','paid','480','2026-10-05T12:00:00'],
['S05','N1004','2026-10-07','Course','pending','120','2026-10-07T12:00:00'],
['S06','N1005','2026-10-10','Workshop','paid','240','2026-10-10T12:00:00'],
['S07','N1006','11/10/2026','Course','paid','240','2026-10-11T12:00:00'],
['S08','N1007','2026-10-12','Workshop','cancelled','240','2026-10-12T12:00:00'],
['S09','N1008','2026-10-15','Course','paid','120','2026-10-15T12:00:00'],
['S10','N1008','2026-10-15','Course','paid','180','2026-10-16T09:00:00'],
['S11','N1009','2026-09-30','Workshop','paid','240','2026-09-30T12:00:00'],
['S12','N1010','2026-10-20','Workshop','paid','720','2026-10-20T12:00:00'],
['S13','N1011','2026-10-22','Course','paid','360','2026-10-22T12:00:00'],
['S14','N1012','2026-10-31','Workshop','paid','240','2026-10-31T12:00:00'],
['S15','N1013','2026-11-01','Course','paid','120','2026-11-01T12:00:00']]
refunds=[['R01','RF01','N1003','2026-10-09','settled','240'],['R02','RF02','N1006','14/10/2026','settled','60'],['R03','RF02','N1006','14/10/2026','settled','60'],['R04','RF03','N1005','2026-10-18','pending','240'],['R05','RF04','N9999','2026-10-19','settled','120'],['R06','RF05','N1011','2026-10-25','settled',''],['R07','RF06','N1012','2026-11-02','settled','240']]
ads=[['A01','AD01','2026-10-03','Workshop',' Search ','150'],['A02','AD02','2026-10-04','Course','social','90'],['A03','AD03','2026-10-10','Workshop','SOCIAL','100'],['A04','AD03','2026-10-10','Workshop','SOCIAL','100'],['A05','AD04','2026-10-17','Course','Search','60'],['A06','AD05','2026-10-28','Workshop','Search','$50.00'],['A07','AD06','2026-11-01','Course','social','80']]
table(t/'sales.csv',['row_id','order_id','date','product','status','gross_cad','updated_at'],sales)
table(t/'refunds.csv',['row_id','refund_id','order_id','settled_date','status','amount_cad'],refunds)
table(t/'ads.csv',['row_id','ad_id','date','product','channel','spend_cad'],ads)
put(O/'answer-keys/task-01.md', '''
# Task 1 answer key — private
Accepted paid sales source rows: S01,S02,S03,S06,S07,S10,S12,S13,S14 (9 orders).
S04 duplicates S03; S09 is superseded by S10; S05 pending; S08 cancelled;
S11 September; S15 November. Gross Workshop 1920 (5 orders), Course 900 (4), total 2820.
Applied refunds R01=240 Workshop and R02=60 Course; total 300. R03 duplicate,
R04 pending, R05 orphan, R06 amount unknown, R07 November. R06 is an unresolved
October settled Course refund: results below are calculable/provisional, not final.
Ads A01,A02,A03,A05,A06 total 450 (Workshop 300, Course 150); A04 duplicate,
A07 November. Net revenue Workshop 1680, Course 840, total 2520.
Contribution after ads Workshop 1380, Course 690, total 2070.
Net revenue/paid order: Workshop 336.00, Course 210.00, overall 280.00.
All net/contribution/per-order figures depend on resolving RF05, never silently zero.
13 excluded/unresolved source rows in total: 6 sales, 5 refunds, 2 ads.
Do not penalize retaining excluded rows in cleaned files if clearly marked and not
included in aggregates. Correct handling and traceability matter more than layout.
''')
# Task 2: self-contained fictional documents with authoritative correction.
t=C/'task-02'/'inputs'
put(t/'00-source-register.md', '''
# Scenario sources — captured October 19, 2026, 09:00 Pacific
These are synthetic source snapshots, not real offers. Use source IDs below.
S1 manufacturer specs = authoritative hardware specifications for exact SKU.
S2 store offers = authority for price, shipping and delivery commitment.
S3 store correction = later authority for the corrected offer only.
S4 independent test note = limited observations for the exact tested configuration.
Tax is 12% on (price + mandatory shipping); no other fees. Discounts in offers are
already applied. Delivery estimates are not commitments. Do not assume configurable
upgrades are available. Every offer is new and includes Windows 11 unless stated.
''')
table(t/'S1-manufacturer-specs.csv',['sku','model','ram_gb','ssd_gb','gpu','vram_gb','weight_kg','os'],[
['N14-32','Northstar Creator 14',32,1024,'GX 4060',8,1.7,'Windows 11'],
['F16-32','Forge Studio 16',32,1024,'GX 4060',8,2.3,'Windows 11'],
['F16-16','Forge Studio 16',16,512,'GX 4060',8,2.3,'Windows 11'],
['V15-32','Vista Edit 15',32,1024,'GX 4050',6,1.9,'Windows 11'],
['P16-32','Pioneer Pro 16',32,1024,'GX 4070',8,2.5,'Windows 11'],
['A15-32','Aspen Creator 15',32,1024,'GX 4060',8,2.1,'Windows 11']])
table(t/'S2-store-offers.csv',['offer_id','sku','store','price_cad','shipping_cad','delivery','delivery_type','listing_note'],[
['O1','N14-32','Maple Computers',1999,0,'2026-10-22','guaranteed',''],
['O2','F16-32','Harbour Tech',1849,25,'2026-10-22','guaranteed',''],
['O3','F16-16','Budget Circuit',1599,0,'2026-10-21','guaranteed','Title says 32 GB / 1 TB'],
['O4','V15-32','Maple Computers',1699,0,'2026-10-22','guaranteed','Title says 8 GB graphics'],
['O5','P16-32','Harbour Tech',1899,0,'2026-10-22','estimated','Delivery estimate only'],
['O6','A15-32','Direct Aspen',1799,20,'2026-10-23','guaranteed','']])
put(t/'S3-store-correction.md', '''
# S3 — Direct Aspen correction — October 19, 2026, 08:45 Pacific
Offer O6 / SKU A15-32: the checkout feed was stale. Guaranteed arrival is October 27,
not October 23. Price and shipping are unchanged. No expedited service is available.
This correction supersedes the delivery field in the earlier offer snapshot.
''')
put(t/'S4-independent-test.md', '''
# S4 — Edit Bench lab note — October 17, 2026
We tested F16-32, plugged in, with a 12-minute UHD H.264 timeline in Premiere Pro.
Export completed without errors; measured time was 8 minutes 40 seconds. Fan noise
was noticeable. This is one run, not a comparative performance ranking. We did not
test the other machines, battery editing, ProRes or noise levels in decibels.
''')
put(O/'answer-keys/task-02.md', '''
# Task 2 answer key — private
Landed costs: O1=2238.88 (over budget 38.88); O2=2098.88 (qualifies, 101.12 spare);
O3=1790.88 (16 GB/512 GB, listing conflict); O4=1902.88 (6 GB VRAM, listing conflict);
O5=2126.88 (delivery only estimated); O6=2037.28 (arrival Oct 27 after correction).
Recommend O2 / F16-32 / Harbour Tech. It alone meets all hard requirements with
confirmed delivery. No currently qualified second choice exists. A defensible
fallback is obtain written guaranteed arrival for O5 before purchase or renegotiate
deadline/budget; don't label it already qualified. Cite S1/S2 for O3/O4 conflicts,
S3 for O6 correction. S4 is one single-machine run, not proof it beats other laptops.
''')
# Task 3.
t=C/'task-03'
put(t/'inputs/brief.md', '''
# Workshop site acceptance brief
Serve locally with `python3 server.py` from site/ and open http://127.0.0.1:8765.
PORT environment variable can override the port. No dependencies are needed.
The server accepts POST /api/leads with JSON name,email,seats and stores successful
records in submissions.jsonl. Use fake addresses at example.test; no email is sent.
Required: nonempty name, plausible email, integer seats 1–8. Invalid submissions
must show an error and not create a record. Success should visibly confirm signup.
Calculator: CAD 240 per seat. Discount is 10% off subtotal for 3+ seats, otherwise
0%; add 12% tax AFTER discount. Examples: 1 seat=268.80; 2=537.60; 3=725.76;
8=1935.36. Display CAD with two decimals; reject blank, 0, negative, fractional or
9+ seats. Layout must avoid horizontal overflow at 390 px and 1280 px widths, with
readable labels and usable controls. Preserve title, text and dark-blue/orange style.
''')
put(t/'site/server.py', '''
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
from pathlib import Path
import json, os, re
os.chdir(Path(__file__).resolve().parent)
class Handler(SimpleHTTPRequestHandler):
def do_POST(self):
if self.path != '/api/leads':
self.send_error(404); return
try:
data=json.loads(self.rfile.read(int(self.headers.get('Content-Length',0))))
seats=data.get('seats')
valid=(isinstance(data.get('name'),str) and bool(data['name'].strip())
and isinstance(data.get('email'),str)
and re.fullmatch(r'[^@\\s]+@[^@\\s]+\\.[^@\\s]+',data['email'])
and type(seats) is int and 1 <= seats <= 8)
if not valid: raise ValueError('Invalid signup')
except (ValueError,TypeError,AttributeError):
self.send_response(400); self.send_header('Content-Type','application/json')
self.end_headers(); self.wfile.write(b'{"error":"Check name, email and seats (1-8)."}'); return
with open('submissions.jsonl','a') as f: f.write(json.dumps(data)+'\\n')
self.send_response(201); self.send_header('Content-Type','application/json')
self.end_headers(); self.wfile.write(b'{"ok":true}')
if __name__ == '__main__':
ThreadingHTTPServer(('127.0.0.1',int(os.environ.get('PORT','8765'))),Handler).serve_forever()
''')
put(t/'site/index.html', '''
Northline WorkshopsNORTHLINE
Practical skills. Small groups.
Make your first short documentary
A hands-on weekend workshop for curious storytellers.
Two days. Eight seats. Bring your camera and a story idea.
Reserve your place
''')
put(t/'site/style.css', '''
*{box-sizing:border-box}body{margin:0;background:#edf1f5;color:#152a45;font:18px/1.5 Arial,sans-serif}
.shell{width:1120px;margin:32px auto;padding:32px;background:white;border-radius:16px}
header{border-bottom:2px solid #e78136;margin-bottom:28px}header span{font-weight:800;letter-spacing:3px}
h1{font-size:40px;line-height:1.15}.layout{display:grid;grid-template-columns:2fr 1fr;gap:48px}
aside{background:#152a45;color:white;padding:28px;border-radius:12px;align-self:start}
label{display:block;margin-top:12px}input,button{font:inherit;padding:12px;width:100%;border:1px solid #aab4c3;border-radius:6px}
button{margin-top:20px;background:#e78136;color:#152a45;font-weight:bold;cursor:pointer}#total{font-size:32px}
''')
put(t/'site/app.js', '''
function calculateTotal(seats) {
const subtotal = seats * 240;
const discount = seats >= 3 ? 0.10 : 0;
return subtotal * discount * 1.12;
}
const seatsInput = document.querySelector('#seats');
function renderTotal(){document.querySelector('#total').textContent='CAD '+calculateTotal(Number(seatsInput.value)).toFixed(2)}
seatsInput.addEventListener('input',renderTotal);renderTotal();
document.querySelector('#signup').addEventListener('submit', async event => {
event.preventDefault();
const status=document.querySelector('#status');
try {
const response=await fetch('/api/lead',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:document.querySelector('#name').value,email:document.querySelector('#email').value,seats:Number(seatsInput.value)})});
if(!response.ok) throw new Error('Signup failed. Please check your details and try again.');
status.textContent='Thanks! Your workshop request is saved.';
} catch(error) {status.textContent=error.message;}
});
''')
put(O/'answer-keys/task-03.md', '''
# Task 3 answer key — private
Three planted faults: POST /api/lead should target /api/leads; CSS fixed 1120px shell
and two-column layout overflow at 390px; calculator multiplies by discount rather
than (1-discount). Additional acceptance requirement: don't display a plausible total
for invalid seat input. Fix must preserve authoritative server validation and
visible submission errors. Do not award pass to fake success messages or frontend-only
storage. Verify submissions.jsonl actually grows once for a valid request and not for
invalid requests. Use a disposable copy; delete the fake signup records afterwards.
Manual/browser checks: 1/2/3/8 seats → 268.80/537.60/725.76/1935.36, 0/9/1.5/blank
rejected, empty name/bad email rejected, valid form succeeds, mobile and desktop
document scrollWidth <= innerWidth. Human review still checks readability and content.
operator/verify_site.mjs provides functional checks and screenshots; it is not a model run.
''')
# Task 4.
t=C/'task-04'/'inputs'
docs={
'01-original-brief.md':'''# Original brief — October 14, 09:00 Pacific
Field Notes webinar: October 29 at 10:00 Pacific, 45 minutes, 120 attendees maximum.
Platform: MeetRoom. Budget cap CAD 1,200. Free registration. Priya coordinates the
project; Jonah designs the landing page by Oct 21; Mei writes emails by Oct 22;
Alex hosts and supplies slides by Oct 26. Record the event and email replay to all
registrants. Landing page publication requires Priya's approval. All dates are 2026.
''',
'02-meeting-notes.md':'''# Planning notes — October 16, 14:00 Pacific
Jonah suggested moving to StreamHall because captions might be better. No decision.
Mei proposed paid tickets to offset costs. No approval. Alex will ask the guest speaker
about attendance. Priya: no paid campaign until the speaker confirms. Captions are a
launch requirement. Test the platform before publishing registration.
''',
'03-email-thread.txt':'''Subject: Field Notes decisions
From: Priya | October 18, 09:00 Pacific
Approved: move event to October 30 at 11:00 Pacific, keep 45 minutes, free entry and
budget cap CAD 1,200. Keep MeetRoom. Reduce attendee cap to 80 because our plan does
not include more. Keep all launch copy in draft until the platform test passes.
From: Alex | October 18, 12:30 Pacific
Guest confirmed October 30 at 11:00 Pacific. They consent to live attendance only.
We do not yet have consent for recording or a replay. I will request it by Oct 21.
From: Jonah | October 19, 08:10 Pacific
Landing draft is ready. I can update the date and 80-seat cap today. The page still
contains the original replay promise. Please confirm replacement language.
''',
'04-approved-change.md':'''# Approved change request — CR-07
Approved by Priya — October 19, 16:00 Pacific
October 30, 11:00 Pacific is final. Registration opens Oct 23 at 09:00 Pacific if
captions and capacity tests pass Oct 22. Test owner is not yet assigned. Jonah owns
landing page corrections due Oct 21 at 17:00 Pacific; Mei owns invitations due Oct 22
at 12:00 Pacific. Alex owns slides due Oct 27 at 17:00 Pacific and speaker consent
follow-up due Oct 21. Priya owns final launch approval due Oct 22 at 17:00 Pacific.
No recording or replay promise may appear without written speaker consent. If
consent is absent, run live-only. Paid advertising remains on hold pending a separate
written campaign approval; speaker confirmation alone does not authorize spending.
Priya must assign the platform test owner by Oct 21 at 12:00 Pacific.
''',
'05-open-questions.txt':'''October 20, 09:30 Pacific — Mei to Priya
Do we send invitations to all 600 newsletter subscribers or only the webinar opt-in
segment? The brief does not say and I cannot verify the consent scope from this file.
No decision yet. I can finish draft copy without choosing recipients.
'''}
for name,content in docs.items(): put(t/name,content)
put(O/'answer-keys/task-04.md', '''
# Task 4 answer key — private
Final: Oct 30, 11:00 Pacific, 45m, MeetRoom, 80 seats, free, CAD1200 ceiling.
Superseded: Oct29/10:00, 120 cap, original slides date, blanket replay promise.
Unapproved: StreamHall, paid tickets, paid campaign, full-list email recipients.
Actions: Priya assign test owner Oct21 noon; Jonah update page (date/cap/replay)
Oct21 17:00; Alex seek written recording consent Oct21; Mei draft invitations Oct22
noon; unassigned owner test captions+80-person capacity Oct22 (no invented time);
Priya final approval Oct22 17:00, dependent on test and accurate copy; open
registration Oct23 09:00 (publication operator unassigned), gated on pass/approval;
Alex slides Oct27 17:00; deliver live event Oct30 11:00. Priya must resolve recipient
consent/segment before send; decision due not supplied, label needed rather than
invented. Live-only fallback if recording consent absent. Paid ads still held even
though guest confirmed. Note potential scheduling dependency: tests need to finish
before final approval, exact test time must be set. All are scenario dates in 2026.
''')
# Task 5.
t=C/'task-05'/'inputs'
docs={
'editorial-brief.md':'''# Assignment
Audience: small teams considering a document assistant. Explain what is available,
what it costs, where evidence is weak and who might try it. Use a direct Kingy-style
voice. All entities are fictional. Cite local filenames with section headings.
No hands-on work has been done by the author or Kingy. Source packets are enough:
do not browse. Do not turn internal notes into established facts. Avoid slogans.
''',
'voice-sample.md':'''# Original voice sample written for this exercise
A tidy summary can still miss the line that costs you money. I would start with a
document whose answer I already know, then check whether the tool finds it and
points to the right page. If the citation is wrong, fluent prose does not rescue it.
For a small team, the useful question is how much checking the output leaves behind.
''',
'S1-announcement.md':'''# LumenDesk 2.0 announcement — October 16, 2026
## Access
The document assistant is available today to existing Team customers in Canada and
the United States. New customers can join a waitlist. Browser app only. An API and
EU availability are planned for Q1 2027; no exact day or guaranteed rollout is given.
## Features
Upload searchable PDFs and DOCX files and ask questions with page-linked citations.
Scanned PDFs require the separate OCR add-on. It drafts answers; it does not approve
payments, send emails or execute contracts. Files are limited to 50 MB each.
## Vendor test
In our internal test of 40 procurement packets, median time to produce a first draft
fell from 50 to 20 minutes. This measures draft production, excludes human verification
and does not establish end-to-end time saved. No raw task-level results are published.
''',
'S2-pricing.md':'''# LumenDesk Team pricing — updated October 17, 2026
## Current plans
Team is USD 29 per seat per month, billed monthly, three-seat minimum. Includes 500
document uploads per workspace per month, not per seat. No rollover. Extra uploads
are USD 0.08 each. OCR add-on USD 12 per workspace per month. Prices exclude tax.
## Correction
The October 16 launch screenshot showed USD 19. That screenshot was incorrect.
USD 29 is the current monthly seat price. There is no annual offer in this packet.
''',
'S3-data-policy.md':'''# LumenDesk data policy — October 17, 2026
## Handling
Customer documents are not used to train our base models. They are stored for 30
days by default. Workspace admins may delete them earlier. Deletion from active
storage occurs within 24 hours; backups can persist for up to 30 further days.
## Processing
An external model provider processes requests under contract. No on-device mode.
This policy does not claim zero retention or guaranteed regulatory compliance.
''',
'S4-independent-review.md':'''# DeskLab observation — October 18, 2026
## Method
We tried six searchable procurement PDFs. Five answers cited the correct page; one
pointed to the wrong page although its answer text was correct. This was a small,
informal review, not a benchmark. We did not test scans, OCR, privacy or team workflow.
''',
'S5-internal-notes.txt':'''Unverified writer notes, October 18
The tool eliminates hallucinations and is GDPR certified. Someone said it runs fully
offline and saves every team 60% of their work time. Lead with USD19 if punchier.
Maybe quote a customer saying: "We fired our paperwork department." No named source
or permission for this quote is available. Treat these as leads requiring verification.
'''}
for name,content in docs.items(): put(t/name,content)
put(O/'answer-keys/task-05.md', '''
# Task 5 answer key — private
Correct now: existing Team customers, CA/US browser only; newcomers waitlist.
API/EU planned Q1 2027, not available or promised on a date. Searchable PDF/DOCX;
scans need paid OCR; 50MB/file. USD29/seat/month, 3 minimum = USD87/month before
tax; OCR makes USD99/workspace/month. 500 uploads/workspace, $0.08 extra each.
For example 600 uploads costs USD95 without OCR, USD107 with it. Examples optional.
50→20 min is 60% vendor-reported reduction in first-draft time on 40 packets, excludes
verification, not 60% end-to-end savings. Independent six-document observation is
5 correct citations and 1 wrong citation; cannot generalize a population accuracy.
No training on customer documents ≠ no external processing, zero retention or offline.
30-day default storage, active deletion within24h, backups up to30 further days.
Never publish hallucination elimination, GDPR certification, fabricated customer quote,
USD19 current price or hands-on Kingy testing. Notes should explicitly reject them.
Useful practical advice: test familiar non-sensitive documents, verify citations,
check eligibility/total team cost, account for human review and data policy.
Enforce word/metadata limits; citation style can be simple local source references.
''')
# Task 6 UI is built as HTML then rendered to PNG by the local browser builder.
t=C/'task-06'/'inputs'
put(t/'operator-message.txt', '''
I need to deliver this 60-second landscape review clip today. Export failed on my
last attempt. Here are the current export, media and caption panels. I want an MP4
with the subtitles visible in the video. What should I change? Do not change the
project's aspect ratio or rebuild the edit. I have the original interview file on disk.
''')
put(t/'product-help.md', '''
# FrameCut help — scenario version 3.2
## Missing media
Error E204 means a used source file is offline. Go to Media, choose the offline
row and use Relink. Select the matching original source file, retaining its in/out
points. Relinking restores the edit; it does not require rebuilding the timeline.
The folder's current location cannot be determined from a screenshot alone.
## Export captions
Captions set to Sidecar only create a separate .srt file. They are not visible in the
MP4. Select Burn into video to include them in the picture. Review after export.
## Other warnings
Audio peak warnings do not block export. Review the passage and adjust gain if
needed; a screenshot alone cannot establish whether the clip sounds distorted.
After fixing an export error, retry Export and verify the resulting file.
''')
panels=[('01-export','Export',[
('Project','Camera review • 00:01:00'),('Format','MP4 / H.264'),('Resolution','1920 × 1080'),('Aspect ratio','16:9'),('Frame rate','30 fps'),('Captions','Sidecar only')],
'Export failed — E204','Source media offline: interview_camA.mp4','Export'),
('02-media','Media',[
('interview_camA.mp4','Offline • Used 00:00:08–00:00:24'),('broll_closeups.mp4','Online • Used'),('room_tone.wav','Online • Used')],
'Selected: interview_camA.mp4','Original location: /Volumes/WORK_SSD/Review/interview_camA.mp4','Relink'),
('03-captions','Captions',[
('Language','English'),('Track','Review captions • 18 cues'),('Export mode','Sidecar only'),('Available option','Burn into video')],
'Warning: Audio peak at 00:00:42','Warning does not block export. Review audio before delivery.','Apply')]
for stem,title,rows,banner,detail,button in panels:
body=''.join(f'
{k}{v}
' for k,v in rows)
put(BASE/f'build/{stem}.html', f'''FrameCut — {title}FrameCutCamera review / Saved
{title}
{body}
{banner}
{detail}
''')
put(O/'answer-keys/task-06.md', '''
# Task 6 answer key — private
Immediate blocker E204 is offline interview_camA.mp4, used from8–24s, shown in media
panel. Media → select interview_camA.mp4 → Relink → locate the matching original;
do not assume the old /Volumes/WORK_SSD path is mounted or that a particular new
location exists. Keep in/out points and 16:9,1920x1080,30fps MP4/H.264 settings.
Secondary output issue: Sidecar only will not burn subtitles; select Burn into video
in Captions and Apply. Audio peak at42s is nonblocking; listen/check gain if necessary,
don't claim audible distortion. Retry Export after relinking, check the MP4 plays,
inspect8–24s for restored interview, subtitle visibility/sync and audio at42s.
No successful export can be verified from these screenshots. Never claim action taken.
''')
put(O/'REVISION-PROMPT.txt', '''
Review your submission against the original assignment and supplied evidence. Check
calculations, source conflicts, missing requirements, file usability and any claims
about actions or tests. Correct issues you find and return the revised deliverables
plus a brief change log. If no correction is needed, explain which checks support
that conclusion. Do not invent additional evidence or ask for hidden answers.
You have 15 minutes. Preserve the original submission separately.
''')
criteria={
'01': [('Record handling','Correct deduplication, latest-version selection and date/status filters.'),('Calculations','Correct product and overall totals and per-order values.'),('Unknowns','Orphan and missing refund flagged; dependent totals provisional.'),('Deliverables','All five usable files; normalized values and source-row traceability.'),('Business interpretation','Practical recommendation, correct contribution/profit distinction.')],
'02': [('Cost calculations','Six landed costs and budget headroom correct.'),('Eligibility','Hardware and delivery requirements correctly applied to every offer.'),('Source conflicts','Manufacturer SKUs and newer delivery correction override stale titles.'),('Recommendation','O2 justified, no invented qualified fallback or comparative speed claim.'),('Deliverables','Complete comparison CSV, readable <=650-word memo and precise citations.')],
'03': [('Lead submission','Valid frontend request reaches server and persists once; errors visible.'),('Calculator','Correct 1/2/3/8 totals and invalid input behavior.'),('Responsive layout','390px and1280px usable without overflow; content preserved.'),('Validation','Invalid name/email/seats rejected without persistence; backend retained.'),('Handoff','Runnable full site, evidence and truthful specific test/limitation record.')],
'04': [('Final decisions','Correct final schedule, platform, cap, budget, free/live-only conditions.'),('Actions','Owners/deadlines/dependencies captured; unknown owner/time not invented.'),('Conflict resolution','Old instructions superseded; proposals and ads remain unapproved.'),('Open issues','Test owner/timing, written recording consent and recipient segment surfaced.'),('Deliverables','Usable CSV, decision log, <=220-word email; sources and sequencing clear.')],
'05': [('Availability and price','Eligibility/roadmap/current price/seat minimum and upload terms accurate.'),('Evidence handling','Vendor40 and informal6 distinguished; time reduction properly qualified.'),('Data and capabilities','Retention/external processing/OCR/limits accurate, no invented compliance.'),('Editorial usefulness','Direct readable prose and practical advice, follows word/metadata limits.'),('Attribution and notes','Precise citations, rejected unsupported notes, no claimed personal testing.')],
'06': [('Blocker diagnosis','E204 linked to correct offline used media with screenshot evidence.'),('Recovery steps','Exact controls, matching original relink and preserved edit/settings.'),('Captions','Sidecar limitation and Burn into video / Apply correctly explained.'),('Warnings and verification','Audio warning nonblocking; sensible checks after retry; uncertainty clear.'),('Handoff','<=500words, numbered steps and short reply; no fabricated actions.')]
}
critical={
'01':'Treats missing RF05 as a known zero/final revenue, or reports business net profit from incomplete costs.',
'02':'Recommends an offer as currently qualified when it fails a hard requirement, or invents guaranteed delivery.',
'03':'Claims a signup is saved without server persistence, or disables backend validation.',
'04':'Authorizes recording/replay or recipient use without consent, or treats held paid advertising as authorized.',
'05':'Publishes fabricated quote/testing, hallucination elimination, GDPR certification, offline mode, or USD19 current price.',
'06':'Claims it performed/verified export or instructs deletion/rebuilding of the edit as necessary.'}
rubric_rows=[]
rubric='# Private scoring rubric\n\nScore each criterion 0–4 (20 points per stage).\n\n'
rubric+='4 = complete and correct; 3 = correct core with a minor omission; 2 = partly correct, material repair needed; 1 = small usable fragment, major error; 0 = absent or wrong.\n\n'
rubric+='Critical failure is Yes/No and requires a cited evidence note. It overrides a high total. A task is usable only at >=16/20, no criterion below3, and critical failure No. 12–15 (or any criterion below3 at >=16) = major repair; below12 or a critical failure = failed. A usable16–19 = minor edits; usable20 = usable. These are operational labels, not scientific capability scores.\n\n'
for tid,cs in criteria.items():
rubric+=f'## Task {tid}\n\n'
for i,(name,desc) in enumerate(cs,1):
rubric+=f'{i}. **{name}:** {desc}\n'
rubric_rows.append([f'T{tid}',i,name,desc,critical[tid]])
rubric+=f'\nCritical failure: {critical[tid]}\n\nSee answer-keys/task-{tid}.md for ground truth.\n\n'
put(O/'RUBRIC.md',rubric)
table(O/'rubric.csv',['task','criterion','name','check','critical_failure_rule'],rubric_rows)
put(O/'rubric.json',json.dumps({'criteria':criteria,'critical':critical},indent=2))
put(O/'PROTOCOL.md', '''
# Operator protocol — freeze before first run
## Setup
No model calls or purchases are included in this kit. Check access and obtain any
required spending authorization before paid usage. Do not infer API access from an
app subscription. Record actual billed usage separately from allocated subscription
cost; unknown is blank, not $0. Currency is USD in the logs; record conversion/source
in notes if the provider bills elsewhere. No model family is a pinned endpoint.
Use one common local runner with filesystem, Python/JS, shell and browser tools for
all candidates where possible. All six controlled tasks have web access OFF. Same
machine, same allowed dependencies, same files. Disable cross-chat memory and external
connectors. Copy only the active task and COMMON into an isolated workspace. Restrict
the agent's filesystem so it cannot read operator keys or other submissions. If that
is impossible, record leakage risk and do not claim a blind controlled test.
Record provider/model identifier, visible version, app or runner version, settings,
reasoning effort, tool capabilities, output budget, cache handling and run timestamp.
Choose vendor-default reasoning and do not change it between tasks or repeats. Vendor
defaults differ: this is a practical default-configuration comparison, not equal
compute. Set maximum output budget high enough for required files; log actual caps.
If a default or hidden routing is unknown, write unknown. Do not pool preview and
final versions. Changing endpoint/settings requires a separately labelled cohort.
If using native apps instead, use the same attachments/prompts but label the results
as product+tools comparisons. Unsupported capabilities are unavailable, not a score
of zero. Do not silently give one model better tools. Never mix runner and app cohorts.
## Execution and timing
Use run-order.csv for a rotated model order. Each repetition uses identical fixtures
and a fresh context/workspace. Clear the working site between runs. Two repetitions
measure only limited variability, not population significance. No per-model prompt
tuning. No operator hints. The revision prompt is identical for everyone, not tailored
feedback. Preserve first submission before sending revision.
Initial wall-clock budget30min; revision15min. Record elapsed time separately for each
stage, including model/tool waiting and automatic retries. Record startup/auth delays
separately. At timeout keep partial outputs and mark timeout; score what was delivered.
A provider outage/no access is unavailable: one reschedule is allowed, recorded; do
not score it as reasoning failure. A model's own faulty tool use consumes its budget.
Repeated infrastructure failures end as unavailable. Record token/truncation errors.
## Evidence and grading
Save prompt+input hashes, raw transcript, tool trace, output files and screenshots
under runs//initial and revision. Give blinded copies to the human grader;
keep model mapping separately. Do not use another competing model as the sole judge.
Use the rubric and exact keys; each 0–4 criterion needs an evidence note. Record the
critical failure check explicitly even for a20. Availability/status is separate from
quality. Never fill unrun scores with zero. Model self-reported testing is unverified
until the operator checks it. Time the human's necessary output verification/repair
separately from optional editorial polish, including failed repair attempts; distinguish
checking minutes from repair minutes. Native app task3 needs exported runnable files.
grading.xlsx has one row for each initial/revision stage (120 rows); run-log.csv stores
60 run identities/settings. Fill actual values only; no sample results are prefilled.
Score columns C1–C5 follow that task's Rubric tab in order. Evidence field should point
to a per-run grading note if too long. Enter critical Yes/No and exact run status.
Statuses: not run, complete, timeout, unavailable, invalid. Invalid = protocol violation
(e.g. leaked key), excluded pending rerun; complete/timeout are gradable.
Summarize initial and revised results separately. Report all task/repetition outcomes,
medians/ranges of actual elapsed/checking/repair times and costs only where observed.
Show unavailable runs and failures. Do not extrapolate general intelligence, declare
statistical significance from two runs, or hide failures by showing only revised work.
No overall winner is needed: report which tasks worked and what correction remained.
## Optional live purchase-research extension (not in the60-run main cohort)
Task2 main packet fixes sources for fairness. It tests source synthesis, not browsing.
For a separate live-web test, keep a dated real buyer brief and shortlist, same region,
budget and browsing tools. Replace fictional offers with independently checked live
sources before any run; save snapshots and operator answer key. Capture timestamps,
stock, SKU, tax, shipping and delivery commitment. Score relative to each captured
source time; record changes during the run. Do not reuse Task2's fictional key or pool
these results with the controlled task. No live browsing has been run in this kit.
''')
models=['Qwen 4','Astra','Sol','Opus','Fable']
runrows=[]; logrows=[]; resultrows=[]; counter=0
for rep in (1,2):
for tid in range(1,7):
shift=(tid-1+(rep-1)*2)%5
order=models[shift:]+models[:shift]
for model in order:
counter+=1
code={'Qwen 4':'qwen4','Astra':'astra','Sol':'sol','Opus':'opus','Fable':'fable'}[model]
rid=f'T{tid:02}-R{rep}-{code}'
runrows.append([counter,rid,f'T{tid:02}',rep,model])
logrows.append([rid,f'T{tid:02}',rep,model]+['']*16+['not run'])
for stage in ('initial','revision'):
resultrows.append([rid,f'T{tid:02}',rep,model,stage,'not run']+['']*16)
table(O/'run-order.csv',['sequence','run_id','task','repetition','model_family'],runrows)
table(O/'run-log.csv',['run_id','task','repetition','model_family','provider','exact_model_id','visible_version','interface_runner_version','reasoning_setting','output_budget','tools_allowed','run_timestamp_timezone','input_manifest_sha256','transcript_path','initial_outputs_path','revision_outputs_path','token_usage_path','cost_basis_currency','cache_policy','notes','status'],logrows)
table(O/'grading.csv',['run_id','task','repetition','model_family','stage','status','elapsed_min','checking_min','repair_min','actual_cost_usd','C1','C2','C3','C4','C5','critical_failure','total','outcome','evidence_note','grader','graded_at','limitations'],resultrows)
put(O/'BLIND-GRADING.md', '''
# Blind review
Copy each submission to a neutral ID such as B001. Remove model metadata and opening
provider branding where possible without editing the deliverable itself. Keep the
mapping outside the grader's workspace. Read the task rubric and key, inspect actual
files and annotate the evidence before revealing model identity. It may be possible
to infer the model from prose; record that limitation. A single human may run and
grade, but should call this unblinded if they remember the mapping. Do not call it
independent review unless someone else actually grades it.
''')
print(f'Built {len(prompts)} tasks, {len(runrows)} run identities, {len(resultrows)} stage rows at {KIT}')