#!/usr/bin/env python3
"""Build a deterministic, model-blinded writing-evaluation packet.

The public packet omits model identities and mechanical pass/fail labels.  A
separate key allows the scores to be joined back to the benchmark only after
the evaluator has completed the rubric.
"""

from __future__ import annotations

import argparse
import csv
import hashlib
import json
import random
from pathlib import Path


MODELS = ("qwen38", "qwen36", "gemma4")
SEED = 20260816


def load(path: Path):
    return json.loads(path.read_text(encoding="utf-8"))


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--results-root", type=Path, required=True)
    parser.add_argument("--fixtures", type=Path, required=True)
    parser.add_argument("--out", type=Path, required=True)
    args = parser.parse_args()

    suite = load(args.fixtures)
    prompts = {row["id"]: row["prompt"] for row in suite["writing"]}
    rows: list[dict] = []
    for model in MODELS:
        for path in sorted((args.results_root / model / "writing").glob("*.json")):
            data = load(path)
            case_id = data.get("id") or path.stem
            rows.append(
                {
                    "model": model,
                    "case_id": case_id,
                    "prompt": prompts[case_id],
                    "output": data["output"],
                    "output_sha256": hashlib.sha256(data["output"].encode()).hexdigest(),
                }
            )

    expected = len(MODELS) * len(prompts)
    if len(rows) != expected:
        raise SystemExit(f"expected {expected} outputs, found {len(rows)}")

    rng = random.Random(SEED)
    rng.shuffle(rows)
    args.out.mkdir(parents=True, exist_ok=True)

    packet: list[dict] = []
    key: list[dict] = []
    for index, row in enumerate(rows, 1):
        sample_id = f"sample-{index:03d}"
        packet.append(
            {
                "sample_id": sample_id,
                "case_id": row["case_id"],
                "prompt": row["prompt"],
                "output": row["output"],
                "output_sha256": row["output_sha256"],
            }
        )
        key.append(
            {
                "sample_id": sample_id,
                "model": row["model"],
                "case_id": row["case_id"],
                "output_sha256": row["output_sha256"],
            }
        )

    (args.out / "blind-writing-packet.json").write_text(
        json.dumps(
            {
                "method": "One evaluator; model identities hidden; deterministic shuffle.",
                "rubric": {
                    "instruction_adherence": "0 misses, 1 partial, 2 complete",
                    "factual_consistency": "0 changes/adds facts, 1 minor issue, 2 preserves facts",
                    "style": "0 poor, 1 usable, 2 clear and natural",
                    "edit_preservation": "0 loses essentials, 1 partial, 2 preserves essentials",
                },
                "samples": packet,
            },
            ensure_ascii=False,
            indent=2,
        )
        + "\n",
        encoding="utf-8",
    )
    (args.out / "blind-writing-key.json").write_text(
        json.dumps({"shuffle_seed": SEED, "key": key}, ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )

    with (args.out / "blind-writing-scores.csv").open("w", newline="", encoding="utf-8") as handle:
        writer = csv.writer(handle)
        writer.writerow(
            [
                "sample_id",
                "instruction_adherence_0_2",
                "factual_consistency_0_2",
                "style_0_2",
                "edit_preservation_0_2",
                "note",
            ]
        )
        for row in packet:
            writer.writerow([row["sample_id"], "", "", "", "", ""])

    print(json.dumps({"samples": len(packet), "out": str(args.out)}))


if __name__ == "__main__":
    main()
