#!/usr/bin/env python3
"""Validate a completed blinded rubric and reveal aggregate model scores."""

from __future__ import annotations

import argparse
import csv
import json
import statistics
from collections import defaultdict
from pathlib import Path


FIELDS = (
    "instruction_adherence_0_2",
    "factual_consistency_0_2",
    "style_0_2",
    "edit_preservation_0_2",
)
NAMES = {"qwen38": "Qwen3.8-27B", "qwen36": "Qwen3.6-27B", "gemma4": "Gemma 4 31B-it"}


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--scores", type=Path, required=True)
    parser.add_argument("--key", type=Path, required=True)
    parser.add_argument("--out", type=Path, required=True)
    args = parser.parse_args()

    key_data = json.loads(args.key.read_text(encoding="utf-8"))
    identities = {row["sample_id"]: row for row in key_data["key"]}
    scored: list[dict] = []
    with args.scores.open(newline="", encoding="utf-8") as handle:
        for row in csv.DictReader(handle):
            if row["sample_id"] not in identities:
                raise SystemExit(f"unknown sample ID: {row['sample_id']}")
            parsed = {field: int(row[field]) for field in FIELDS}
            if any(value not in (0, 1, 2) for value in parsed.values()):
                raise SystemExit(f"rubric score outside 0-2: {row['sample_id']}")
            scored.append({**identities[row["sample_id"]], **parsed, "note": row.get("note", "")})

    if set(row["sample_id"] for row in scored) != set(identities):
        missing = sorted(set(identities) - {row["sample_id"] for row in scored})
        raise SystemExit(f"missing completed scores: {missing}")

    by_model: dict[str, list[dict]] = defaultdict(list)
    for row in scored:
        by_model[row["model"]].append(row)
    models = {}
    for model, rows in by_model.items():
        criterion = {
            field.removesuffix("_0_2"): {
                "points": sum(row[field] for row in rows),
                "possible": 2 * len(rows),
                "mean_0_2": statistics.mean(row[field] for row in rows),
            }
            for field in FIELDS
        }
        total = sum(row[field] for row in rows for field in FIELDS)
        possible = len(rows) * len(FIELDS) * 2
        models[model] = {
            "display_name": NAMES[model],
            "samples": len(rows),
            "total_points": total,
            "possible_points": possible,
            "percent_of_possible": 100 * total / possible,
            "criteria": criterion,
        }

    result = {
        "method": "Single blinded evaluator; four 0-2 criteria; model identities revealed only for aggregation.",
        "limitations": "Directional subjective result from one evaluator, not a population estimate or native-speaker multilingual assessment.",
        "models": models,
        "scored_rows": scored,
    }
    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(json.dumps({model: round(row["percent_of_possible"], 1) for model, row in models.items()}))


if __name__ == "__main__":
    main()
