"""Verify a MAGI benchmark bundle with the Python standard library alone.

Usage, from anywhere (Python 3.10 or later, nothing to install)::

    python3 verify_bundle.py [BUNDLE_DIR]

``BUNDLE_DIR`` defaults to the folder this file is in. A rescore bundle (``manifest.json`` kind ``rescore``)
is checked against its parent, which must sit beside it (``../<parent_run_id>``).

It checks, without the MAGI repository:

1. every line of ``SHA256SUMS``: the file exists and hashes to the digest, the list is sorted, and no file
   of the bundle is missing from it; the run digest is the sha256 of ``SHA256SUMS`` itself;
2. every attachment's name is the sha256 of its bytes;
3. every result's ``task_sha256`` is the sha256 of its ``specs/<task_id>.json``;
4. the registry chain in ``../registry`` when it is present: events numbered 1..N, each ``prev_sha256`` the
   hash of the previous event file, and this run's ``final`` event naming this ``manifest.json`` and this
   ``SHA256SUMS``;
5. ``summary.json`` and ``summary.md`` are recomputed from ``results.jsonl``, the specs, the sources and the
   manifest, and must be byte-identical.

Exit status 0 when everything holds, 1 otherwise. Replaying the tool calls needs the repository at the
bundle's commit (``magi-bench verify``); this file does not do that.

This file is also the module the benchmark harness computes ``summary.json`` with
(``magi_bench.report.summary_json``), so the numbers a reader recomputes are computed by the same code. Float
text in every JSON file is Python's ``repr``.
"""

from __future__ import annotations

import hashlib
import json
import math
import random
import sys
from pathlib import Path
from typing import Any

#: Percentile bootstrap: resamples and seed (recorded in each interval).
BOOTSTRAP_B = 10_000
BOOTSTRAP_SEED = 20261015
WILSON_Z = 1.959963984540054
NUMERIC_KINDS = ("answer_figure_close",)

# ------------------------------------------------------------------------------------------------ helpers


def sha256(b: bytes) -> str:
    return hashlib.sha256(b).hexdigest()


def pretty(obj: Any) -> bytes:
    """The human-facing JSON files (summary, manifest, sources, registry): sorted keys, indented, UTF-8."""
    return (json.dumps(obj, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode()


def jsonl(path: Path) -> list[dict[str, Any]]:
    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]


def _mean(xs: list[float]) -> float:
    return sum(xs) / len(xs)


def _ratio(k: int, n: int) -> dict[str, Any]:
    return {"k": k, "n": n, "rate": (k / n) if n else None}


def wilson(k: int, n: int, z: float = WILSON_Z) -> tuple[float, float]:
    p = k / n
    denom = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / denom
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
    return centre - half, centre + half


def cluster_bootstrap(
    clusters: list[list[float]], b: int = BOOTSTRAP_B, seed: int = BOOTSTRAP_SEED
) -> tuple[float, float]:
    """Percentile interval of the mean task score, resampling clusters of tasks with replacement."""
    rng = random.Random(seed)
    m = len(clusters)
    sums = [sum(c) for c in clusters]
    sizes = [len(c) for c in clusters]
    stats = []
    for _ in range(b):
        tot = 0.0
        cnt = 0
        for _ in range(m):
            i = rng.randrange(m)
            tot += sums[i]
            cnt += sizes[i]
        stats.append(tot / cnt)
    stats.sort()
    return stats[int(0.025 * b)], stats[int(math.ceil(0.975 * b)) - 1]


def published_decimals(printed: str) -> int:
    digits = printed.strip().lstrip("-−+").replace(",", "")
    return len(digits.split(".")[1]) if "." in digits else 0


def published_tolerance(printed: str) -> float:
    value = abs(float(printed.strip().lstrip("−").replace("−", "-").replace(",", "")))
    return 0.5 * 10.0 ** -published_decimals(printed) + 1e-9 * max(1.0, value)


def apply_format(value: float, fmt: dict[str, Any]) -> float:
    t = fmt.get("figure_transform") or "identity"
    x = abs(value) if t == "abs" else (-value if t == "neg" else value)
    x = x * float(fmt.get("unit_scale", 1.0) if fmt.get("unit_scale") is not None else 1.0)
    if fmt.get("round_to") is not None:
        x = round(x, int(fmt["round_to"]))
    return x


def figure_check_passes(check: dict[str, Any], x: float, sources: dict[str, dict[str, Any]]) -> bool:
    """An ``answer_figure_close`` check on the value ``x`` (already formatted), by the harness's rule."""
    if check.get("interval"):
        lo, hi = float(check["value"][0]), float(check["value"][1])
        return lo < x < hi if check.get("open_interval") else lo <= x <= hi
    expected = float(check["value"])
    abs_tol, rel_tol = check.get("abs_tol"), check.get("rel_tol")
    if abs_tol is None and rel_tol is None:
        src = sources.get(check.get("source_id") or "")
        # a printed SE has its own decimals: the cell's rule reads them from se_as_published
        field = "se_as_published" if check.get("field") == "se" else "value_as_published"
        if src and src.get("kind") in ("paper", "textbook") and src.get(field):
            abs_tol = published_tolerance(src[field])
    if abs_tol is None and rel_tol is None:
        return abs(x - expected) <= 1e-6 * max(abs(expected), 1.0)
    ok = False
    if abs_tol is not None:
        ok = ok or abs(x - expected) <= abs_tol
    if rel_tol is not None:
        ok = ok or abs(x - expected) <= rel_tol * abs(expected)
    return ok


#: Reference-path provenances MAGI built (by a committed metadata rule, or by hand) rather than transcribed
#: from the benchmark's own script: their replay is reference-path agreement, never replication fidelity.
RULE_BUILT_PATHS = ("metadata_rule", "hand_inferred")


def provenance(spec: dict[str, Any] | None) -> str | None:
    """The provenance of a task's reference path (``meta.reference_path.provenance``); None without one."""
    path = ((spec or {}).get("meta") or {}).get("reference_path")
    return path.get("provenance") if isinstance(path, dict) else None


def _paper_target(task_id: str, spec: dict[str, Any] | None) -> tuple[str, str]:
    meta = (spec or {}).get("meta") or {}
    parts = task_id.split(".")
    paper = meta.get("paper") or (parts[1] if len(parts) > 3 else task_id)
    target = meta.get("target") or (parts[2] if len(parts) > 3 else task_id)
    return str(paper), str(target)


# ------------------------------------------------------------------------------------------------ summary

#: The task kinds ``by_kind`` splits a suite into (``bundle_schema.TaskKind``).
TASK_KINDS = ("tool", "agent", "design", "protocol")


def task_kind(target: str | None, split: str | None) -> str:
    """A task's kind, from its target and split: ``tool`` (target ``tool``: a T1 reference call, a certified
    value), ``protocol`` (target ``protocol``), ``design`` (target ``agent`` on split ``design``: the T3
    tasks of ``paper_repro``) and ``agent`` (every other agent task: T2 and the external suites)."""
    if target == "tool":
        return "tool"
    if target == "protocol":
        return "protocol"
    return "design" if split == "design" else "agent"


def gold_defect(spec: dict[str, Any] | None) -> bool:
    """Whether a task's gold is known to contradict its own question or to be defective (``meta.gold_defect``
    holds the reason). Such a task is scored as the benchmark scores it; the summary also gives each rate
    without it, beside the rate as scored."""
    return bool(((spec or {}).get("meta") or {}).get("gold_defect"))


def _scores(last: dict[tuple[str, int], dict[str, Any]], specs: dict[str, dict[str, Any]]) -> dict[str, Any]:
    """The pass rates of a set of scored attempts (the last attempt of each task and epoch): per-task means
    over epochs, incidents excluded (``pass_rate``) and counted as failures, the same rate over the tasks
    whose gold is not defective (``pass_rate_excl_gold_defects``; ``n_gold_defects`` counts the scored tasks
    it leaves out), the 95% interval, the status counts and the failure classes."""
    status_counts: dict[str, int] = {}
    scored: dict[str, list[float]] = {}
    counted: dict[str, list[float]] = {}
    ran = incidents = 0
    for (task_id, _), r in sorted(last.items()):
        st = r["status"]
        status_counts[st] = status_counts.get(st, 0) + 1
        if st == "skipped":
            continue
        ran += 1
        if st == "error" and r.get("error_kind") == "provider":
            incidents += 1
            counted.setdefault(task_id, []).append(0.0)
            continue
        v = 1.0 if st == "passed" else 0.0
        scored.setdefault(task_id, []).append(v)
        counted.setdefault(task_id, []).append(v)
    n = len(scored)
    means = {t: _mean(v) for t, v in scored.items()}
    counted_means = {t: _mean(v) for t, v in counted.items()}

    ci = None
    if n:
        clusters: dict[str, list[float]] = {}
        for t in sorted(means):
            meta = (specs.get(t) or {}).get("meta") or {}
            clusters.setdefault(str(meta.get("cluster") or t), []).append(means[t])
        single = all(len(v) == 1 for v in scored.values()) and all(len(c) == 1 for c in clusters.values())
        if single:
            lo, hi = wilson(int(sum(means.values())), n)
            ci = {"low": lo, "high": hi, "method": "wilson", "resamples": None, "seed": None, "n_clusters": n}
        else:
            groups = [clusters[k] for k in sorted(clusters)]
            lo, hi = cluster_bootstrap(groups)
            ci = {
                "low": lo,
                "high": hi,
                "method": "cluster_bootstrap",
                "resamples": BOOTSTRAP_B,
                "seed": BOOTSTRAP_SEED,
                "n_clusters": len(groups),
            }
    failure_classes: dict[str, int] = {}
    for r in last.values():
        if r.get("failure_class"):
            failure_classes[r["failure_class"]] = failure_classes.get(r["failure_class"], 0) + 1
    defects = sorted(t for t in means if gold_defect(specs.get(t)))
    kept = [means[t] for t in sorted(means) if t not in defects]
    return {
        "status_counts": dict(sorted(status_counts.items())),
        "scored": scored,
        "means": means,
        "counted_means": counted_means,
        "n": n,
        "ran": ran,
        "incidents": incidents,
        "pass_rate": _mean(list(means.values())) if n else None,
        "pass_rate_incidents_as_fail": _mean(list(counted_means.values())) if counted_means else None,
        "pass_rate_excl_gold_defects": _mean(kept) if kept else None,
        "n_gold_defects": len(defects),
        "ci95": ci,
        "failure_classes": dict(sorted(failure_classes.items())),
    }


def by_kind(last: dict[tuple[str, int], dict[str, Any]], specs: dict[str, dict[str, Any]]) -> dict[str, Any]:
    """``SuiteSummary.by_kind``: every kind the suite's results have (:func:`task_kind`), each scored on its
    own tasks alone, so T2 and T3 rates are never pooled."""
    groups: dict[str, dict[tuple[str, int], dict[str, Any]]] = {}
    for key, r in last.items():
        groups.setdefault(task_kind(r.get("target"), r.get("split")), {})[key] = r
    out: dict[str, Any] = {}
    for kind in [k for k in TASK_KINDS if k in groups]:
        s = _scores(groups[kind], specs)
        counts = s["status_counts"]
        out[kind] = {
            "n": s["n"],
            "passed": counts.get("passed", 0),
            "failed": counts.get("failed", 0),
            "error": counts.get("error", 0),
            "skipped": counts.get("skipped", 0),
            "pass_rate": s["pass_rate"],
            "ci95": s["ci95"],
            "ci_method": s["ci95"]["method"] if s["ci95"] else None,
            "pass_rate_incidents_as_fail": s["pass_rate_incidents_as_fail"],
            "pass_rate_excl_gold_defects": s["pass_rate_excl_gold_defects"],
            "n_gold_defects": s["n_gold_defects"],
            "failure_classes": s["failure_classes"],
        }
    return out


def _whole_suite(msuite: dict[str, Any]) -> bool:
    """Whether the run ran every task the suite builds (always true of a manifest that does not say)."""
    whole = msuite.get("n_suite_tasks")
    return whole is None or int(msuite.get("n_tasks") or 0) == int(whole)


def _num(v: Any) -> float | None:
    return (
        float(v)
        if isinstance(v, bool | int | float) and not (isinstance(v, float) and math.isnan(v))
        else None
    )


def _median(xs: list[float]) -> float:
    s = sorted(xs)
    m = len(s) // 2
    return s[m] if len(s) % 2 else (s[m - 1] + s[m]) / 2


def _native_stats(final: list[dict[str, Any]], medians: list[str]) -> dict[str, float | None]:
    """Per native key, the mean over the attempts that have a number for it (None ignored), and
    ``<key>_median`` over the same numbers for each key in ``medians``."""
    out: dict[str, float | None] = {}
    keys = sorted({k for r in final for k, v in (r.get("native") or {}).items() if _num(v) is not None})
    for k in keys:
        vals = [x for r in final if (x := _num((r.get("native") or {}).get(k))) is not None]
        out[k] = _mean(vals) if vals else None
        if k in medians:
            out[f"{k}_median"] = _median(vals) if vals else None
    return out


def _native_full(
    final: list[dict[str, Any]], tasks: list[str], fill: dict[str, Any], split: int
) -> dict[str, float | None]:
    """``<key>_full`` per key of ``fill``: over ``split`` items, each task's mean over its epochs, an
    unanswered task and every item not among ``tasks`` counting as the fill value."""
    out: dict[str, float | None] = {}
    for k, f in sorted(fill.items()):
        total = 0.0
        for t in tasks:
            vals = [
                x
                for r in final
                if r["task_id"] == t and (x := _num((r.get("native") or {}).get(k))) is not None
            ]
            total += _mean(vals) if vals else float(f)
        total += float(f) * max(0, split - len(tasks))
        out[f"{k}_full"] = total / split if split > 0 else None
    return out


def native_summary(
    final: list[dict[str, Any]],
    rows: list[dict[str, Any]],
    msuite: dict[str, Any],
    specs: dict[str, dict[str, Any]] | None = None,
) -> dict[str, float | None]:
    """``summary.native``: per native key, the mean over the scored attempts that have a number for it (None
    ignored), and for each key the manifest's ``native_median`` names (a benchmark whose leaderboard reports
    a median) its median over the same numbers as ``<key>_median``. For each key in the manifest's
    ``native_fill``, when the run ran the whole suite, a ``<key>_full`` variant over the source split: each
    task's mean over its epochs, an unanswered task and every item the suite excludes counting as the fill
    value (binary 0, a relative error 1).

    When a task of the run has a defective gold (:func:`gold_defect`), every number above also has an
    ``<name>_egd`` variant ("excluding gold defects"): the same rule without those tasks, a full variant's
    split reduced by them. The variants stand beside the numbers as scored, never instead of them."""
    specs = specs or {}
    medians = [str(k) for k in msuite.get("native_median") or []]
    out = _native_stats(final, medians)
    fill = msuite.get("native_fill") or {}
    split = int(
        msuite.get("source_split") or (int(msuite.get("n_tasks") or 0) + len(msuite.get("excluded") or []))
    )
    whole = bool(fill) and split > 0 and _whole_suite(msuite)
    tasks = sorted({r["task_id"] for r in rows})
    if whole:
        out |= _native_full(final, tasks, fill, split)
    defects = {t for t in tasks if gold_defect(specs.get(t))}
    if not defects:
        return out
    kept = [r for r in final if r["task_id"] not in defects]
    egd = _native_stats(kept, medians)
    if whole:
        egd |= _native_full(kept, [t for t in tasks if t not in defects], fill, split - len(defects))
    return out | {f"{k}_egd": egd.get(k) for k in out}


def suite_summary(
    suite: str,
    results: list[dict[str, Any]],
    manifest: dict[str, Any],
    specs: dict[str, dict[str, Any]],
    sources: dict[str, dict[str, Any]],
) -> dict[str, Any]:
    """One suite of ``summary.json`` (``DESIGN.md`` section 3.8)."""
    msuite = next((s for s in manifest.get("suites") or [] if s.get("id") == suite), {})
    excluded = msuite.get("excluded") or []
    agent_name = (manifest.get("agent") or {}).get("name")
    price = (manifest.get("cost") or {}).get("price_table")
    rs = [r for r in results if r["suite"] == suite]

    last: dict[tuple[str, int], dict[str, Any]] = {}
    for r in rs:
        key = (r["task_id"], int(r.get("epoch", 1)))
        if key not in last or int(r.get("attempt", 1)) > int(last[key].get("attempt", 1)):
            last[key] = r
    sc = _scores(last, specs)
    status_counts, scored, means = sc["status_counts"], sc["scored"], sc["means"]
    counted_means, n, ran, incidents = sc["counted_means"], sc["n"], sc["ran"], sc["incidents"]
    pass_rate, ci = sc["pass_rate"], sc["ci95"]

    full_rate = None
    if excluded and _whole_suite(msuite):
        total = int(msuite.get("source_split") or (int(msuite.get("n_tasks") or 0) + len(excluded)))
        full_rate = sum(counted_means.values()) / total if total else None

    task_ids = sorted({r["task_id"] for r in rs})
    suite_specs = {t: specs[t] for t in task_ids if t in specs}
    tool_cov = agent_cov = None
    if msuite.get("tool_coverage") is not None or msuite.get("agent_coverage") is not None:
        # static coverage: the manifest's, computed from the whole suite at run start (DESIGN.md 1.4)
        tool_cov, agent_cov = msuite.get("tool_coverage"), msuite.get("agent_coverage")
    elif msuite.get("source_split") is not None:
        pass  # a suite the static rule gives no coverage (no reference paths)
    elif suite == "paper_repro":
        targets = {_paper_target(t, suite_specs.get(t)) for t in task_ids}
        tot = len(targets)
        cov = {"covered": tot, "total": tot, "rate": 1.0 if tot else None, "basis": "by_construction"}
        tool_cov, agent_cov = dict(cov), dict(cov)
    elif any(((s.get("meta") or {}).get("reference_path")) for s in suite_specs.values()):
        paths = [((s.get("meta") or {}).get("reference_path") or {}) for s in suite_specs.values()]
        total = len(paths) + len(excluded)
        covered = sum(1 for p in paths if p.get("provenance", "none") != "none")
        reach = sum(
            1
            for p in paths
            if p.get("provenance", "none") != "none"
            and not p.get("reference_datasets")
            and not p.get("needs")
        )
        tool_cov = {
            "covered": covered,
            "total": total,
            "rate": covered / total if total else None,
            "basis": "source_split",
        }
        agent_cov = {
            "covered": reach,
            "total": total,
            "rate": reach / total if total else None,
            "basis": "source_split",
        }

    final = [r for r in last.values() if r["status"] in ("passed", "failed")]
    fidelity_rows = [
        r for r in final if r["target"] == "tool" or (agent_name == "scripted" and r["target"] == "agent")
    ]

    def kind_of(c: dict[str, Any]) -> str | None:
        src = sources.get(c.get("source_id") or "")
        return src.get("kind") if src else None

    def cells_of(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
        return [
            c
            for r in rows
            for c in r.get("checks") or []
            if kind_of(c) not in (None, "software", "leaderboard")
        ]

    soft = [[c for c in r.get("checks") or [] if kind_of(c) == "software"] for r in fidelity_rows]
    soft = [cs for cs in soft if cs]
    impl = _ratio(sum(all(c["passed"] for c in cs) for cs in soft), len(soft)) if soft else None
    # Replication fidelity counts only the rows whose specification is the authors' own: a reference path
    # transcribed from the benchmark's script, or a tool task with no reference path (paper_repro's T1, the
    # paper's specification). A path MAGI built (a metadata rule, by hand) is reference-path agreement, a
    # separate number: its misses measure the path as much as the tool (DESIGN.md 1.2, validity-5).
    prov = {r["task_id"]: provenance(specs.get(r["task_id"])) for r in fidelity_rows}
    repl_cells = cells_of(
        [
            r
            for r in fidelity_rows
            if prov[r["task_id"]] == "benchmark_script"
            or (prov[r["task_id"]] is None and r["target"] == "tool")
        ]
    )
    agree_cells = cells_of([r for r in fidelity_rows if prov[r["task_id"]] in RULE_BUILT_PATHS])
    repl = _ratio(sum(1 for c in repl_cells if c["passed"]), len(repl_cells)) if repl_cells else None
    agree = _ratio(sum(1 for c in agree_cells if c["passed"]), len(agree_cells)) if agree_cells else None

    given_t1 = None
    if suite == "paper_repro":
        tool_pass = {
            r["task_id"].rsplit(".", 1)[0]: r["status"] == "passed"
            for r in final
            if r["target"] == "tool" and r["task_id"].endswith(".tool")
        }
        # T2 tasks whose target's T1 passed in this run: k counts those passed in every epoch, rate is the
        # mean over tasks of the per-task mean over epochs
        rows = [
            r_id
            for r_id in sorted(means)
            if r_id.endswith(".agent") and tool_pass.get(r_id.rsplit(".", 1)[0]) is True
        ]
        given_t1 = {
            "k": sum(1 for t in rows if means[t] == 1.0),
            "n": len(rows),
            "rate": _mean([means[t] for t in rows]) if rows else None,
        }

    native = native_summary(final, rs, msuite, specs)

    zero_rows = []
    for s in suite_specs.values():
        if s.get("target") != "agent":
            continue
        figs = [
            c
            for c in s.get("checks") or []
            if c.get("kind") == "answer_figure_close"
            and c.get("required", True)
            and c.get("field", "value") == "value"
        ]
        if figs:
            x = apply_format(0.0, (s.get("meta") or {}).get("answer_format") or {})
            zero_rows.append(1.0 if all(figure_check_passes(c, x, sources) for c in figs) else 0.0)
    baselines = {
        "zero": _mean(zero_rows) if zero_rows else None,
        "naive": pass_rate if agent_name == "naive" else None,
    }

    error_kinds: dict[str, int] = {}
    for r in rs:
        if r.get("error_kind"):
            error_kinds[r["error_kind"]] = error_kinds.get(r["error_kind"], 0) + 1

    tokens = {"input": 0, "output": 0, "thought": 0, "cached": 0}
    for r in last.values():
        u = r.get("usage") or {}
        tokens["input"] += int(u.get("input_tokens", 0))
        tokens["output"] += int(u.get("output_tokens", 0))
        tokens["thought"] += int(u.get("thought_tokens", 0))
        tokens["cached"] += int(u.get("cached_tokens", 0))
    cost = None
    if price:
        cost = round(
            (
                tokens["input"] * float(price["input_per_mtok"])
                + (tokens["output"] + tokens["thought"]) * float(price["output_per_mtok"])
            )
            / 1e6,
            6,
        )

    agent_rows = [r for r in final if r["target"] == "agent"]
    agent = None
    if agent_rows:
        expected = [r for r in agent_rows if (r.get("meta") or {}).get("expected_tool")]
        used = sum(
            1
            for r in expected
            if {r["meta"]["expected_tool"], *r["meta"].get("acceptable_tools", ())}
            & set(r.get("tools_called") or [])
        )
        traps = [r for r in agent_rows if (r.get("meta") or {}).get("trap_tools")]
        sprung = sum(1 for r in traps if set(r["meta"]["trap_tools"]) & set(r.get("tools_called") or []))
        backed = [c for r in agent_rows for c in r.get("checks") or [] if c.get("kind") == "answer_backed"]
        first_tool: dict[str, dict[str, int]] = {}
        for r in expected:
            chosen = next((t for t in r.get("tools_called") or [] if t != "data.describe"), "(none)")
            row = first_tool.setdefault(r["meta"]["expected_tool"], {})
            row[chosen] = row.get(chosen, 0) + 1
        agent = {
            "tasks": len(agent_rows),
            "expected_tool_used": _ratio(used, len(expected)),
            "trap_tool_called": _ratio(sprung, len(traps)),
            "answers_backed": _ratio(sum(1 for c in backed if c["passed"]), len(backed)),
            "first_tool": first_tool,
        }

    return {
        "suite": suite,
        "version": msuite.get("version"),
        "n": n,
        "n_results": len(rs),
        "status_counts": status_counts,
        "pass_rate": pass_rate,
        "pass_rate_incidents_as_fail": sc["pass_rate_incidents_as_fail"],
        "pass_rate_excl_gold_defects": sc["pass_rate_excl_gold_defects"],
        "n_gold_defects": sc["n_gold_defects"],
        "incident_rate": incidents / ran if ran else None,
        "ci95": ci,
        "full_rate": full_rate,
        "tool_coverage": tool_cov,
        "agent_coverage": agent_cov,
        "implementation_fidelity": impl,
        "replication_fidelity": repl,
        "reference_path_agreement": agree,
        "accuracy_given_t1": given_t1,
        "native": native,
        "baselines": baselines,
        "pass_all_epochs": (sum(1 for v in scored.values() if all(x == 1.0 for x in v)) / n) if n else None,
        "failure_classes": sc["failure_classes"],
        "error_kinds": dict(sorted(error_kinds.items())),
        "precision_limited": sorted(
            t for t, s in suite_specs.items() if (s.get("meta") or {}).get("precision_limited")
        ),
        "tokens": tokens,
        "cost": cost,
        "agent": agent,
        "by_kind": by_kind(last, specs),
    }


def summary_from(
    results: list[dict[str, Any]],
    manifest: dict[str, Any],
    specs: dict[str, dict[str, Any]],
    sources: dict[str, dict[str, Any]],
) -> dict[str, Any]:
    """``summary.json``: suite id -> its summary, for every suite of the manifest and of the results."""
    suites = sorted({r["suite"] for r in results} | {s["id"] for s in manifest.get("suites") or []})
    return {s: suite_summary(s, results, manifest, specs, sources) for s in suites}


def _f(x: Any, fmt: str = ".3f") -> str:
    return "—" if x is None else format(x, fmt)


def summary_md(summary: dict[str, Any], manifest: dict[str, Any]) -> str:
    """``summary.md``, rendered from ``summary.json`` alone."""
    lines = [
        f"# Benchmark run {manifest.get('run_id', '')}",
        "",
        f"Status `{manifest.get('status', '')}`, agent `{(manifest.get('agent') or {}).get('name', '')}`, "
        f"commit `{(manifest.get('code') or {}).get('short', '')}`.",
        "",
        "| suite | tasks | pass rate | 95% CI | incidents as fail | incident rate | tokens in / out | cost |",
        "|---|---|---|---|---|---|---|---|",
    ]
    for sid, s in summary.items():
        ci = s.get("ci95")
        ci_txt = "—" if not ci else f"[{ci['low']:.3f}, {ci['high']:.3f}] ({ci['method']})"
        tok = s.get("tokens") or {}
        cells = [
            sid,
            str(s["n"]),
            _f(s.get("pass_rate")),
            ci_txt,
            _f(s.get("pass_rate_incidents_as_fail")),
            _f(s.get("incident_rate")),
            f"{tok.get('input', 0)} / {tok.get('output', 0)}",
            _f(s.get("cost"), ".6f"),
        ]
        lines.append("| " + " | ".join(cells) + " |")
    extra = []
    for sid, s in summary.items():
        bits = []
        kinds = s.get("by_kind") or {}
        if s.get("n_gold_defects"):  # beside the rate as scored, never instead of it
            bits.append(
                f"pass rate excluding {s['n_gold_defects']} gold defect(s) "
                f"{_f(s.get('pass_rate_excl_gold_defects'))}"
            )
        if len(kinds) > 1:  # the rates the pooled row above must never stand for alone
            bits.append(
                "by kind "
                + ", ".join(
                    f"{k} {_f(v.get('pass_rate'))} (n={v.get('n', 0)}"
                    + (
                        f"; excluding {v['n_gold_defects']} gold defect(s) "
                        f"{_f(v.get('pass_rate_excl_gold_defects'))}"
                        if v.get("n_gold_defects")
                        else ""
                    )
                    + ")"
                    for k, v in kinds.items()
                )
            )
        for name in (
            "implementation_fidelity",
            "replication_fidelity",
            "reference_path_agreement",
            "accuracy_given_t1",
        ):
            r = s.get(name)
            if r:
                bits.append(f"{name.replace('_', ' ')} {r['k']}/{r['n']}")
        for name in ("tool_coverage", "agent_coverage"):
            c = s.get(name)
            if c:
                bits.append(f"{name.replace('_', ' ')} {c['covered']}/{c['total']} ({c['basis']})")
        if s.get("failure_classes"):
            bits.append("failures " + ", ".join(f"{k} {v}" for k, v in s["failure_classes"].items()))
        if s.get("error_kinds"):
            bits.append("errors " + ", ".join(f"{k} {v}" for k, v in s["error_kinds"].items()))
        if bits:
            extra.append(f"- {sid}: " + "; ".join(bits))
    if extra:
        lines += ["", *extra]
    lines += ["", "Rendered from summary.json; every number there is recomputable from results.jsonl."]
    return "\n".join(lines) + "\n"


# ----------------------------------------------------------------------------------------------- checking


def _load_dir_json(d: Path) -> dict[str, dict[str, Any]]:
    if not d.is_dir():
        return {}
    return {p.stem: json.loads(p.read_text(encoding="utf-8")) for p in sorted(d.glob("*.json"))}


def check_sums(root: Path) -> list[str]:
    problems = []
    sums = root / "SHA256SUMS"
    if not sums.exists():
        return ["SHA256SUMS is missing"]
    lines = sums.read_text(encoding="utf-8").splitlines()
    listed = []
    for line in lines:
        digest, sep, path = line.partition("  ")
        if not sep or len(digest) != 64 or path.startswith("/") or ".." in Path(path).parts:
            problems.append(f"malformed SHA256SUMS line: {line!r}")
            continue
        listed.append(path)
        p = root / path
        if not p.is_file():
            problems.append(f"MISSING {path}")
        elif sha256(p.read_bytes()) != digest:
            problems.append(f"MISMATCH {path}")
    if listed != sorted(listed):
        problems.append("SHA256SUMS is not sorted by path")
    present = {p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file()}
    unlisted = sorted(present - set(listed) - {"SHA256SUMS"})
    problems += [f"UNLISTED {p}" for p in unlisted]
    return problems


def check_attachments(root: Path) -> list[str]:
    return [
        f"BAD NAME attachments/{p.name}"
        for p in sorted((root / "attachments").glob("*.json"))
        if sha256(p.read_bytes()) != p.stem
    ]


def check_specs(root: Path, results: list[dict[str, Any]], spec_root: Path) -> list[str]:
    problems = []
    for r in results:
        p = spec_root / "specs" / f"{r['task_id']}.json"
        if not p.exists():
            problems.append(f"no spec for {r['task_id']}")
        elif r.get("task_sha256") and sha256(p.read_bytes()) != r["task_sha256"]:
            problems.append(f"task_sha256 of {r['task_id']} does not match specs/{p.name}")
    return problems


def check_registry(root: Path, manifest: dict[str, Any]) -> tuple[list[str], str]:
    reg = root.parent / "registry"
    if not reg.is_dir():
        return [], "no registry folder beside the bundle: chain not checked"
    events = []
    for p in reg.glob("*.json"):
        try:
            events.append((json.loads(p.read_text(encoding="utf-8")), p))
        except ValueError:
            return [f"registry/{p.name} is not JSON"], ""
    events.sort(key=lambda e: (e[0].get("seq", 0), e[1].name))
    problems = []
    prev = None
    for i, (ev, p) in enumerate(events, start=1):
        if ev.get("seq") != i:
            problems.append(f"registry/{p.name}: seq {ev.get('seq')}, expected {i}")
        want = sha256(prev) if prev is not None else None
        if ev.get("prev_sha256") != want:
            problems.append(f"registry/{p.name}: prev_sha256 breaks the chain")
        prev = p.read_bytes()
    run_id = manifest.get("run_id")
    final = next((ev for ev, p in events if ev.get("run_id") == run_id and ev.get("event") == "final"), None)
    if not any(ev.get("run_id") == run_id and ev.get("event") == "started" for ev, _ in events):
        problems.append(f"registry has no started event for {run_id}")
    if final is None:
        problems.append(f"registry has no final event for {run_id}")
    else:
        if final.get("run_digest") != sha256((root / "SHA256SUMS").read_bytes()):
            problems.append("the final registry event names another SHA256SUMS")
        if final.get("manifest_sha256") != sha256((root / "manifest.json").read_bytes()):
            problems.append("the final registry event names another manifest.json")
    return problems, f"registry chain of {len(events)} event(s)"


def verify(root: Path) -> tuple[list[str], list[str]]:
    """(problems, notes) for the bundle at ``root``."""
    notes: list[str] = []
    problems = check_sums(root)
    problems += check_attachments(root)
    try:
        manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
        results = jsonl(root / "results.jsonl")
    except (OSError, ValueError) as exc:
        return problems + [f"cannot read manifest.json or results.jsonl: {exc}"], notes
    data_root = root
    if manifest.get("kind") == "rescore":
        parent = root.parent / str(manifest.get("parent_run_id"))
        if not (parent / "SHA256SUMS").exists():
            return problems + [f"rescore: the parent bundle {parent.name} is not beside this one"], notes
        if sha256((parent / "SHA256SUMS").read_bytes()) != manifest.get("parent_files_sha256"):
            problems.append("rescore: the parent's SHA256SUMS is not the one this child references")
        problems += [f"parent: {p}" for p in check_sums(parent)]
        data_root = parent
    problems += check_specs(root, results, data_root)
    reg_problems, reg_note = check_registry(root, manifest)
    problems += reg_problems
    if reg_note:
        notes.append(reg_note)
    specs = _load_dir_json(data_root / "specs")
    sources = _load_dir_json(data_root / "sources")
    summary = summary_from(results, manifest, specs, sources)
    if pretty(summary) != (root / "summary.json").read_bytes():
        problems.append("summary.json is not what results.jsonl recomputes to")
    if summary_md(summary, manifest).encode() != (root / "summary.md").read_bytes():
        problems.append("summary.md is not what summary.json renders to")
    notes.append(f"run digest {sha256((root / 'SHA256SUMS').read_bytes())}")
    return problems, notes


def main(argv: list[str] | None = None) -> int:
    args = sys.argv[1:] if argv is None else argv
    root = Path(args[0]).resolve() if args else Path(__file__).resolve().parent
    problems, notes = verify(root)
    for n in notes:
        print(n)
    for p in problems:
        print(p)
    print("ok" if not problems else f"{len(problems)} problem(s)")
    return 1 if problems else 0


if __name__ == "__main__":
    sys.exit(main())