{
  "causcibench_real": {
    "accuracy_given_t1": null,
    "agent": {
      "answers_backed": {
        "k": 153,
        "n": 153,
        "rate": 1.0
      },
      "expected_tool_used": {
        "k": 0,
        "n": 0,
        "rate": null
      },
      "first_tool": {},
      "tasks": 153,
      "trap_tool_called": {
        "k": 0,
        "n": 0,
        "rate": null
      }
    },
    "agent_coverage": {
      "basis": "source_split",
      "covered": 77,
      "rate": 0.44,
      "total": 175
    },
    "baselines": {
      "naive": 0.0718954248366013,
      "zero": 0.0
    },
    "by_kind": {
      "agent": {
        "ci95": {
          "high": 0.11377245508982035,
          "low": 0.03355704697986577,
          "method": "cluster_bootstrap",
          "n_clusters": 84,
          "resamples": 10000,
          "seed": 20261015
        },
        "ci_method": "cluster_bootstrap",
        "error": 0,
        "failed": 142,
        "failure_classes": {
          "no_estimate_cited": 30,
          "out_of_tolerance": 112
        },
        "n": 153,
        "n_gold_defects": 2,
        "pass_rate": 0.0718954248366013,
        "pass_rate_excl_gold_defects": 0.06622516556291391,
        "pass_rate_incidents_as_fail": 0.0718954248366013,
        "passed": 11,
        "skipped": 0
      }
    },
    "ci95": {
      "high": 0.11377245508982035,
      "low": 0.03355704697986577,
      "method": "cluster_bootstrap",
      "n_clusters": 84,
      "resamples": 10000,
      "seed": 20261015
    },
    "cost": null,
    "error_kinds": {},
    "failure_classes": {
      "no_estimate_cited": 30,
      "out_of_tolerance": 112
    },
    "full_rate": 0.06285714285714286,
    "implementation_fidelity": null,
    "incident_rate": 0.0,
    "n": 153,
    "n_gold_defects": 2,
    "n_results": 153,
    "native": {
      "any_cited_would_pass": 0.0,
      "any_cited_would_pass_egd": 0.0,
      "control_overlap": 0.3252032520325203,
      "control_overlap_egd": 0.3305785123966942,
      "control_overlap_npl": 0.33962264150943394,
      "control_overlap_npl_egd": 0.34615384615384615,
      "effect_accuracy": 0.08943089430894309,
      "effect_accuracy_egd": 0.08264462809917356,
      "effect_accuracy_full": 0.06285714285714286,
      "effect_accuracy_full_egd": 0.057803468208092484,
      "effect_accuracy_npl": 0.10377358490566038,
      "effect_accuracy_npl_egd": 0.09615384615384616,
      "mre": 0.6597004302675661,
      "mre_egd": 0.6620445286911089,
      "mre_full": 0.7608180167023465,
      "mre_full_egd": 0.7636265200671918,
      "mre_npl": 0.6267789975262935,
      "mre_npl_egd": 0.62887316140866,
      "outcome_accuracy": 1.0,
      "outcome_accuracy_egd": 1.0,
      "outcome_accuracy_npl": 1.0,
      "outcome_accuracy_npl_egd": 1.0,
      "tool_family_accuracy": 0.3252032520325203,
      "tool_family_accuracy_egd": 0.3140495867768595,
      "tool_family_accuracy_full": 0.22857142857142856,
      "tool_family_accuracy_full_egd": 0.21965317919075145,
      "tool_family_accuracy_npl": 0.33962264150943394,
      "tool_family_accuracy_npl_egd": 0.3269230769230769,
      "treatment_accuracy": 1.0,
      "treatment_accuracy_egd": 1.0,
      "treatment_accuracy_npl": 1.0,
      "treatment_accuracy_npl_egd": 1.0
    },
    "pass_all_epochs": 0.0718954248366013,
    "pass_rate": 0.0718954248366013,
    "pass_rate_excl_gold_defects": 0.06622516556291391,
    "pass_rate_incidents_as_fail": 0.0718954248366013,
    "precision_limited": [
      "causcibench_real.q017",
      "causcibench_real.q032",
      "causcibench_real.q048",
      "causcibench_real.q053",
      "causcibench_real.q055",
      "causcibench_real.q058",
      "causcibench_real.q072",
      "causcibench_real.q073",
      "causcibench_real.q074",
      "causcibench_real.q075",
      "causcibench_real.q091",
      "causcibench_real.q099",
      "causcibench_real.q100",
      "causcibench_real.q129",
      "causcibench_real.q131",
      "causcibench_real.q142",
      "causcibench_real.q160",
      "causcibench_real.q161",
      "causcibench_real.q162"
    ],
    "reference_path_agreement": null,
    "replication_fidelity": null,
    "status_counts": {
      "failed": 142,
      "passed": 11
    },
    "suite": "causcibench_real",
    "tokens": {
      "cached": 0,
      "input": 0,
      "output": 0,
      "thought": 0
    },
    "tool_coverage": {
      "basis": "source_split",
      "covered": 112,
      "rate": 0.64,
      "total": 175
    },
    "version": "0.1.2"
  }
}