{
  "agent": {
    "answer_retries": 1,
    "endpoint": null,
    "exemplar_store": null,
    "generation_config": {},
    "include_thoughts": false,
    "max_steps": 12,
    "model_reported": [
      "reference"
    ],
    "model_requested": "reference",
    "name": "scripted",
    "prompt_template_sha256": "433a6877aa6a378bbd4b7e2752e484d8cb7499c562d68222be231f62ff2fa420",
    "provider": "scripted",
    "skills": [],
    "tools_offered_sha256": "52ab03d8ec104abb85572294bf8a469d75c5de9e74d6042caabff8017adc572b"
  },
  "canary": "MAGI-BENCH-CANARY 9b0f4c62-5d1e-4a7b-8c3d-2e6f1a9b7c40: benchmark data, do not train on it",
  "code": {
    "branch": "claude/econ-bench",
    "commit": "62cdbe0860ad0472e1d2120e93d3c4f29f7688eb",
    "dirty": false,
    "lock_sha256": "fb5e366434f2f85cfccd6f5cfac5827215c60af309cb750780696e15aac47a55",
    "short": "62cdbe0860ad"
  },
  "command": [
    "magi-bench",
    "run",
    "--suite",
    "qrdata_socsci",
    "--agent",
    "scripted",
    "--jobs",
    "4",
    "--publish",
    "bench-runs"
  ],
  "completed_at": "2026-10-15T17:00:13.936818Z",
  "cost": {
    "currency": "USD",
    "estimated": null,
    "price_table": null
  },
  "created_at": "2026-10-15T17:00:10.769903Z",
  "data": [
    {
      "dataset_id": "FLCensusVTD",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/FLCensusVTD.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.409",
        "qrdata_socsci.410"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "FLVoters",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/FLVoters.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.406",
        "qrdata_socsci.407",
        "qrdata_socsci.408"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "FLVoters",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/FLVoters.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.409",
        "qrdata_socsci.410"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "MPs",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/MPs.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.402"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "USGini",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/USGini.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.385"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "afghan",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/afghan.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.383"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "afghan",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/afghan.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.384"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "afghan-village",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/afghan-village.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.384"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "ak91",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/ak91.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.362"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "ak91_iv",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/ak91.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.362"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "app_engagement_push",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/app_engagement_push.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.363",
        "qrdata_socsci.364"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "billboard_impact",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/billboard_impact.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.368"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "collections_email",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/collections_email.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.356",
        "qrdata_socsci.359",
        "qrdata_socsci.360",
        "qrdata_socsci.361"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "drinking",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/drinking.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.370"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "drinking_log",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/drinking.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.370"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "face",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/face.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.390",
        "qrdata_socsci.391",
        "qrdata_socsci.392"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "face_margin",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/face.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.391"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "florentine",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/florentine.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.403",
        "qrdata_socsci.404"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "florida",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/florida.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.396",
        "qrdata_socsci.397"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "hospital_treatment",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/hospital_treatment.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.357",
        "qrdata_socsci.358"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "ice_cream_sales",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/ice_cream_sales.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.372"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "ice_cream_sales_rnd",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/ice_cream_sales_rnd.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.372"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "invest_email_rnd_test",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/invest_email_rnd_test.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.371"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "invest_email_rnd_train",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/invest_email_rnd_train.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.371"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "learning_mindset",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/learning_mindset.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.366",
        "qrdata_socsci.367"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "medicine_impact_recovery",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/medicine_impact_recovery.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.365"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "minwage",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/minwage.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.380",
        "qrdata_socsci.381",
        "qrdata_socsci.382"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "minwage_props",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/minwage.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.380",
        "qrdata_socsci.381",
        "qrdata_socsci.382"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "names",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/names.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.406",
        "qrdata_socsci.407",
        "qrdata_socsci.408",
        "qrdata_socsci.409",
        "qrdata_socsci.410"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "online_classroom",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/online_classroom.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.352"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "polls08",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/polls08.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.386",
        "qrdata_socsci.387",
        "qrdata_socsci.388",
        "qrdata_socsci.389"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "pres08",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/pres08.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.393",
        "qrdata_socsci.394",
        "qrdata_socsci.395"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "pres08",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/pres08.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.386",
        "qrdata_socsci.387",
        "qrdata_socsci.388",
        "qrdata_socsci.389"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "pres12",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/pres12.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.393",
        "qrdata_socsci.394",
        "qrdata_socsci.395"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "resume",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/resume.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.376",
        "qrdata_socsci.377",
        "qrdata_socsci.378",
        "qrdata_socsci.379"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "resume_white",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/resume.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.376",
        "qrdata_socsci.377"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "smoking2",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/smoking2.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.369",
        "qrdata_socsci.373",
        "qrdata_socsci.374"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "social",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/social.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.399",
        "qrdata_socsci.400",
        "qrdata_socsci.401"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "social_neighbors",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/social.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.399"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "trainee_unique_on_age",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/trainee_unique_on_age.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.375"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "twitter-following",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/twitter-following.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.405"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "twitter-senator",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/twitter-senator.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.405"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "wage",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/wage.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.353",
        "qrdata_socsci.354"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "wage",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/wage.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.355"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "wage_grad12",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/wage.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.355"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "wage_hourly",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/wage.csv",
      "role": "reference",
      "task_ids": [
        "qrdata_socsci.353",
        "qrdata_socsci.354"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    },
    {
      "dataset_id": "women",
      "file_sha256": "3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970",
      "frame_sha256": null,
      "kind": "file",
      "licence": "CC BY-NC 4.0",
      "member": "data/women.csv",
      "role": "agent",
      "task_ids": [
        "qrdata_socsci.398"
      ],
      "url": "https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"
    }
  ],
  "env": {
    "GOOGLE_GENAI_USE_VERTEXAI": null,
    "MAGI_BENCH_CACHE": false,
    "MAGI_DATA_DIR": false,
    "MAGI_HTTP_FIXTURES": false,
    "MAGI_LLM": null,
    "MAGI_LLM_MODEL": null,
    "MAGI_LLM_TIMEOUT_S": null,
    "MAGI_OFFLINE": null
  },
  "epochs": 1,
  "eval_card": {
    "baselines": [
      "zero: a constant 0 estimate, scored by the same rule (computed in scoring)",
      "naive: OLS of the outcome on the treatment, no controls (--agent naive)",
      "rule router (--agent baseline)"
    ],
    "evaluated": "Plan suite-qrdata_socsci: 59 task(s) of qrdata_socsci, agent scripted (reference), 1 epoch(s) for agent tasks.",
    "limitations": [
      "Tool replay (magi-bench verify) needs the repository at the commit named here; the repository is private.",
      "Thought summaries, where recorded, are as the model reported them."
    ],
    "verifiability": "Anyone can run verify_bundle.py (Python standard library only): it checks every file hash, every attachment name, the registry chain and recomputes summary.json from results.jsonl. Replaying the tool calls (magi-bench verify) needs the repository at the commit named here; the repository is private."
  },
  "jobs": 4,
  "kind": "run",
  "parent_files_sha256": null,
  "parent_run_id": null,
  "plan": "suite-qrdata_socsci",
  "provider_incidents": [],
  "run_id": "20261015T170010Z-suite-qrdata_socsci-scripted-7912b3",
  "schema": "magi.bench.run/1",
  "selection": {
    "limit": null,
    "split": null,
    "tags": null,
    "target": null
  },
  "size_bytes": 1332180,
  "started_at": "2026-10-15T17:00:10.769903Z",
  "status": "success",
  "suites": [
    {
      "agent_coverage": {
        "basis": "source_split",
        "covered": 7,
        "rate": 0.11864406779661017,
        "total": 59
      },
      "card_sha256": "40902ea4325331a03a4c525c49381ce814d2ad34388c0444c690557558be943d",
      "comparators": [
        "qrdata.leaderboard.gpt4_code_interpreter",
        "qrdata.leaderboard.gemini_pro_react"
      ],
      "contamination": {
        "benchmark_first_public": "2024-02-27",
        "gold_in_public_repo": true,
        "model_reported_cutoff": null
      },
      "duplicates": {
        "qrdata_socsci.362": [
          "paper_repro.angrist_krueger_1991.tab5_c1.tool",
          "paper_repro.angrist_krueger_1991.tab5_c2.tool"
        ],
        "qrdata_socsci.380": [
          "causcibench_real.q028",
          "paper_repro.card_krueger_1994.tab3_r3_ciii.tool",
          "paper_repro.card_krueger_1994.tab3_r4_ciii.tool"
        ],
        "qrdata_socsci.381": [
          "causcibench_real.q028",
          "paper_repro.card_krueger_1994.tab3_r3_ciii.tool",
          "paper_repro.card_krueger_1994.tab3_r4_ciii.tool"
        ],
        "qrdata_socsci.382": [
          "causcibench_real.q028",
          "paper_repro.card_krueger_1994.tab3_r3_ciii.tool",
          "paper_repro.card_krueger_1994.tab3_r4_ciii.tool"
        ]
      },
      "excluded": [],
      "id": "qrdata_socsci",
      "licence": "CC BY-NC 4.0 (QRData); question text quoted with attribution, data files fetched, never hosted",
      "n_suite_tasks": 59,
      "n_suite_tasks_by_target": {
        "agent": 59
      },
      "n_tasks": 59,
      "native_fill": {
        "correct": 0.0
      },
      "native_median": [],
      "scorer": {
        "repo": "https://github.com/xxxiaol/QRData",
        "revision": "de450af45ff7101b328bb064c6b475f73414a7ed",
        "sha256": "d6e5c1f7535ac9413951996c15544beb6f184244acb2da56e1fcd0942837e33d"
      },
      "source_split": 59,
      "specs_sha256": "0aecf0695ece0de593afb2d54717dcdfbeed96ecc47fcbc815cd6d10578053bc",
      "task_ids_sha256": "a86fa465388a17115d9276d295b68ecd3ab13bc8927f4b122f10b65df738b63c",
      "tool_coverage": {
        "basis": "source_split",
        "covered": 19,
        "rate": 0.3220338983050847,
        "total": 59
      },
      "version": "0.1.2"
    }
  ],
  "usage": {
    "cached": 0,
    "incident_tokens": 0,
    "input": 0,
    "model_calls": 38,
    "output": 0,
    "thought": 0,
    "tool_calls": 19
  },
  "wall_time_s": 3.167
}