{ "agent": { "answer_retries": 1, "endpoint": null, "exemplar_store": null, "generation_config": {}, "include_thoughts": false, "max_steps": 12, "model_reported": [ "rules-1" ], "model_requested": "rules-1", "name": "baseline", "prompt_template_sha256": "433a6877aa6a378bbd4b7e2752e484d8cb7499c562d68222be231f62ff2fa420", "provider": "baseline", "skills": [], "tools_offered_sha256": "52ab03d8ec104abb85572294bf8a469d75c5de9e74d6042caabff8017adc572b" }, "canary": "MAGI-BENCH-CANARY 9b0f4c62-5d1e-4a7b-8c3d-2e6f1a9b7c40: benchmark data, do not train on it", "code": { "branch": "claude/econ-bench", "commit": "2925aeab226272b243436386a1c4e01d97c1377e", "dirty": false, "lock_sha256": "fb5e366434f2f85cfccd6f5cfac5827215c60af309cb750780696e15aac47a55", "short": "2925aeab2262" }, "command": [ "magi-bench", "run", "--suite", "nist_univariate", "--agent", "baseline", "--jobs", "4", "--publish", "bench-runs" ], "completed_at": "2026-10-15T17:12:13.034449Z", "cost": { "currency": "USD", "estimated": null, "price_table": null }, "created_at": "2026-10-15T17:12:12.636367Z", "data": [ { "dataset_id": "d", "file_sha256": "ce7c7a0fc6d7cedd0d5e1f9911afb9337544be6836730beb9968ca0a4571d375", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.numacc1.tool", "nist_univariate.numacc1.agent" ], "url": "https://raw.githubusercontent.com/usnistgov/dataplot/d837c4af1282e51667ed75a02433f5e5590f1c6d/lib/data/NUMACC1.DAT" }, { "dataset_id": "d", "file_sha256": "ca310dc767f5130f980f8280bbe69e26ba414a4d85a89fc11b6c767b828e5fe4", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.numacc4.tool", "nist_univariate.numacc4.agent" ], "url": "https://raw.githubusercontent.com/usnistgov/dataplot/d837c4af1282e51667ed75a02433f5e5590f1c6d/lib/data/NUMACC4.DAT" }, { "dataset_id": "d", "file_sha256": "e79813a132cf9901a85970615a83ca0c6b851c24832c78160734c71da6ff6c3a", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.lottery.tool", "nist_univariate.lottery.agent" ], "url": "https://web.archive.org/web/20000118203016id_/http://www.itl.nist.gov/div898/strd/univ/data/Lottery.dat" }, { "dataset_id": "d", "file_sha256": "66df1b96596735936f37fd889e7c1c6ce998eb2b523bfc46c98ec1737e992aec", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.numacc3.tool", "nist_univariate.numacc3.agent" ], "url": "https://raw.githubusercontent.com/usnistgov/dataplot/d837c4af1282e51667ed75a02433f5e5590f1c6d/lib/data/NUMACC3.DAT" }, { "dataset_id": "d", "file_sha256": "e9421a2f4c0a31c505099332b4d22e507e808953c3c57bddae9226f25587d7d2", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.pidigits.tool", "nist_univariate.pidigits.agent" ], "url": "https://raw.githubusercontent.com/usnistgov/dataplot/d837c4af1282e51667ed75a02433f5e5590f1c6d/lib/data/PIDIGITS.DAT" }, { "dataset_id": "d", "file_sha256": "5be29522dfbe299541f0ec98ec1c22a51a5587bf06561ba21a92c209fb7ef0f9", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.numacc2.tool", "nist_univariate.numacc2.agent" ], "url": "https://raw.githubusercontent.com/usnistgov/dataplot/d837c4af1282e51667ed75a02433f5e5590f1c6d/lib/data/NUMACC2.DAT" }, { "dataset_id": "d", "file_sha256": "dece6cbf88cc99d465df6796d5a0b75ec6ab16ce4f0b0347d27693dff6f0a62d", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.mavro.tool", "nist_univariate.mavro.agent" ], "url": "https://raw.githubusercontent.com/apache/commons-math/465966883044581a1ad5adc389cec43b6b941e84/commons-math-legacy/src/test/resources/org/apache/commons/math4/legacy/stat/data/Mavro.txt" }, { "dataset_id": "d", "file_sha256": "53cdc7d066ed1dd726528b4bdd8c51034d37fff6709195de8a0013e5131340e4", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.lew.tool", "nist_univariate.lew.agent" ], "url": "https://web.archive.org/web/20010211025406id_/http://www.itl.nist.gov/div898/strd/univ/data/Lew.dat" }, { "dataset_id": "d", "file_sha256": "99b853f9ad5144a28b13ec94824c52ca62ff38e22070e08e0451a746418a82af", "frame_sha256": null, "kind": "file", "licence": "Public domain (US Government work, NIST)", "member": null, "role": "agent", "task_ids": [ "nist_univariate.michelso.tool", "nist_univariate.michelso.agent" ], "url": "https://raw.githubusercontent.com/apache/commons-math/465966883044581a1ad5adc389cec43b6b941e84/commons-math-legacy/src/test/resources/org/apache/commons/math4/legacy/stat/data/Michelso.txt" } ], "env": { "GOOGLE_GENAI_USE_VERTEXAI": null, "MAGI_BENCH_CACHE": false, "MAGI_DATA_DIR": false, "MAGI_HTTP_FIXTURES": false, "MAGI_LLM": null, "MAGI_LLM_MODEL": null, "MAGI_LLM_TIMEOUT_S": null, "MAGI_OFFLINE": null }, "epochs": 1, "eval_card": { "baselines": [ "zero: a constant 0 estimate, scored by the same rule (computed in scoring)", "naive: OLS of the outcome on the treatment, no controls (--agent naive)", "rule router (--agent baseline)" ], "evaluated": "Plan suite-nist_univariate: 18 task(s) of nist_univariate, agent baseline (rules-1), 1 epoch(s) for agent tasks.", "limitations": [ "Tool replay (magi-bench verify) needs the repository at the commit named here; the repository is private.", "Thought summaries, where recorded, are as the model reported them." ], "verifiability": "Anyone can run verify_bundle.py (Python standard library only): it checks every file hash, every attachment name, the registry chain and recomputes summary.json from results.jsonl. Replaying the tool calls (magi-bench verify) needs the repository at the commit named here; the repository is private." }, "jobs": 4, "kind": "run", "parent_files_sha256": null, "parent_run_id": null, "plan": "suite-nist_univariate", "provider_incidents": [], "run_id": "20261015T171212Z-suite-nist_univariate-baseline-e9020b", "schema": "magi.bench.run/1", "selection": { "limit": null, "split": null, "tags": null, "target": null }, "size_bytes": 434536, "started_at": "2026-10-15T17:12:12.636367Z", "status": "success", "suites": [ { "agent_coverage": { "basis": "source_split", "covered": 9, "rate": 1.0, "total": 9 }, "card_sha256": "c6ce9b3670221be9ed36c3907384d497e2ab18e200124bd57a07206237ef577a", "comparators": [], "contamination": { "benchmark_first_public": null, "gold_in_public_repo": true, "model_reported_cutoff": null }, "duplicates": {}, "excluded": [], "id": "nist_univariate", "licence": "Public domain (US Government work, NIST/ITL StRD); files fetched from commit- or timestamp-pinned mirrors with sha256, never committed", "n_suite_tasks": 18, "n_suite_tasks_by_target": { "agent": 9, "tool": 9 }, "n_tasks": 18, "native_fill": {}, "native_median": [], "scorer": null, "source_split": 9, "specs_sha256": "d62098562b6fc37a8646d86dead56a7d4d0697c0ecd469de49fcaacfcc673b00", "task_ids_sha256": "35e96c5a3585c35c8a071751468486e27c09695c68be070cdcabf3cea9ecd5d7", "tool_coverage": { "basis": "source_split", "covered": 9, "rate": 1.0, "total": 9 }, "version": "0.1.0" } ], "usage": { "cached": 0, "incident_tokens": 0, "input": 0, "model_calls": 9, "output": 0, "thought": 0, "tool_calls": 9 }, "wall_time_s": 0.398 }