{ "agent": { "answer_retries": 1, "endpoint": null, "exemplar_store": null, "generation_config": {}, "include_thoughts": false, "max_steps": 12, "model_reported": [ "rules-1" ], "model_requested": "rules-1", "name": "baseline", "prompt_template_sha256": "433a6877aa6a378bbd4b7e2752e484d8cb7499c562d68222be231f62ff2fa420", "provider": "baseline", "skills": [], "tools_offered_sha256": "52ab03d8ec104abb85572294bf8a469d75c5de9e74d6042caabff8017adc572b" }, "canary": "MAGI-BENCH-CANARY 9b0f4c62-5d1e-4a7b-8c3d-2e6f1a9b7c40: benchmark data, do not train on it", "code": { "branch": "claude/econ-bench", "commit": "62cdbe0860ad0472e1d2120e93d3c4f29f7688eb", "dirty": false, "lock_sha256": "fb5e366434f2f85cfccd6f5cfac5827215c60af309cb750780696e15aac47a55", "short": "62cdbe0860ad" }, "command": [ "magi-bench", "run", "--suite", "paper_repro", "--agent", "baseline", "--jobs", "4", "--publish", "bench-runs" ], "completed_at": "2026-10-15T16:59:24.791457Z", "cost": { "currency": "USD", "estimated": null, "price_table": null }, "created_at": "2026-10-15T16:59:11.596437Z", "data": [ { "dataset_id": "asciiqob", "file_sha256": "0b6c505e52fd6c9f6194c356c91688acf2f8937b5023a84e754e942c9a2f2dcc", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data archive); fetched, not redistributed.", "member": "asciiqob.txt", "role": "agent", "task_ids": [ "paper_repro.angrist_krueger_1991.tab5_c1.tool", "paper_repro.angrist_krueger_1991.tab5_c1.agent", "paper_repro.angrist_krueger_1991.tab5_c1.design", "paper_repro.angrist_krueger_1991.tab5_c2.tool", "paper_repro.angrist_krueger_1991.tab5_c2.agent", "paper_repro.angrist_krueger_1991.tab5_c2.design" ], "url": "https://economics.mit.edu/sites/default/files/publications/asciiqob.zip" }, { "dataset_id": "bacon_example", "file_sha256": "5a555448e605f75603160be20f56ca343600ec81863f3060994bac1285c29094", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's site); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.goodman_bacon_2021.fig5_dd.tool", "paper_repro.goodman_bacon_2021.fig5_dd.agent", "paper_repro.goodman_bacon_2021.fig5_dd.design" ], "url": "http://pped.org/bacon_example.dta" }, { "dataset_id": "bcg", "file_sha256": "8508024604252ddecbbba88a5ebbbf76ddec7d32f9e5df56521f602ce5398fdb", "frame_sha256": null, "kind": "file", "licence": "metadat R package (GPL-2 or later); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.colditz_1994.trials_rr.tool" ], "url": "https://raw.githubusercontent.com/wviechtb/metadat/b19c23b4c09e16d0c14a31af94fca190c7dd0357/data/dat.bcg.rda" }, { "dataset_id": "card", "file_sha256": "326dad817467b20d51bdfd015033e3e3e313931f51f812c38c0776249e7fd711", "frame_sha256": null, "kind": "file", "licence": "GPL-3 (R package wooldridge); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.card_1995.ex15_4_iv.tool", "paper_repro.card_1995.ex15_4_iv.agent", "paper_repro.card_1995.ex15_4_iv.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/wooldridge/card.csv" }, { "dataset_id": "caschool", "file_sha256": "644e606cd8ff21557375a091f744f623323bb84de99da616923a97942e4fe54b", "frame_sha256": null, "kind": "file", "licence": "R package Ecdat (GPL-2 | GPL-3), via Rdatasets; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.stock_watson_caschool.ch5_str.tool", "paper_repro.stock_watson_caschool.ch5_str.agent", "paper_repro.stock_watson_caschool.ch5_str.design", "paper_repro.stock_watson_caschool.tab7_1_c2.tool", "paper_repro.stock_watson_caschool.tab7_1_c2.agent", "paper_repro.stock_watson_caschool.tab7_1_c2.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/Ecdat/Caschool.csv" }, { "dataset_id": "castle", "file_sha256": "804633c161827b6c0824f86f239046386d1a8266a866f83bf5ddb2aa762a5f29", "frame_sha256": null, "kind": "file", "licence": "No licence stated (Mixtape repository); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.cheng_hoekstra_2013.tab5_pa_c1.tool", "paper_repro.cheng_hoekstra_2013.tab5_pa_c1.agent", "paper_repro.cheng_hoekstra_2013.tab5_pa_c1.design", "paper_repro.cheng_hoekstra_2013.tab5_pa_c2.tool", "paper_repro.cheng_hoekstra_2013.tab5_pa_c2.agent", "paper_repro.cheng_hoekstra_2013.tab5_pb_c1.tool", "paper_repro.cheng_hoekstra_2013.tab5_pb_c1.agent", "paper_repro.cheng_hoekstra_2013.tab5_pb_c2.tool", "paper_repro.cheng_hoekstra_2013.tab5_pb_c2.agent" ], "url": "https://raw.githubusercontent.com/scunning1975/mixtape/9471dd0c45477386bf208516882895af340f3d33/castle.dta" }, { "dataset_id": "cigarettes", "file_sha256": "629e887a8104d483d450a9bbe487cb5613e62d116c4ecc84d6c8893b7393fcd2", "frame_sha256": null, "kind": "file", "licence": "GPL-2 | GPL-3 (R package AER); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.stock_watson_2007.eq12_15.tool", "paper_repro.stock_watson_2007.eq12_15.agent", "paper_repro.stock_watson_2007.eq12_15.design", "paper_repro.stock_watson_2007.eq12_16.tool", "paper_repro.stock_watson_2007.eq12_16.agent" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/AER/CigarettesSW.csv" }, { "dataset_id": "fatalities", "file_sha256": "ae265c1b14b4a023179b3af5db33b6a659ccf804987f26e8349826d4083ee149", "frame_sha256": null, "kind": "file", "licence": "R package AER (GPL-2 | GPL-3), via Rdatasets; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.stock_watson_fatalities.tab10_1_c2.tool", "paper_repro.stock_watson_fatalities.tab10_1_c2.agent", "paper_repro.stock_watson_fatalities.tab10_1_c2.design", "paper_repro.stock_watson_fatalities.tab10_1_c3.tool", "paper_repro.stock_watson_fatalities.tab10_1_c3.agent", "paper_repro.stock_watson_fatalities.tab10_1_c3.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/AER/Fatalities.csv" }, { "dataset_id": "fertility", "file_sha256": "ca9be592b79dddbc2f49ff80f45d0dbe31aac4afb57ff88dec57376c3f3e3452", "frame_sha256": null, "kind": "file", "licence": "GPL-2 | GPL-3 (R package AER); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.angrist_evans_1998.tab6_c4.tool", "paper_repro.angrist_evans_1998.tab6_c4.agent", "paper_repro.angrist_evans_1998.tab6_c4.design", "paper_repro.angrist_evans_1998.tab7_c5_weeks.tool", "paper_repro.angrist_evans_1998.tab7_c5_weeks.agent", "paper_repro.angrist_evans_1998.tab7_c5_weeks.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/AER/Fertility.csv" }, { "dataset_id": "final5", "file_sha256": "bf443edee9033926b4e74dc2c17a98b04405b899735bf4c372e0e6d468431a54", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data archive); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.angrist_lavy_1999.tab4_c1.tool", "paper_repro.angrist_lavy_1999.tab4_c1.agent", "paper_repro.angrist_lavy_1999.tab4_c1.design", "paper_repro.angrist_lavy_1999.tab4_c7.tool", "paper_repro.angrist_lavy_1999.tab4_c7.agent", "paper_repro.angrist_lavy_1999.tab4_c7.design" ], "url": "https://economics.mit.edu/sites/default/files/inline-files/final5.dta" }, { "dataset_id": "gov_transfers", "file_sha256": "51312ecd5b3cb22b5dc027966a81e0a8579f24a708ce4b476674fc527652ebd0", "frame_sha256": null, "kind": "file", "licence": "causaldata R package, MIT; fetched from Rdatasets, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.mmv_2011.tab20_1_c1.tool", "paper_repro.mmv_2011.tab20_1_c1.agent", "paper_repro.mmv_2011.tab20_1_c1.design", "paper_repro.mmv_2011.tab20_1_c2.tool", "paper_repro.mmv_2011.tab20_1_c2.agent" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/causaldata/gov_transfers.csv" }, { "dataset_id": "growthdj", "file_sha256": "3d06303f1da9959175e990eb068bf242fac0e9bc1f9daad1f5fffe96d100e796", "frame_sha256": null, "kind": "file", "licence": "R package AER (GPL-2 | GPL-3), via Rdatasets; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.mrw_1992.tab1_nonoil_ls.tool", "paper_repro.mrw_1992.tab1_nonoil_ls.agent", "paper_repro.mrw_1992.tab1_nonoil_ls.design", "paper_repro.mrw_1992.tab1_nonoil_lngd.tool", "paper_repro.mrw_1992.tab1_nonoil_lngd.agent", "paper_repro.mrw_1992.tab1_nonoil_lngd.design", "paper_repro.mrw_1992.tab1_inter_ls.tool", "paper_repro.mrw_1992.tab1_inter_ls.agent", "paper_repro.mrw_1992.tab1_inter_ls.design", "paper_repro.mrw_1992.tab1_inter_lngd.tool", "paper_repro.mrw_1992.tab1_inter_lngd.agent", "paper_repro.mrw_1992.tab1_inter_lngd.design", "paper_repro.mrw_1992.tab1_oecd_ls.tool", "paper_repro.mrw_1992.tab1_oecd_ls.agent", "paper_repro.mrw_1992.tab1_oecd_ls.design", "paper_repro.mrw_1992.tab1_oecd_lngd.tool", "paper_repro.mrw_1992.tab1_oecd_lngd.agent", "paper_repro.mrw_1992.tab1_oecd_lngd.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/AER/GrowthDJ.csv" }, { "dataset_id": "grunfeld", "file_sha256": "12ff1bcf9491d2349115203f3bb7212c158754051ab180a814585d8656458204", "frame_sha256": null, "kind": "file", "licence": "R package plm (GPL-2 | GPL-3), via Rdatasets; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.baltagi_grunfeld.tab2_1_within_value.tool", "paper_repro.baltagi_grunfeld.tab2_1_within_value.agent", "paper_repro.baltagi_grunfeld.tab2_1_within_value.design", "paper_repro.baltagi_grunfeld.tab2_1_within_capital.tool", "paper_repro.baltagi_grunfeld.tab2_1_within_capital.agent", "paper_repro.baltagi_grunfeld.tab2_1_within_capital.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/plm/Grunfeld.csv" }, { "dataset_id": "jtrain", "file_sha256": "2820079b04c953a3555a3722e4b14430bc9efedfd6870715a863717de8177383", "frame_sha256": null, "kind": "file", "licence": "R package wooldridge (GPL-3), via Rdatasets; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.wooldridge_jtrain.ex14_1_grant.tool", "paper_repro.wooldridge_jtrain.ex14_1_grant.agent", "paper_repro.wooldridge_jtrain.ex14_1_grant.design", "paper_repro.wooldridge_jtrain.ex14_1_grant_1.tool", "paper_repro.wooldridge_jtrain.ex14_1_grant_1.agent", "paper_repro.wooldridge_jtrain.ex14_1_grant_1.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/wooldridge/jtrain.csv" }, { "dataset_id": "kielmc", "file_sha256": "1ad63b59aff51c713725147032dd5803f3dbfc97163eabcde3b5f731ebbf03e0", "frame_sha256": null, "kind": "file", "licence": "R package wooldridge (GPL-3), via Rdatasets; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.kiel_mcclain_1995.ex13_3_rprice.tool", "paper_repro.kiel_mcclain_1995.ex13_3_rprice.agent", "paper_repro.kiel_mcclain_1995.ex13_3_rprice.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/cc03c29690889dbe83089f5206e2422db8c3f71f/csv/wooldridge/kielmc.csv" }, { "dataset_id": "lmb", "file_sha256": "58cf9d8ef80b2871a96c00844fae975051a5aac443dff3d317fe92ffd6a516b7", "frame_sha256": null, "kind": "file", "licence": "causaldata R package, MIT; fetched from Rdatasets, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.lmb_2004.tab1_c1.tool", "paper_repro.lmb_2004.tab1_c1.agent", "paper_repro.lmb_2004.tab1_c1.design", "paper_repro.lmb_2004.tab1_c3.tool", "paper_repro.lmb_2004.tab1_c3.agent", "paper_repro.lmb_2004.tab1_c3.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/causaldata/close_elections_lmb.csv" }, { "dataset_id": "maketable2", "file_sha256": "f8213d9b42290cf0d3a2e968483bf3b5964e33261b8e48f92e9ea4c87c709b92", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data archive); fetched, not redistributed.", "member": "maketable2.dta", "role": "agent", "task_ids": [ "paper_repro.ajr_2001.tab2_c2.tool", "paper_repro.ajr_2001.tab2_c2.agent", "paper_repro.ajr_2001.tab2_c2.design" ], "url": "https://www.dropbox.com/s/1s1yb7a3hkjwzjh/maketable2.zip?dl=1" }, { "dataset_id": "maketable4", "file_sha256": "cd3a1cb197e3065411fce6fabbe06161555468049e6810a28c608d6e361aed34", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data archive); fetched, not redistributed.", "member": "maketable4.dta", "role": "agent", "task_ids": [ "paper_repro.ajr_2001.tab4_pa_c1.tool", "paper_repro.ajr_2001.tab4_pa_c1.agent", "paper_repro.ajr_2001.tab4_pa_c1.design" ], "url": "https://www.dropbox.com/s/mntre4yhzothfgq/maketable4.zip?dl=1" }, { "dataset_id": "mpdta", "file_sha256": "0be3bc5eaaedaa50e047deaf45ad0e193e7210df1644103c63a6c1baa75965d4", "frame_sha256": null, "kind": "file", "licence": "R package did (GPL-2); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.callaway_santanna_2021.mpdta_att_gt.tool", "paper_repro.callaway_santanna_2021.mpdta_att_gt.agent", "paper_repro.callaway_santanna_2021.mpdta_att_gt.design", "paper_repro.callaway_santanna_2021.mpdta_event_time.tool", "paper_repro.callaway_santanna_2021.mpdta_event_time.agent" ], "url": "https://raw.githubusercontent.com/bcallaway11/did/9c930239f718c4196353aa27525c09f458f29712/data/mpdta.rda" }, { "dataset_id": "mroz", "file_sha256": "bcfc04b6a87746b185ed972f1149c459cc1af5a6ff44443ce5dd004eb8204a78", "frame_sha256": null, "kind": "file", "licence": "GPL-3 (R package wooldridge); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.mroz_1987.ex15_1.tool", "paper_repro.mroz_1987.ex15_1.agent", "paper_repro.mroz_1987.ex15_1.design", "paper_repro.mroz_1987.ex15_5.tool", "paper_repro.mroz_1987.ex15_5.agent", "paper_repro.mroz_1987.ex15_5.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/1dcc2bf5f955cc1224a3e1307256e1fe86b68dae/csv/wooldridge/mroz.csv" }, { "dataset_id": "munic", "file_sha256": "5cd5b8796355e025807dc303263b1fbb16f442c3c7551f14ee3557768a310894", "frame_sha256": null, "kind": "file", "licence": "Econometric Society supplemental material, copyrighted; fetched, not redistributed.", "member": "munic.dta", "role": "agent", "task_ids": [ "paper_repro.fujiwara_2015.tab2_pa_valid98_c1.tool", "paper_repro.fujiwara_2015.tab2_pa_valid98_c1.agent", "paper_repro.fujiwara_2015.tab2_pa_valid98_c1.design", "paper_repro.fujiwara_2015.tab2_pa_valid98_c2.tool", "paper_repro.fujiwara_2015.tab2_pa_valid98_c2.agent", "paper_repro.fujiwara_2015.tab2_pa_valid98_c3.tool", "paper_repro.fujiwara_2015.tab2_pa_valid98_c3.agent", "paper_repro.fujiwara_2015.tab2_pb_valid94_c1.tool", "paper_repro.fujiwara_2015.tab2_pb_valid94_c1.agent", "paper_repro.fujiwara_2015.tab2_pb_valid94_c1.design", "paper_repro.fujiwara_2015.tab2_pb_valid94_c2.tool", "paper_repro.fujiwara_2015.tab2_pb_valid94_c2.agent", "paper_repro.fujiwara_2015.tab2_pb_valid94_c3.tool", "paper_repro.fujiwara_2015.tab2_pb_valid94_c3.agent", "paper_repro.fujiwara_2015.tab2_pb_valid02_c1.tool", "paper_repro.fujiwara_2015.tab2_pb_valid02_c1.agent", "paper_repro.fujiwara_2015.tab2_pb_valid02_c1.design", "paper_repro.fujiwara_2015.tab2_pb_valid02_c2.tool", "paper_repro.fujiwara_2015.tab2_pb_valid02_c2.agent", "paper_repro.fujiwara_2015.tab2_pb_valid02_c3.tool", "paper_repro.fujiwara_2015.tab2_pb_valid02_c3.agent" ], "url": "https://www.econometricsociety.org/publications/econometrica/2015/03/01/voting-technology-political-responsiveness-and-infant-health/supp/11520_Data_and_Programs.zip" }, { "dataset_id": "njmin_balanced", "file_sha256": "41a29aec6b489658b20030d044a2b20a5f5a04b871cde41df057afd6013ba587", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data page); fetched, not redistributed.", "member": "public.dat", "role": "agent", "task_ids": [ "paper_repro.card_krueger_1994.tab3_r4_ciii.tool", "paper_repro.card_krueger_1994.tab3_r4_ciii.agent" ], "url": "https://davidcard.berkeley.edu/data_sets/njmin.zip" }, { "dataset_id": "njmin_long", "file_sha256": "41a29aec6b489658b20030d044a2b20a5f5a04b871cde41df057afd6013ba587", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data page); fetched, not redistributed.", "member": "public.dat", "role": "agent", "task_ids": [ "paper_repro.card_krueger_1994.tab3_r3_ciii.tool", "paper_repro.card_krueger_1994.tab3_r3_ciii.agent", "paper_repro.card_krueger_1994.tab3_r3_ciii.design" ], "url": "https://davidcard.berkeley.edu/data_sets/njmin.zip" }, { "dataset_id": "nsw", "file_sha256": "3ee014cb0010e2e7fb990ac497f4217968733e2c9abfa4de84e83f524898c24e", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data page); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.dehejia_wahba_1999.tab2_pa_c1.tool", "paper_repro.dehejia_wahba_1999.tab2_pa_c1.agent", "paper_repro.dehejia_wahba_1999.tab2_pa_c1.design" ], "url": "https://users.nber.org/~rdehejia/data/nsw.dta" }, { "dataset_id": "nsw_dw", "file_sha256": "d1bd2680a1c6f799f1c6d2455bf29633fdf19be01cb19490621c20a560b4e072", "frame_sha256": null, "kind": "file", "licence": "No licence stated (author's data page); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.dehejia_wahba_1999.tab2_pb_c1.tool", "paper_repro.dehejia_wahba_1999.tab2_pb_c1.agent", "paper_repro.dehejia_wahba_1999.tab2_pb_c1.design" ], "url": "https://users.nber.org/~rdehejia/data/nsw_dw.dta" }, { "dataset_id": "organ_donations", "file_sha256": "b758f11dd4f32c404877bc0b79a09b09d6e074318d2f91030060c9adb26fe56a", "frame_sha256": null, "kind": "file", "licence": "R package causaldata (MIT), via Rdatasets; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.kessler_roth_2014.t18_2_twfe.tool", "paper_repro.kessler_roth_2014.t18_2_twfe.agent", "paper_repro.kessler_roth_2014.t18_2_twfe.design" ], "url": "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/cc03c29690889dbe83089f5206e2422db8c3f71f/csv/causaldata/organ_donations.csv" }, { "dataset_id": "senate", "file_sha256": "def1fc6b8ae5af273f2140aa449c81b3ecfe1d0f325aa80ceccb0d9638a2ad44", "frame_sha256": null, "kind": "file", "licence": "rdrobust package, GPL-2; fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.ccft_2017.senate_h16794.tool", "paper_repro.ccft_2017.senate_h16794.agent", "paper_repro.ccft_2017.senate_h16794.design" ], "url": "https://raw.githubusercontent.com/rdpackages/rdrobust/7df239395c4afce2a58e22ef16a5817341683955/R/rdrobust_senate.csv" }, { "dataset_id": "thornton", "file_sha256": "1e05f2cda26106ca979ad70319cae69c237f9fd3369e429d86a4ec5062afb949", "frame_sha256": null, "kind": "file", "licence": "No licence stated (mixtape repository; original data openICPSR 113268); fetched, not redistributed.", "member": null, "role": "agent", "task_ids": [ "paper_repro.thornton_2008.tab4_c1.tool", "paper_repro.thornton_2008.tab4_c1.agent", "paper_repro.thornton_2008.tab4_c1.design" ], "url": "https://raw.githubusercontent.com/scunning1975/mixtape/ca4279a87a6f0759f6b6f02841a53bdd68e27d3c/thornton_hiv.dta" } ], "env": { "GOOGLE_GENAI_USE_VERTEXAI": null, "MAGI_BENCH_CACHE": false, "MAGI_DATA_DIR": false, "MAGI_HTTP_FIXTURES": false, "MAGI_LLM": null, "MAGI_LLM_MODEL": null, "MAGI_LLM_TIMEOUT_S": null, "MAGI_OFFLINE": null }, "epochs": 1, "eval_card": { "baselines": [ "zero: a constant 0 estimate, scored by the same rule (computed in scoring)", "naive: OLS of the outcome on the treatment, no controls (--agent naive)", "rule router (--agent baseline)" ], "evaluated": "Plan suite-paper_repro: 153 task(s) of paper_repro, agent baseline (rules-1), 1 epoch(s) for agent tasks.", "limitations": [ "Tool replay (magi-bench verify) needs the repository at the commit named here; the repository is private.", "Thought summaries, where recorded, are as the model reported them." ], "verifiability": "Anyone can run verify_bundle.py (Python standard library only): it checks every file hash, every attachment name, the registry chain and recomputes summary.json from results.jsonl. Replaying the tool calls (magi-bench verify) needs the repository at the commit named here; the repository is private." }, "jobs": 4, "kind": "run", "parent_files_sha256": null, "parent_run_id": null, "plan": "suite-paper_repro", "provider_incidents": [], "run_id": "20261015T165911Z-suite-paper_repro-baseline-8b4f85", "schema": "magi.bench.run/1", "selection": { "limit": null, "split": null, "tags": null, "target": null }, "size_bytes": 6993593, "started_at": "2026-10-15T16:59:11.596437Z", "status": "success", "suites": [ { "agent_coverage": { "basis": "by_construction", "covered": 55, "rate": 0.9821428571428571, "total": 56 }, "card_sha256": "6aad5845dbf456b00da7edabcecc1234634bb309b5bf36d6dcd07a220cb67c7f", "comparators": [], "contamination": { "benchmark_first_public": "2021-12", "gold_in_public_repo": true, "model_reported_cutoff": null }, "duplicates": { "paper_repro.angrist_krueger_1991.tab5_c1.tool": [ "qrdata_socsci.362" ], "paper_repro.angrist_krueger_1991.tab5_c2.tool": [ "qrdata_socsci.362" ], "paper_repro.angrist_lavy_1999.tab4_c1.tool": [ "causcibench_real.q042", "causcibench_real.q043" ], "paper_repro.angrist_lavy_1999.tab4_c7.tool": [ "causcibench_real.q042", "causcibench_real.q043" ], "paper_repro.card_1995.ex15_4_iv.tool": [ "causcibench_real.q009" ], "paper_repro.card_krueger_1994.tab3_r3_ciii.tool": [ "causcibench_real.q028", "qrdata_socsci.380", "qrdata_socsci.381", "qrdata_socsci.382" ], "paper_repro.card_krueger_1994.tab3_r4_ciii.tool": [ "causcibench_real.q028", "qrdata_socsci.380", "qrdata_socsci.381", "qrdata_socsci.382" ], "paper_repro.cheng_hoekstra_2013.tab5_pa_c1.tool": [ "causcibench_real.q015", "causcibench_real.q016", "causcibench_real.q017", "causcibench_real.q018" ], "paper_repro.cheng_hoekstra_2013.tab5_pa_c2.tool": [ "causcibench_real.q015", "causcibench_real.q016", "causcibench_real.q017", "causcibench_real.q018" ], "paper_repro.cheng_hoekstra_2013.tab5_pb_c1.tool": [ "causcibench_real.q015", "causcibench_real.q016", "causcibench_real.q017", "causcibench_real.q018" ], "paper_repro.cheng_hoekstra_2013.tab5_pb_c2.tool": [ "causcibench_real.q015", "causcibench_real.q016", "causcibench_real.q017", "causcibench_real.q018" ], "paper_repro.kessler_roth_2014.t18_2_twfe.tool": [ "causcibench_real.q020" ], "paper_repro.lmb_2004.tab1_c1.tool": [ "causcibench_real.q022", "causcibench_real.q023" ], "paper_repro.lmb_2004.tab1_c3.tool": [ "causcibench_real.q022", "causcibench_real.q023" ], "paper_repro.mmv_2011.tab20_1_c1.tool": [ "causcibench_real.q019" ], "paper_repro.mmv_2011.tab20_1_c2.tool": [ "causcibench_real.q019" ], "paper_repro.thornton_2008.tab4_c1.tool": [ "causcibench_real.q021" ] }, "excluded": [], "id": "paper_repro", "licence": "Datasets are fetched, never hosted: each keeps its publisher's terms (the authors' pages, the Econometric Society's supplements). Bundles hold digests, tool results and short attributed excerpts only.", "n_suite_tasks": 153, "n_suite_tasks_by_target": { "agent": 97, "tool": 56 }, "n_tasks": 153, "native_fill": {}, "native_median": [], "scorer": null, "source_split": 56, "specs_sha256": "88ea6c34477704438ae8791fbd50530586373738b4126bd2d48223f420d0bc89", "task_ids_sha256": "200b890b0634652115dc4704522a666acb18792ac51e545b8f802c0c4bb2258e", "tool_coverage": { "basis": "by_construction", "covered": 56, "rate": 1.0, "total": 56 }, "version": "0.1.1" } ], "usage": { "cached": 0, "incident_tokens": 0, "input": 0, "model_calls": 104, "output": 0, "thought": 0, "tool_calls": 63 }, "wall_time_s": 13.195 }