{"args":{},"checks":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_backed","min_lre":null,"note":"every number in the answer was stated by a tool call","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"answer_call","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_figure_close","min_lre":null,"note":"CauSciBench effect accuracy: |answer figure \u2212 gold| \u2264 5% of |gold|","open_interval":false,"rel_tol":0.05,"required":true,"source_id":"causcibench_real.gold.q014","table":null,"tool":null,"value":0.128,"weight":1.0,"where":null}],"datasets":{"broockman_black_politicians":{"columns":null,"description":"This data is from a randomized field experiment in which fictional emails, appearing to come from Black constituents, were sent to legislators across the United States. The purpose was to test whether being outside a legislator's district reduced the likelihood of receiving a response, and whether Black legislators were more likely to respond regardless of electoral incentives. The main focus is on differences in responsiveness based on the race of the legislator and the sender's eligibility to vote.\n\nVariables include:\nleg_black: 1 if the legislator is Black, 0 otherwise\ntreat_out: 1 if the email was from an out-of-district sender, 0 otherwise\nresponded: 1 if the legislator responded to the email, 0 otherwise\ntotalpop: Population of the legislator's district\nmedianhhincom: Median household income in the district\nblack_medianhh: Median household income among Black residents in the district\nwhite_medianhh: Median household income among White residents in the district\nblackpercent: Percentage of the district's population that is Black\nstatessquireindex: State legislative professionalism index (Squire index)\nnonblacknonwhite: 1 if the legislator is neither Black nor White, 0 otherwise\nurbanpercent: Percentage of the district that is urban\nleg_senator: 1 if the legislator is a senator, 0 otherwise\nleg_democrat: 1 if the legislator is a Democrat, 0 otherwise\nsouth: 1 if the legislator represents a Southern state, 0 otherwise","file_sha256":"414450c964cacbde724fd24906b61bb65232c0ae9c73a1bc021dbce1ea058022","format":"csv","frame_sha256":null,"generator":null,"kind":"file","licence":"MIT License (CauSciBench data/README.md); original source https://cran.r-project.org/web/packages/causaldata/index.html","member":null,"object":null,"params":{},"path":null,"read_kw":{},"records":null,"role":"agent","sha256":"414450c964cacbde724fd24906b61bb65232c0ae9c73a1bc021dbce1ea058022","transforms":[],"url":"https://huggingface.co/datasets/causal-nlp/causcibench/resolve/d8aca99042e0bf1d73120a9dc1b65d4079fa1c3a/csv_files/realdata/broockman_black_politicians.csv"}},"difficulty":"M","id":"causcibench_real.q014","level":"T2","meta":{"agent_reach":{"blocked_by":["needs_spec:interaction"],"dtypes":{"broockman_black_politicians":{"black_medianhh":"float64","blackpercent":"float64","leg_black":"int64","leg_democrat":"int64","leg_senator":"int64","nonblacknonwhite":"int64","responded":"int64","south":"int64","statessquireindex":"float64","totalpop":"float64","treat_out":"int64","urbanpercent":"float64","white_medianhh":"float64"}},"named":{"outcome":["responded"],"treatment":["treat_out"]},"reachable":false,"rule":"magi_bench.reach/1"},"answer_format":{"figure_transform":"identity","round_to":null,"unit_scale":1.0},"cluster":"paper:Black politicians are more intrinsically motivated to advance blacks interests: A field experiment manipulating political incentives","gold":{"effect":0.128,"effect_as_stored":"0.128","is_significant":1,"source_id":"causcibench_real.gold.q014","std_error":0.051,"std_error_as_stored":"0.051"},"gold_defect":"the golds of 13 and 14 are swapped relative to their queries: 13 asks for the Black-legislator difference and stores the main out-of-district effect (-0.276), 14 asks for the main effect and stores the interaction (0.128); an answer correct for its own query fails both","headline_keys":{"infer.ols":["coef.treat_out"]},"item_caveats":[{"kind":"query_gold_swapped","note":"the golds of 13 and 14 are swapped relative to their queries: 13 asks for the Black-legislator difference and stores the main out-of-district effect (-0.276), 14 asks for the main effect and stores the interaction (0.128); an answer correct for its own query fails both"}],"metadata":{"control_variables":"leg_black,leg_democrat,south,leg_senator,blackpercent,black_medianhh,white_medianhh,statessquireindex,urbanpercent,nonblacknonwhite,totalpop","dataset_path":"data/csv_files/realdata/broockman_black_politicians.csv","effect":0.128,"id":14,"instrument_var":null,"interaction_var":"leg_black","is_rct":true,"is_significant":1,"method":"ols","name":"Black politicians are more intrinsically motivated to advance blacks interests: A field experiment manipulating political incentives","outcome_var":"responded","running_var":null,"state_var":null,"std_error":0.051,"temporal_var":null,"treatment_var":"treat_out"},"naive":{"dataset":"broockman_black_politicians","treatment":"treat_out","y":"responded"},"name":"Black politicians are more intrinsically motivated to advance blacks interests: A field experiment manipulating political incentives","native_id":14,"outcome":"responded","pin":{"data_repo":"causal-nlp/causcibench","data_revision":"d8aca99042e0bf1d73120a9dc1b65d4079fa1c3a","repo":"causalNLP/CauSciBench","revision":"d9bb43a2d90f261978cc5ac0b628c0484f6e8956"},"precision_limited":false,"reference":{"cite":1,"steps":[{"args":{"controls":["leg_black","leg_democrat","south","leg_senator","blackpercent","black_medianhh","white_medianhh","statessquireindex","urbanpercent","nonblacknonwhite","totalpop"],"dataset":"broockman_black_politicians","x":["treat_out"],"y":"responded"},"tool":"infer.ols","why":"causcibench_metadata_rule/1: ols"}]},"reference_notes":["the item's text fixes a interaction the metadata rule does not apply"],"reference_path":{"calls":[{"args":{"controls":["leg_black","leg_democrat","south","leg_senator","blackpercent","black_medianhh","white_medianhh","statessquireindex","urbanpercent","nonblacknonwhite","totalpop"],"dataset":"broockman_black_politicians","x":["treat_out"],"y":"responded"},"tool":"infer.ols"}],"needs":["needs_spec:interaction"],"provenance":"metadata_rule","quotes":{"interaction":"interaction_var: leg_black"},"reference_datasets":[],"source_id":null},"reference_rule":"causcibench_metadata_rule/1","source":"causcibench","treatment":"treat_out"},"question":"Do legislators tend to respond to out-of-district emails?","requires":[],"runner":null,"sources":["causcibench_real.gold.q014"],"split":"public","suite":"causcibench_real","tags":["econ-bench","external","causcibench","method:ols"],"target":"agent","tool":null,"truth":"CauSciBench real_input.json @ d9bb43a, id 14: effect 0.128"}