{"args":{},"checks":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_backed","min_lre":null,"note":"every number of the answer is stated by a tool call","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"choice_equals","min_lre":null,"note":"the answer's first yes/no/letter token; QRData's prefix rule is recorded beside it","open_interval":false,"rel_tol":null,"required":true,"source_id":"qrdata.gold.364","table":null,"tool":null,"value":"b","weight":1.0,"where":null}],"datasets":{"app_engagement_push":{"columns":null,"description":"A study is conducted to measure the effect of a marketing push on user engagement, specifically in-app purchases. Some customers who were assigned to receive the push are not receiving it, because they probably have an older phone that doesn\u2019t support the kind of push the marketing team designed.\nThe dataset app_engagement_push.csv contains records for 10,000 random customers. Each record includes whether an in-app purchase was made (in_app_purchase), if a marketing push was assigned to the user (push_assigned), and if the marketing push was successfully delivered (push_delivered).","file_sha256":"3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970","format":"csv","frame_sha256":null,"generator":null,"kind":"file","licence":"CC BY-NC 4.0","member":"data/app_engagement_push.csv","object":null,"params":{},"path":null,"read_kw":{},"records":null,"role":"agent","sha256":"3c3a97cca6fdd96f6856f7856b5f8bc786a31d1a9be78c9f610ef3070783e970","transforms":[],"url":"https://raw.githubusercontent.com/xxxiaol/QRData/de450af45ff7101b328bb064c6b475f73414a7ed/benchmark/data.zip"}},"difficulty":"M","id":"qrdata_socsci.364","level":"T2","meta":{"agent_reach":{"blocked_by":["no_reference_path","no_tool:multiple_choice_judgement"],"dtypes":{},"named":{},"reachable":false,"rule":"magi_bench.reach/1"},"answer_format":{"figure_transform":"identity","round_to":null,"unit_scale":1.0},"answer_format_from":{"figure_transform":null,"round_to":null,"unit_scale":null},"cluster":"app_engagement_push.csv","family":"multiple_choice","headline_keys":{},"native":{"answer":"B","keywords":["Causality","Observational data","Regression analysis","Average treatment effect","Instrumental variables"],"multiple_choices":["a","b","c"],"question_type":"multiple_choice","reference":"Causal Inference for the Brave and True 9"},"native_id":364,"reference":{"cite":1,"note":"Sem caminho de refer\u00eancia com as ferramentas registradas: no_tool:multiple_choice_judgement. A resposta n\u00e3o cita estimativa de efeito.","steps":[{"args":{"dataset":"app_engagement_push"},"tool":"data.describe","why":"Sem caminho de refer\u00eancia com as ferramentas registradas: no_tool:multiple_choice_judgement."}]},"reference_path":{"calls":[],"needs":["no_tool:multiple_choice_judgement"],"provenance":"none","quotes":{},"reference_datasets":[],"source_id":null},"reference_why":null},"question":"Consider we estimate the Average Treatment Effect of receiving the marketing push on in-app purchases, by conducting linear regression with the formula \"in_app_purchase ~ 1 + push_assigned + push_delivered\". Will the true impact be A) higher than, B) lower than, C) the same with our estimated impact? Please answer with A, B, or C.","requires":[],"runner":null,"sources":["qrdata.gold.364","qrdata.eval_py"],"split":"public","suite":"qrdata_socsci","tags":["econ-bench","external","qrdata","brave_and_true","multiple_choice","no_reference_path"],"target":"agent","tool":null,"truth":"QRData gold: benchmark/QRData.json item 364 at de450af45ff7101b328bb064c6b475f73414a7ed"}