{"args":{},"checks":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"tool_called","min_lre":null,"note":"the canonical tool for this design","open_interval":false,"rel_tol":null,"required":false,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"any_of","min_lre":null,"note":"infer.fisher_exact (or infer.randomization_inference)","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"tool_called","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"tool_called","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":null,"weight":1.0,"where":null}],"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"any_of","min_lre":null,"note":"headline figure against the truth","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":[{"abs_tol":0.002,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"figure_close","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":0.19047619047619047,"weight":1.0,"where":null},{"abs_tol":0.002,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"diagnostic","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":0.19047619047619047,"weight":1.0,"where":null},{"abs_tol":0.03712207412302052,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"figure_close","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":0.19047619047619047,"weight":1.0,"where":null},{"abs_tol":0.03712207412302052,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"diagnostic","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":0.19047619047619047,"weight":1.0,"where":null}],"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_backed","min_lre":null,"note":"every number traced to a tool call or the question","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"any_of","min_lre":null,"note":"the answer cites the expected tool's figures","open_interval":false,"rel_tol":null,"required":false,"source_id":null,"table":null,"tool":null,"value":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_cites","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_cites","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":null,"weight":1.0,"where":null}],"weight":1.0,"where":null}],"datasets":{"dados":{"columns":null,"description":"Uma linha por unidade do piloto. resultado: indicador de desempenho medido ao fim do piloto; tratamento: 1 se a unidade foi sorteada para o tratamento.","file_sha256":null,"format":null,"frame_sha256":null,"generator":"magi_bench.generators:small_rct","kind":"generator","licence":null,"member":null,"object":null,"params":{"effect":1.44,"n":10,"seed":168462773},"path":null,"read_kw":{},"records":null,"role":"agent","sha256":null,"transforms":[],"url":null}},"difficulty":"M","id":"tool_selection.small_rct.test.s1000","level":"T2","meta":{"acceptable_tools":["infer.randomization_inference"],"expected_tool":"infer.fisher_exact","family":"small_rct","features":["random_assignment","small_sample","exact_inference"],"headline_keys":{"infer.fisher_exact":["p_value"],"infer.randomization_inference":["p_value"]},"lesson":"With a handful of randomized units asymptotic errors are not credible: use the exact randomization test, infer.fisher_exact.","params":{"effect":1.44,"n":10,"seed":168462773},"reference":{"cite":1,"note":"","steps":[{"args":{"dataset":"dados","treatment":"tratamento","y":"resultado"},"tool":"infer.fisher_exact","why":"Poucas unidades sorteadas: p-valor exato de Fisher."}]},"seed":1000,"tolerance":0.002,"trap_tools":[],"truth_value":0.19047619047619047,"variant":3},"question":"Temos o resultado de 10 unidades, 5 delas sorteadas para o tratamento. Teste se o tratamento teve algum efeito.","requires":[],"runner":null,"sources":[],"split":"test","suite":"tool_selection","tags":["tool_selection","small_rct","test"],"target":"agent","tool":null,"truth":"DGP generators.small_rct(n=10, effect=1.44); truth = Fisher's exact p-value of the sharp null (|difference in means|, all 252 assignments enumerated) = 0.190476; tolerance 0.002 for the exact test, 0.0371 (4 Monte Carlo SEs at the default 2000 draws) for randomization inference"}