{"args":{},"checks":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"tool_called","min_lre":null,"note":"the canonical tool for this design","open_interval":false,"rel_tol":null,"required":false,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"any_of","min_lre":null,"note":"infer.fisher_exact (or infer.randomization_inference)","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"tool_called","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"tool_called","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":null,"weight":1.0,"where":null}],"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"any_of","min_lre":null,"note":"headline figure against the truth","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":[{"abs_tol":0.002,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"figure_close","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":0.12987012987012986,"weight":1.0,"where":null},{"abs_tol":0.002,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"diagnostic","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":0.12987012987012986,"weight":1.0,"where":null},{"abs_tol":0.03206710883841617,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"figure_close","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":0.12987012987012986,"weight":1.0,"where":null},{"abs_tol":0.03206710883841617,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":"p_value","kind":"diagnostic","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":0.12987012987012986,"weight":1.0,"where":null}],"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_backed","min_lre":null,"note":"every number traced to a tool call or the question","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":null,"value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"any_of","min_lre":null,"note":"the answer cites the expected tool's figures","open_interval":false,"rel_tol":null,"required":false,"source_id":null,"table":null,"tool":null,"value":[{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_cites","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.fisher_exact","value":null,"weight":1.0,"where":null},{"abs_tol":null,"bind":"any","column":null,"failure_class":null,"field":"value","interval":false,"key":null,"kind":"answer_cites","min_lre":null,"note":"","open_interval":false,"rel_tol":null,"required":true,"source_id":null,"table":null,"tool":"infer.randomization_inference","value":null,"weight":1.0,"where":null}],"weight":1.0,"where":null}],"datasets":{"dados":{"columns":null,"description":"Uma linha por unidade do piloto. resultado: indicador de desempenho medido ao fim do piloto; tratamento: 1 se a unidade foi sorteada para o tratamento.","file_sha256":null,"format":null,"frame_sha256":null,"generator":"magi_bench.generators:small_rct","kind":"generator","licence":null,"member":null,"object":null,"params":{"effect":1.04,"n":12,"seed":220415496},"path":null,"read_kw":{},"records":null,"role":"agent","sha256":null,"transforms":[],"url":null}},"difficulty":"M","id":"tool_selection.small_rct.test.s1001","level":"T2","meta":{"acceptable_tools":["infer.randomization_inference"],"expected_tool":"infer.fisher_exact","family":"small_rct","features":["random_assignment","small_sample","exact_inference"],"headline_keys":{"infer.fisher_exact":["p_value"],"infer.randomization_inference":["p_value"]},"lesson":"With a handful of randomized units asymptotic errors are not credible: use the exact randomization test, infer.fisher_exact.","params":{"effect":1.04,"n":12,"seed":220415496},"reference":{"cite":1,"note":"","steps":[{"args":{"dataset":"dados","treatment":"tratamento","y":"resultado"},"tool":"infer.fisher_exact","why":"Poucas unidades sorteadas: p-valor exato de Fisher."}]},"seed":1001,"tolerance":0.002,"trap_tools":[],"truth_value":0.12987012987012986,"variant":1},"question":"O piloto foi aplicado em 6 de 12 escolas, sorteadas. Queremos um p-valor confi\u00e1vel para o efeito do programa, sem depender de aproxima\u00e7\u00f5es de amostra grande.","requires":[],"runner":null,"sources":[],"split":"test","suite":"tool_selection","tags":["tool_selection","small_rct","test"],"target":"agent","tool":null,"truth":"DGP generators.small_rct(n=12, effect=1.04); truth = Fisher's exact p-value of the sharp null (|difference in means|, all 924 assignments enumerated) = 0.129870; tolerance 0.002 for the exact test, 0.0321 (4 Monte Carlo SEs at the default 2000 draws) for randomization inference"}