{ "by_family": { "agriculture": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "craigslist_data_wrangler": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "crime_data_wrangler": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 0, "visible_exec_ok_rate": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "potters_wheel_divide": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "potters_wheel_fold": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "potters_wheel_fold_2": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "potters_wheel_merge_split": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 5, "visible_pass_rate": 1.0 }, "potters_wheel_split_fold": { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, "potters_wheel_unfold": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "n": 5, "oracle_union": 2, "oracle_union_rate": 0.4, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_pass": 0, "visible_pass_rate": 0.0 }, "potters_wheel_unfold2": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "proactive_wrangling_complex": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "proactive_wrangling_fold": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 3, "visible_exec_ok_rate": 0.6, "visible_pass": 0, "visible_pass_rate": 0.0 }, "reshape_table_structure_data_wrangler": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_10": { "agreement": 5, "agreement_correct": 5, "agreement_precision": 1.0, "agreement_rate": 1.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 1.0, "program_hidden_exact": 5, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 5, "visible_pass_rate": 1.0 }, "synthetic_11": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.2, "program_hidden_exact": 1, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 1, "visible_pass_rate": 0.2 }, "synthetic_12": { "agreement": 5, "agreement_correct": 5, "agreement_precision": 1.0, "agreement_rate": 1.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 1.0, "program_hidden_exact": 5, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 5, "visible_pass_rate": 1.0 }, "synthetic_13": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.6, "program_hidden_exact": 3, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 3, "visible_pass_rate": 0.6 }, "synthetic_15": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_17": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_18": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_19": { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, "synthetic_2": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_22": { "agreement": 3, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.6, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 3, "visible_pass_rate": 0.6 }, "synthetic_24": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_25": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_26": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_27": { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 3, "visible_exec_ok_rate": 0.6, "visible_pass": 1, "visible_pass_rate": 0.2 }, "synthetic_28": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_29": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_3": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_30": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_33": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 4, "direct_parse_rate": 0.8, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_pass": 1, "visible_pass_rate": 0.2 }, "synthetic_34": { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 1, "visible_pass_rate": 0.2 }, "synthetic_36": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_37": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_4": { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, "synthetic_40": { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_pass": 1, "visible_pass_rate": 0.2 }, "synthetic_41": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 2, "direct_parse_rate": 0.4, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_43": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_44": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_45": { "agreement": 3, "agreement_correct": 3, "agreement_precision": 1.0, "agreement_rate": 0.6, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, "synthetic_46": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_47": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_48": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.2, "direct_exact": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.2, "hybrid_exact": 1, "n": 5, "oracle_union": 1, "oracle_union_rate": 0.2, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_49": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_5": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "n": 5, "oracle_union": 2, "oracle_union_rate": 0.4, "program_accuracy": 0.2, "program_hidden_exact": 1, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 2, "visible_pass_rate": 0.4 }, "synthetic_51": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.4, "program_hidden_exact": 2, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 2, "visible_pass_rate": 0.4 }, "synthetic_6": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_7": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_pass": 0, "visible_pass_rate": 0.0 }, "synthetic_8": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "hybrid_accuracy": 0.2, "hybrid_exact": 1, "n": 5, "oracle_union": 1, "oracle_union_rate": 0.2, "program_accuracy": 0.2, "program_hidden_exact": 1, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_pass": 1, "visible_pass_rate": 0.2 } }, "by_num_samples": { "1": { "agreement": 7, "agreement_correct": 4, "agreement_precision": 0.5714285714285714, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.38, "direct_exact": 19, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.38, "hybrid_exact": 19, "n": 50, "oracle_union": 19, "oracle_union_rate": 0.38, "program_accuracy": 0.08, "program_hidden_exact": 4, "visible_exec_ok": 43, "visible_exec_ok_rate": 0.86, "visible_pass": 10, "visible_pass_rate": 0.2 }, "2": { "agreement": 7, "agreement_correct": 6, "agreement_precision": 0.8571428571428571, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.58, "direct_exact": 29, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "n": 50, "oracle_union": 33, "oracle_union_rate": 0.66, "program_accuracy": 0.2, "program_hidden_exact": 10, "visible_exec_ok": 39, "visible_exec_ok_rate": 0.78, "visible_pass": 11, "visible_pass_rate": 0.22 }, "3": { "agreement": 7, "agreement_correct": 6, "agreement_precision": 0.8571428571428571, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 30, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "n": 50, "oracle_union": 31, "oracle_union_rate": 0.62, "program_accuracy": 0.14, "program_hidden_exact": 7, "visible_exec_ok": 41, "visible_exec_ok_rate": 0.82, "visible_pass": 8, "visible_pass_rate": 0.16 }, "4": { "agreement": 7, "agreement_correct": 6, "agreement_precision": 0.8571428571428571, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.62, "direct_exact": 31, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.62, "hybrid_exact": 31, "n": 50, "oracle_union": 33, "oracle_union_rate": 0.66, "program_accuracy": 0.16, "program_hidden_exact": 8, "visible_exec_ok": 37, "visible_exec_ok_rate": 0.74, "visible_pass": 9, "visible_pass_rate": 0.18 }, "5": { "agreement": 6, "agreement_correct": 6, "agreement_precision": 1.0, "agreement_rate": 0.12, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.58, "direct_exact": 29, "direct_parse_ok": 48, "direct_parse_rate": 0.96, "hybrid_accuracy": 0.58, "hybrid_exact": 29, "n": 50, "oracle_union": 32, "oracle_union_rate": 0.64, "program_accuracy": 0.18, "program_hidden_exact": 9, "visible_exec_ok": 42, "visible_exec_ok_rate": 0.84, "visible_pass": 9, "visible_pass_rate": 0.18 } }, "overall": { "agreement": 34, "agreement_correct": 28, "agreement_precision": 0.8235294117647058, "agreement_rate": 0.136, "code_found": 250, "code_found_rate": 1.0, "direct_accuracy": 0.552, "direct_exact": 138, "direct_parse_ok": 236, "direct_parse_rate": 0.944, "hybrid_accuracy": 0.556, "hybrid_exact": 139, "n": 250, "oracle_union": 148, "oracle_union_rate": 0.592, "program_accuracy": 0.152, "program_hidden_exact": 38, "visible_exec_ok": 202, "visible_exec_ok_rate": 0.808, "visible_pass": 47, "visible_pass_rate": 0.188 } }