{ "by_family": [ { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "agriculture", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "craigslist_data_wrangler", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "family": "crime_data_wrangler", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 0, "visible_exec_ok_rate": 0.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_divide", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_fold", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_fold_2", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_merge_split", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.2, "visible_pass": 5, "visible_pass_rate": 1.0 }, { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_split_fold", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_unfold", "hybrid_accuracy": 0.4, "hybrid_exact": 2, "n": 5, "oracle_union": 2, "oracle_union_rate": 0.4, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_unfold2", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "proactive_wrangling_complex", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "proactive_wrangling_fold", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 3, "visible_exec_ok_rate": 0.6, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "reshape_table_structure_data_wrangler", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 5, "agreement_correct": 5, "agreement_precision": 1.0, "agreement_rate": 1.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_10", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 1.0, "program_hidden_exact": 5, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 5, "visible_pass_rate": 1.0 }, { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_11", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.2, "program_hidden_exact": 1, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 1, "visible_pass_rate": 0.2 }, { "agreement": 5, "agreement_correct": 5, "agreement_precision": 1.0, "agreement_rate": 1.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_12", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 1.0, "program_hidden_exact": 5, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 5, "visible_pass_rate": 1.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_13", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.6, "program_hidden_exact": 3, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 3, "visible_pass_rate": 0.6 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_15", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_17", "hybrid_accuracy": 0.6, "hybrid_exact": 3, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_18", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_19", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_2", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 3, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.6, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_22", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 3, "visible_false_pass_rate_among_visible": 1.0, "visible_pass": 3, "visible_pass_rate": 0.6 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_24", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_25", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_26", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_27", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 3, "visible_exec_ok_rate": 0.6, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0, "visible_pass": 1, "visible_pass_rate": 0.2 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_28", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_29", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_3", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_30", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 4, "direct_parse_rate": 0.8, "family": "synthetic_33", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0, "visible_pass": 1, "visible_pass_rate": 0.2 }, { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_34", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0, "visible_pass": 1, "visible_pass_rate": 0.2 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_36", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_37", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_4", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_40", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0, "visible_pass": 1, "visible_pass_rate": 0.2 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 2, "direct_parse_rate": 0.4, "family": "synthetic_41", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_43", "hybrid_accuracy": 1.0, "hybrid_exact": 5, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_44", "hybrid_accuracy": 0.6, "hybrid_exact": 3, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 3, "agreement_correct": 3, "agreement_precision": 1.0, "agreement_rate": 0.6, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_45", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 5, "oracle_union_rate": 1.0, "program_accuracy": 0.8, "program_hidden_exact": 4, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 4, "visible_pass_rate": 0.8 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_46", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_47", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 2, "visible_exec_ok_rate": 0.4, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.2, "direct_exact": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_48", "hybrid_accuracy": 0.2, "hybrid_exact": 1, "n": 5, "oracle_union": 1, "oracle_union_rate": 0.2, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_49", "hybrid_accuracy": 0.6, "hybrid_exact": 3, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 4, "visible_exec_ok_rate": 0.8, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_5", "hybrid_accuracy": 0.4, "hybrid_exact": 2, "n": 5, "oracle_union": 2, "oracle_union_rate": 0.4, "program_accuracy": 0.2, "program_hidden_exact": 1, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.5, "visible_pass": 2, "visible_pass_rate": 0.4 }, { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_51", "hybrid_accuracy": 0.4, "hybrid_exact": 2, "n": 5, "oracle_union": 3, "oracle_union_rate": 0.6, "program_accuracy": 0.4, "program_hidden_exact": 2, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 2, "visible_pass_rate": 0.4 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_6", "hybrid_accuracy": 0.0, "hybrid_exact": 0, "n": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_7", "hybrid_accuracy": 0.8, "hybrid_exact": 4, "n": 5, "oracle_union": 4, "oracle_union_rate": 0.8, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 1, "visible_exec_ok_rate": 0.2, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 5, "code_found_rate": 1.0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "family": "synthetic_8", "hybrid_accuracy": 0.2, "hybrid_exact": 1, "n": 5, "oracle_union": 1, "oracle_union_rate": 0.2, "program_accuracy": 0.2, "program_hidden_exact": 1, "visible_exec_ok": 5, "visible_exec_ok_rate": 1.0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 1, "visible_pass_rate": 0.2 } ], "by_num_samples": { "1": { "agreement": 7, "agreement_correct": 4, "agreement_precision": 0.5714285714285714, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.38, "direct_exact": 19, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.38, "hybrid_exact": 19, "n": 50, "oracle_union": 19, "oracle_union_rate": 0.38, "program_accuracy": 0.08, "program_hidden_exact": 4, "visible_exec_ok": 43, "visible_exec_ok_rate": 0.86, "visible_false_pass": 6, "visible_false_pass_rate_among_visible": 0.6, "visible_pass": 10, "visible_pass_rate": 0.2 }, "2": { "agreement": 7, "agreement_correct": 6, "agreement_precision": 0.8571428571428571, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.58, "direct_exact": 29, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "n": 50, "oracle_union": 33, "oracle_union_rate": 0.66, "program_accuracy": 0.2, "program_hidden_exact": 10, "visible_exec_ok": 39, "visible_exec_ok_rate": 0.78, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.09090909090909091, "visible_pass": 11, "visible_pass_rate": 0.22 }, "3": { "agreement": 7, "agreement_correct": 6, "agreement_precision": 0.8571428571428571, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.6, "direct_exact": 30, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "n": 50, "oracle_union": 31, "oracle_union_rate": 0.62, "program_accuracy": 0.14, "program_hidden_exact": 7, "visible_exec_ok": 41, "visible_exec_ok_rate": 0.82, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.125, "visible_pass": 8, "visible_pass_rate": 0.16 }, "4": { "agreement": 7, "agreement_correct": 6, "agreement_precision": 0.8571428571428571, "agreement_rate": 0.14, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.62, "direct_exact": 31, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "hybrid_accuracy": 0.62, "hybrid_exact": 31, "n": 50, "oracle_union": 33, "oracle_union_rate": 0.66, "program_accuracy": 0.16, "program_hidden_exact": 8, "visible_exec_ok": 37, "visible_exec_ok_rate": 0.74, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.1111111111111111, "visible_pass": 9, "visible_pass_rate": 0.18 }, "5": { "agreement": 6, "agreement_correct": 6, "agreement_precision": 1.0, "agreement_rate": 0.12, "code_found": 50, "code_found_rate": 1.0, "direct_accuracy": 0.58, "direct_exact": 29, "direct_parse_ok": 48, "direct_parse_rate": 0.96, "hybrid_accuracy": 0.58, "hybrid_exact": 29, "n": 50, "oracle_union": 32, "oracle_union_rate": 0.64, "program_accuracy": 0.18, "program_hidden_exact": 9, "visible_exec_ok": 42, "visible_exec_ok_rate": 0.84, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 9, "visible_pass_rate": 0.18 } }, "overall": { "agreement": 34, "agreement_correct": 28, "agreement_precision": 0.8235294117647058, "agreement_rate": 0.136, "code_found": 250, "code_found_rate": 1.0, "direct_accuracy": 0.552, "direct_exact": 138, "direct_parse_ok": 236, "direct_parse_rate": 0.944, "hybrid_accuracy": 0.556, "hybrid_exact": 139, "n": 250, "oracle_union": 148, "oracle_union_rate": 0.592, "program_accuracy": 0.152, "program_hidden_exact": 38, "visible_exec_ok": 202, "visible_exec_ok_rate": 0.808, "visible_false_pass": 9, "visible_false_pass_rate_among_visible": 0.19148936170212766, "visible_pass": 47, "visible_pass_rate": 0.188 }, "overlap": { "agreement_hidden_wrong": 6, "both_direct_and_program": 28, "direct_only": 110, "neither": 102, "program_only": 10, "visible_pass_hidden_wrong": 9 }, "prompt_iteration": [ { "agreement_precision": 1.0, "direct_accuracy": 1.0, "n": 8, "name": "induce prefix8", "oracle_union_rate": 1.0, "program_accuracy": 0.625, "visible_pass_rate": 0.625 }, { "agreement_precision": 1.0, "direct_accuracy": 1.0, "n": 8, "name": "context prefix8", "oracle_union_rate": 1.0, "program_accuracy": 0.875, "visible_pass_rate": 0.875 }, { "agreement_precision": 0.3333333333333333, "direct_accuracy": 0.44, "n": 25, "name": "context spread25", "oracle_union_rate": 0.48, "program_accuracy": 0.12, "visible_pass_rate": 0.28 }, { "agreement_precision": 0.75, "direct_accuracy": 0.44, "n": 25, "name": "context_v2 spread25", "oracle_union_rate": 0.44, "program_accuracy": 0.12, "visible_pass_rate": 0.24 } ], "prompt_name": "context", "records": "reports/eval_records_context.jsonl", "target_char_buckets": { "201-500": { "agreement": 18, "agreement_correct": 17, "agreement_precision": 0.9444444444444444, "agreement_rate": 0.21428571428571427, "code_found": 84, "code_found_rate": 1.0, "direct_accuracy": 0.6309523809523809, "direct_exact": 53, "direct_parse_ok": 84, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.6309523809523809, "hybrid_exact": 53, "n": 84, "oracle_union": 56, "oracle_union_rate": 0.6666666666666666, "program_accuracy": 0.23809523809523808, "program_hidden_exact": 20, "visible_exec_ok": 76, "visible_exec_ok_rate": 0.9047619047619048, "visible_false_pass": 2, "visible_false_pass_rate_among_visible": 0.09090909090909091, "visible_pass": 22, "visible_pass_rate": 0.2619047619047619 }, "501-1000": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "code_found": 18, "code_found_rate": 1.0, "direct_accuracy": 0.7777777777777778, "direct_exact": 14, "direct_parse_ok": 18, "direct_parse_rate": 1.0, "hybrid_accuracy": 0.7777777777777778, "hybrid_exact": 14, "n": 18, "oracle_union": 14, "oracle_union_rate": 0.7777777777777778, "program_accuracy": 0.0, "program_hidden_exact": 0, "visible_exec_ok": 16, "visible_exec_ok_rate": 0.8888888888888888, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0, "visible_pass": 0, "visible_pass_rate": 0.0 }, "<=200": { "agreement": 15, "agreement_correct": 10, "agreement_precision": 0.6666666666666666, "agreement_rate": 0.11450381679389313, "code_found": 131, "code_found_rate": 1.0, "direct_accuracy": 0.5190839694656488, "direct_exact": 68, "direct_parse_ok": 127, "direct_parse_rate": 0.9694656488549618, "hybrid_accuracy": 0.5190839694656488, "hybrid_exact": 68, "n": 131, "oracle_union": 74, "oracle_union_rate": 0.5648854961832062, "program_accuracy": 0.12213740458015267, "program_hidden_exact": 16, "visible_exec_ok": 99, "visible_exec_ok_rate": 0.7557251908396947, "visible_false_pass": 6, "visible_false_pass_rate_among_visible": 0.2727272727272727, "visible_pass": 22, "visible_pass_rate": 0.16793893129770993 }, ">1000": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.058823529411764705, "code_found": 17, "code_found_rate": 1.0, "direct_accuracy": 0.17647058823529413, "direct_exact": 3, "direct_parse_ok": 7, "direct_parse_rate": 0.4117647058823529, "hybrid_accuracy": 0.23529411764705882, "hybrid_exact": 4, "n": 17, "oracle_union": 4, "oracle_union_rate": 0.23529411764705882, "program_accuracy": 0.11764705882352941, "program_hidden_exact": 2, "visible_exec_ok": 11, "visible_exec_ok_rate": 0.6470588235294118, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.3333333333333333, "visible_pass": 3, "visible_pass_rate": 0.17647058823529413 } }, "visible_exec_errors": { "IndexError": 12, "NameError": 3, "None": 202, "SyntaxError": 10, "ValueError": 23 } }