{ "by_family": [ { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "agriculture", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "craigslist_data_wrangler", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "family": "crime_data_wrangler", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_divide", "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.8, "n": 5, "neither": 4, "oracle_union": 1, "oracle_union_rate": 0.2, "program_only": 1, "repair_added_program_correct": 1, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_fold", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_fold_2", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_merge_split", "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "mean_round_count": 1.6, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 3, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.25 }, { "agreement": 3, "agreement_correct": 3, "agreement_precision": 1.0, "both_direct_and_program": 3, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_split_fold", "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "mean_round_count": 2.2, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_unfold", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 3, "oracle_union": 2, "oracle_union_rate": 0.4, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "potters_wheel_unfold2", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.6, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "proactive_wrangling_complex", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "both_direct_and_program": 2, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "proactive_wrangling_fold", "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "mean_round_count": 3.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 1, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "reshape_table_structure_data_wrangler", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "both_direct_and_program": 4, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_10", "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "both_direct_and_program": 4, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_11", "final_hidden_exact": 5, "final_program_accuracy": 1.0, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "mean_round_count": 1.8, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 1, "repair_added_program_correct": 4, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 5, "agreement_correct": 5, "agreement_precision": 1.0, "both_direct_and_program": 5, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_12", "final_hidden_exact": 5, "final_program_accuracy": 1.0, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 1.0, "initial_hidden_exact": 5, "mean_round_count": 1.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_13", "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "mean_round_count": 1.8, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 3, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.25 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_15", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_17", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_18", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "both_direct_and_program": 4, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_19", "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_2", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 3, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_22", "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 1.8, "n": 5, "neither": 4, "oracle_union": 1, "oracle_union_rate": 0.2, "program_only": 1, "repair_added_program_correct": 1, "repair_lost_program_correct": 0, "visible_false_pass": 3, "visible_false_pass_rate_among_visible": 0.75 }, { "agreement": 2, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_24", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.4, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 2, "visible_false_pass_rate_among_visible": 1.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_25", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 2, "agreement_correct": 1, "agreement_precision": 0.5, "both_direct_and_program": 1, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_26", "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 1, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.5 }, { "agreement": 3, "agreement_correct": 2, "agreement_precision": 0.6666666666666666, "both_direct_and_program": 2, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_27", "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 2, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.3333333333333333 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_28", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_29", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.6, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_3", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_30", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "both_direct_and_program": 2, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 2, "direct_parse_ok": 4, "direct_parse_rate": 0.8, "family": "synthetic_33", "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "mean_round_count": 2.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 1, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.3333333333333333 }, { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_34", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.4, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_36", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_37", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "both_direct_and_program": 4, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_4", "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 5, "agreement_correct": 4, "agreement_precision": 0.8, "both_direct_and_program": 4, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_40", "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 2.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 4, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.2 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "family": "synthetic_41", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_43", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "both_direct_and_program": 1, "direct_accuracy": 0.6, "direct_exact": 3, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_44", "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "mean_round_count": 3.4, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 3, "agreement_correct": 3, "agreement_precision": 1.0, "both_direct_and_program": 3, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_45", "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 1, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_46", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "both_direct_and_program": 2, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_47", "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 2, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.2, "direct_exact": 1, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_48", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.2, "hybrid_exact": 1, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 4, "oracle_union": 1, "oracle_union_rate": 0.2, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_49", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "both_direct_and_program": 2, "direct_accuracy": 0.4, "direct_exact": 2, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_5", "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "initial_accuracy": 0.4, "initial_hidden_exact": 2, "mean_round_count": 1.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 2, "repair_added_program_correct": 2, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.2 }, { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "both_direct_and_program": 1, "direct_accuracy": 0.4, "direct_exact": 2, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_51", "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "mean_round_count": 2.2, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 2, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_6", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "family": "synthetic_7", "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "repair_added_program_correct": 0, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "family": "synthetic_8", "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "mean_round_count": 2.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 4, "repair_added_program_correct": 3, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 } ], "by_num_samples": { "1": { "agreement": 8, "agreement_correct": 4, "agreement_precision": 0.5, "both_direct_and_program": 4, "direct_accuracy": 0.38, "direct_exact": 19, "direct_only": 15, "direct_parse_ok": 46, "direct_parse_rate": 0.92, "final_hidden_exact": 6, "final_program_accuracy": 0.12, "final_visible_pass": 15, "final_visible_pass_rate": 0.3, "hybrid_accuracy": 0.4, "hybrid_exact": 20, "initial_accuracy": 0.08, "initial_hidden_exact": 4, "mean_round_count": 3.18, "n": 50, "neither": 29, "oracle_union": 21, "oracle_union_rate": 0.42, "program_only": 2, "repair_added_program_correct": 2, "repair_lost_program_correct": 0, "visible_false_pass": 9, "visible_false_pass_rate_among_visible": 0.6 }, "2": { "agreement": 9, "agreement_correct": 8, "agreement_precision": 0.8888888888888888, "both_direct_and_program": 8, "direct_accuracy": 0.58, "direct_exact": 29, "direct_only": 21, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 10, "final_program_accuracy": 0.2, "final_visible_pass": 13, "final_visible_pass_rate": 0.26, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "initial_accuracy": 0.14, "initial_hidden_exact": 7, "mean_round_count": 3.38, "n": 50, "neither": 19, "oracle_union": 31, "oracle_union_rate": 0.62, "program_only": 2, "repair_added_program_correct": 3, "repair_lost_program_correct": 0, "visible_false_pass": 3, "visible_false_pass_rate_among_visible": 0.23076923076923078 }, "3": { "agreement": 16, "agreement_correct": 12, "agreement_precision": 0.75, "both_direct_and_program": 12, "direct_accuracy": 0.6, "direct_exact": 30, "direct_only": 18, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 17, "final_program_accuracy": 0.34, "final_visible_pass": 19, "final_visible_pass_rate": 0.38, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "initial_accuracy": 0.22, "initial_hidden_exact": 11, "mean_round_count": 3.08, "n": 50, "neither": 15, "oracle_union": 35, "oracle_union_rate": 0.7, "program_only": 5, "repair_added_program_correct": 6, "repair_lost_program_correct": 0, "visible_false_pass": 2, "visible_false_pass_rate_among_visible": 0.10526315789473684 }, "4": { "agreement": 12, "agreement_correct": 11, "agreement_precision": 0.9166666666666666, "both_direct_and_program": 11, "direct_accuracy": 0.62, "direct_exact": 31, "direct_only": 20, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 15, "final_program_accuracy": 0.3, "final_visible_pass": 17, "final_visible_pass_rate": 0.34, "hybrid_accuracy": 0.64, "hybrid_exact": 32, "initial_accuracy": 0.18, "initial_hidden_exact": 9, "mean_round_count": 3.16, "n": 50, "neither": 15, "oracle_union": 35, "oracle_union_rate": 0.7, "program_only": 4, "repair_added_program_correct": 6, "repair_lost_program_correct": 0, "visible_false_pass": 2, "visible_false_pass_rate_among_visible": 0.11764705882352941 }, "5": { "agreement": 10, "agreement_correct": 9, "agreement_precision": 0.9, "both_direct_and_program": 9, "direct_accuracy": 0.58, "direct_exact": 29, "direct_only": 20, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 14, "final_program_accuracy": 0.28, "final_visible_pass": 14, "final_visible_pass_rate": 0.28, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "initial_accuracy": 0.18, "initial_hidden_exact": 9, "mean_round_count": 3.3, "n": 50, "neither": 16, "oracle_union": 34, "oracle_union_rate": 0.68, "program_only": 5, "repair_added_program_correct": 5, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 } }, "final_visible_exec_errors": { "CodeParse": 1, "IndentationError": 1, "IndexError": 11, "NameError": 2, "None": 220, "SyntaxError": 3, "TimeoutExpired": 1, "TypeError": 2, "ValueError": 9 }, "overall": { "agreement": 55, "agreement_correct": 44, "agreement_precision": 0.8, "both_direct_and_program": 44, "direct_accuracy": 0.552, "direct_exact": 138, "direct_only": 94, "direct_parse_ok": 234, "direct_parse_rate": 0.936, "final_hidden_exact": 62, "final_program_accuracy": 0.248, "final_visible_pass": 78, "final_visible_pass_rate": 0.312, "hybrid_accuracy": 0.568, "hybrid_exact": 142, "initial_accuracy": 0.16, "initial_hidden_exact": 40, "mean_round_count": 3.22, "n": 250, "neither": 94, "oracle_union": 156, "oracle_union_rate": 0.624, "program_only": 18, "repair_added_program_correct": 22, "repair_lost_program_correct": 0, "visible_false_pass": 16, "visible_false_pass_rate_among_visible": 0.20512820512820512 }, "prompt_iteration": [ { "direct_accuracy": 1.0, "final_program_accuracy": 0.8333333333333334, "initial_accuracy": 0.8333333333333334, "n": 6, "name": "prefix6 r2", "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass_rate_among_visible": 0.0 }, { "direct_accuracy": 0.44, "final_program_accuracy": 0.08, "initial_accuracy": 0.08, "n": 25, "name": "standard spread25 r2", "oracle_union_rate": 0.44, "program_only": 0, "visible_false_pass_rate_among_visible": 0.7142857142857143 }, { "direct_accuracy": 0.44, "final_program_accuracy": 0.16, "initial_accuracy": 0.08, "n": 25, "name": "strict spread25 r2", "oracle_union_rate": 0.52, "program_only": 2, "visible_false_pass_rate_among_visible": 0.42857142857142855 } ], "records": "/workspace/experiments/qwen35_4b_foofah_program_repair_agent/reports/eval_records.jsonl", "round_stats": { "0": { "attempted": 250, "hidden_exact": 41, "visible_pass": 50, "visible_verified_hidden_exact": 40 }, "1": { "attempted": 200, "hidden_exact": 17, "visible_pass": 20, "visible_verified_hidden_exact": 16 }, "2": { "attempted": 180, "hidden_exact": 6, "visible_pass": 5, "visible_verified_hidden_exact": 5 }, "3": { "attempted": 175, "hidden_exact": 1, "visible_pass": 3, "visible_verified_hidden_exact": 1 } }, "target_char_buckets": { "201-500": { "agreement": 22, "agreement_correct": 20, "agreement_precision": 0.9090909090909091, "both_direct_and_program": 20, "direct_accuracy": 0.6309523809523809, "direct_exact": 53, "direct_only": 33, "direct_parse_ok": 84, "direct_parse_rate": 1.0, "final_hidden_exact": 25, "final_program_accuracy": 0.2976190476190476, "final_visible_pass": 29, "final_visible_pass_rate": 0.34523809523809523, "hybrid_accuracy": 0.6309523809523809, "hybrid_exact": 53, "initial_accuracy": 0.23809523809523808, "initial_hidden_exact": 20, "mean_round_count": 3.0595238095238093, "n": 84, "neither": 26, "oracle_union": 58, "oracle_union_rate": 0.6904761904761905, "program_only": 5, "repair_added_program_correct": 5, "repair_lost_program_correct": 0, "visible_false_pass": 4, "visible_false_pass_rate_among_visible": 0.13793103448275862 }, "501-1000": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "both_direct_and_program": 1, "direct_accuracy": 0.7777777777777778, "direct_exact": 14, "direct_only": 13, "direct_parse_ok": 18, "direct_parse_rate": 1.0, "final_hidden_exact": 2, "final_program_accuracy": 0.1111111111111111, "final_visible_pass": 2, "final_visible_pass_rate": 0.1111111111111111, "hybrid_accuracy": 0.7777777777777778, "hybrid_exact": 14, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "mean_round_count": 3.7777777777777777, "n": 18, "neither": 3, "oracle_union": 15, "oracle_union_rate": 0.8333333333333334, "program_only": 1, "repair_added_program_correct": 2, "repair_lost_program_correct": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "<=200": { "agreement": 31, "agreement_correct": 22, "agreement_precision": 0.7096774193548387, "both_direct_and_program": 22, "direct_accuracy": 0.5190839694656488, "direct_exact": 68, "direct_only": 46, "direct_parse_ok": 125, "direct_parse_rate": 0.9541984732824428, "final_hidden_exact": 29, "final_program_accuracy": 0.22137404580152673, "final_visible_pass": 40, "final_visible_pass_rate": 0.3053435114503817, "hybrid_accuracy": 0.5190839694656488, "hybrid_exact": 68, "initial_accuracy": 0.1297709923664122, "initial_hidden_exact": 17, "mean_round_count": 3.2748091603053435, "n": 131, "neither": 56, "oracle_union": 75, "oracle_union_rate": 0.5725190839694656, "program_only": 7, "repair_added_program_correct": 12, "repair_lost_program_correct": 0, "visible_false_pass": 11, "visible_false_pass_rate_among_visible": 0.275 }, ">1000": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "both_direct_and_program": 1, "direct_accuracy": 0.17647058823529413, "direct_exact": 3, "direct_only": 2, "direct_parse_ok": 7, "direct_parse_rate": 0.4117647058823529, "final_hidden_exact": 6, "final_program_accuracy": 0.35294117647058826, "final_visible_pass": 7, "final_visible_pass_rate": 0.4117647058823529, "hybrid_accuracy": 0.4117647058823529, "hybrid_exact": 7, "initial_accuracy": 0.17647058823529413, "initial_hidden_exact": 3, "mean_round_count": 3.0, "n": 17, "neither": 9, "oracle_union": 8, "oracle_union_rate": 0.47058823529411764, "program_only": 5, "repair_added_program_correct": 3, "repair_lost_program_correct": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.14285714285714285 } } }