{ "by_family": { "agriculture": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "craigslist_data_wrangler": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "crime_data_wrangler": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "potters_wheel_divide": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 1, "any_round_hidden_exact_rate": 0.2, "any_visible_verified_hidden_exact": 1, "any_visible_verified_hidden_exact_rate": 0.2, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 3.8, "n": 5, "neither": 4, "oracle_union": 1, "oracle_union_rate": 0.2, "program_only": 1, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "potters_wheel_fold": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "potters_wheel_fold_2": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "potters_wheel_merge_split": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 3, "any_round_hidden_exact_rate": 0.6, "any_visible_verified_hidden_exact": 3, "any_visible_verified_hidden_exact_rate": 0.6, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "initial_visible_pass": 4, "initial_visible_pass_rate": 0.8, "mean_round_count": 1.6, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 3, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.25 }, "potters_wheel_split_fold": { "agreement": 3, "agreement_correct": 3, "agreement_precision": 1.0, "agreement_rate": 0.6, "any_round_hidden_exact": 3, "any_round_hidden_exact_rate": 0.6, "any_visible_verified_hidden_exact": 3, "any_visible_verified_hidden_exact_rate": 0.6, "both_direct_and_program": 3, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "initial_visible_pass": 3, "initial_visible_pass_rate": 0.6, "mean_round_count": 2.2, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "potters_wheel_unfold": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.4, "direct_exact": 2, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 3, "oracle_union": 2, "oracle_union_rate": 0.4, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "potters_wheel_unfold2": { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 3.6, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, "proactive_wrangling_complex": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "proactive_wrangling_fold": { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "agreement_rate": 0.4, "any_round_hidden_exact": 2, "any_round_hidden_exact_rate": 0.4, "any_visible_verified_hidden_exact": 2, "any_visible_verified_hidden_exact_rate": 0.4, "both_direct_and_program": 2, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 3.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "reshape_table_structure_data_wrangler": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_10": { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "any_round_hidden_exact": 4, "any_round_hidden_exact_rate": 0.8, "any_visible_verified_hidden_exact": 4, "any_visible_verified_hidden_exact_rate": 0.8, "both_direct_and_program": 4, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "initial_visible_pass": 4, "initial_visible_pass_rate": 0.8, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_11": { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "any_round_hidden_exact": 5, "any_round_hidden_exact_rate": 1.0, "any_visible_verified_hidden_exact": 5, "any_visible_verified_hidden_exact_rate": 1.0, "both_direct_and_program": 4, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 5, "final_program_accuracy": 1.0, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 1.8, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 1, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_12": { "agreement": 5, "agreement_correct": 5, "agreement_precision": 1.0, "agreement_rate": 1.0, "any_round_hidden_exact": 5, "any_round_hidden_exact_rate": 1.0, "any_visible_verified_hidden_exact": 5, "any_visible_verified_hidden_exact_rate": 1.0, "both_direct_and_program": 5, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 5, "final_program_accuracy": 1.0, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 1.0, "initial_hidden_exact": 5, "initial_visible_pass": 5, "initial_visible_pass_rate": 1.0, "mean_round_count": 1.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_13": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 3, "any_round_hidden_exact_rate": 0.6, "any_visible_verified_hidden_exact": 3, "any_visible_verified_hidden_exact_rate": 0.6, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "initial_visible_pass": 3, "initial_visible_pass_rate": 0.6, "mean_round_count": 1.8, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 3, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.25 }, "synthetic_15": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_17": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_18": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_19": { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "any_round_hidden_exact": 4, "any_round_hidden_exact_rate": 0.8, "any_visible_verified_hidden_exact": 4, "any_visible_verified_hidden_exact_rate": 0.8, "both_direct_and_program": 4, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "initial_visible_pass": 4, "initial_visible_pass_rate": 0.8, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_2": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_22": { "agreement": 3, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.6, "any_round_hidden_exact": 1, "any_round_hidden_exact_rate": 0.2, "any_visible_verified_hidden_exact": 1, "any_visible_verified_hidden_exact_rate": 0.2, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 3, "initial_visible_pass_rate": 0.6, "mean_round_count": 1.8, "n": 5, "neither": 4, "oracle_union": 1, "oracle_union_rate": 0.2, "program_only": 1, "visible_false_pass": 3, "visible_false_pass_rate_among_visible": 0.75 }, "synthetic_24": { "agreement": 2, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.4, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 3.4, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 2, "visible_false_pass_rate_among_visible": 1.0 }, "synthetic_25": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_26": { "agreement": 2, "agreement_correct": 1, "agreement_precision": 0.5, "agreement_rate": 0.4, "any_round_hidden_exact": 1, "any_round_hidden_exact_rate": 0.2, "any_visible_verified_hidden_exact": 1, "any_visible_verified_hidden_exact_rate": 0.2, "both_direct_and_program": 1, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 3.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.5 }, "synthetic_27": { "agreement": 3, "agreement_correct": 2, "agreement_precision": 0.6666666666666666, "agreement_rate": 0.6, "any_round_hidden_exact": 2, "any_round_hidden_exact_rate": 0.4, "any_visible_verified_hidden_exact": 2, "any_visible_verified_hidden_exact_rate": 0.4, "both_direct_and_program": 2, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 3.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.3333333333333333 }, "synthetic_28": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_29": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 3.6, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, "synthetic_3": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_30": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_33": { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "agreement_rate": 0.4, "any_round_hidden_exact": 2, "any_round_hidden_exact_rate": 0.4, "any_visible_verified_hidden_exact": 2, "any_visible_verified_hidden_exact_rate": 0.4, "both_direct_and_program": 2, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 2, "direct_parse_ok": 4, "direct_parse_rate": 0.8, "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "initial_visible_pass": 2, "initial_visible_pass_rate": 0.4, "mean_round_count": 2.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.3333333333333333 }, "synthetic_34": { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "any_round_hidden_exact": 1, "any_round_hidden_exact_rate": 0.2, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 3.4, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, "synthetic_36": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_37": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_4": { "agreement": 4, "agreement_correct": 4, "agreement_precision": 1.0, "agreement_rate": 0.8, "any_round_hidden_exact": 4, "any_round_hidden_exact_rate": 0.8, "any_visible_verified_hidden_exact": 4, "any_visible_verified_hidden_exact_rate": 0.8, "both_direct_and_program": 4, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "initial_visible_pass": 4, "initial_visible_pass_rate": 0.8, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_40": { "agreement": 5, "agreement_correct": 4, "agreement_precision": 0.8, "agreement_rate": 1.0, "any_round_hidden_exact": 4, "any_round_hidden_exact_rate": 0.8, "any_visible_verified_hidden_exact": 4, "any_visible_verified_hidden_exact_rate": 0.8, "both_direct_and_program": 4, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 2.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.2 }, "synthetic_41": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_43": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 1.0, "direct_exact": 5, "direct_only": 5, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 1.0, "hybrid_exact": 5, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_44": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "any_round_hidden_exact": 1, "any_round_hidden_exact_rate": 0.2, "any_visible_verified_hidden_exact": 1, "any_visible_verified_hidden_exact_rate": 0.2, "both_direct_and_program": 1, "direct_accuracy": 0.6, "direct_exact": 3, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 1, "final_program_accuracy": 0.2, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 3.4, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_45": { "agreement": 3, "agreement_correct": 3, "agreement_precision": 1.0, "agreement_rate": 0.6, "any_round_hidden_exact": 4, "any_round_hidden_exact_rate": 0.8, "any_visible_verified_hidden_exact": 4, "any_visible_verified_hidden_exact_rate": 0.8, "both_direct_and_program": 3, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.8, "initial_hidden_exact": 4, "initial_visible_pass": 4, "initial_visible_pass_rate": 0.8, "mean_round_count": 1.6, "n": 5, "neither": 0, "oracle_union": 5, "oracle_union_rate": 1.0, "program_only": 1, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_46": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_47": { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "agreement_rate": 0.4, "any_round_hidden_exact": 2, "any_round_hidden_exact_rate": 0.4, "any_visible_verified_hidden_exact": 2, "any_visible_verified_hidden_exact_rate": 0.4, "both_direct_and_program": 2, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 2, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 2, "final_program_accuracy": 0.4, "final_visible_pass": 2, "final_visible_pass_rate": 0.4, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 3.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_48": { "agreement": 1, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.2, "any_round_hidden_exact": 1, "any_round_hidden_exact_rate": 0.2, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.2, "direct_exact": 1, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 1, "final_visible_pass_rate": 0.2, "hybrid_accuracy": 0.2, "hybrid_exact": 1, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 4, "oracle_union": 1, "oracle_union_rate": 0.2, "program_only": 0, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 1.0 }, "synthetic_49": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.6, "direct_exact": 3, "direct_only": 3, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.6, "hybrid_exact": 3, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 2, "oracle_union": 3, "oracle_union_rate": 0.6, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_5": { "agreement": 2, "agreement_correct": 2, "agreement_precision": 1.0, "agreement_rate": 0.4, "any_round_hidden_exact": 4, "any_round_hidden_exact_rate": 0.8, "any_visible_verified_hidden_exact": 4, "any_visible_verified_hidden_exact_rate": 0.8, "both_direct_and_program": 2, "direct_accuracy": 0.4, "direct_exact": 2, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 5, "final_visible_pass_rate": 1.0, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "initial_accuracy": 0.4, "initial_hidden_exact": 2, "initial_visible_pass": 3, "initial_visible_pass_rate": 0.6, "mean_round_count": 1.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 2, "visible_false_pass": 1, "visible_false_pass_rate_among_visible": 0.2 }, "synthetic_51": { "agreement": 1, "agreement_correct": 1, "agreement_precision": 1.0, "agreement_rate": 0.2, "any_round_hidden_exact": 3, "any_round_hidden_exact_rate": 0.6, "any_visible_verified_hidden_exact": 3, "any_visible_verified_hidden_exact_rate": 0.6, "both_direct_and_program": 1, "direct_accuracy": 0.4, "direct_exact": 2, "direct_only": 1, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 3, "final_program_accuracy": 0.6, "final_visible_pass": 3, "final_visible_pass_rate": 0.6, "hybrid_accuracy": 0.4, "hybrid_exact": 2, "initial_accuracy": 0.6, "initial_hidden_exact": 3, "initial_visible_pass": 3, "initial_visible_pass_rate": 0.6, "mean_round_count": 2.2, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 2, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_6": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.0, "hybrid_exact": 0, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 5, "oracle_union": 0, "oracle_union_rate": 0.0, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_7": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 0, "any_round_hidden_exact_rate": 0.0, "any_visible_verified_hidden_exact": 0, "any_visible_verified_hidden_exact_rate": 0.0, "both_direct_and_program": 0, "direct_accuracy": 0.8, "direct_exact": 4, "direct_only": 4, "direct_parse_ok": 5, "direct_parse_rate": 1.0, "final_hidden_exact": 0, "final_program_accuracy": 0.0, "final_visible_pass": 0, "final_visible_pass_rate": 0.0, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.0, "initial_hidden_exact": 0, "initial_visible_pass": 0, "initial_visible_pass_rate": 0.0, "mean_round_count": 4.0, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 0, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 }, "synthetic_8": { "agreement": 0, "agreement_correct": 0, "agreement_precision": 0.0, "agreement_rate": 0.0, "any_round_hidden_exact": 4, "any_round_hidden_exact_rate": 0.8, "any_visible_verified_hidden_exact": 4, "any_visible_verified_hidden_exact_rate": 0.8, "both_direct_and_program": 0, "direct_accuracy": 0.0, "direct_exact": 0, "direct_only": 0, "direct_parse_ok": 0, "direct_parse_rate": 0.0, "final_hidden_exact": 4, "final_program_accuracy": 0.8, "final_visible_pass": 4, "final_visible_pass_rate": 0.8, "hybrid_accuracy": 0.8, "hybrid_exact": 4, "initial_accuracy": 0.2, "initial_hidden_exact": 1, "initial_visible_pass": 1, "initial_visible_pass_rate": 0.2, "mean_round_count": 2.4, "n": 5, "neither": 1, "oracle_union": 4, "oracle_union_rate": 0.8, "program_only": 4, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 } }, "by_num_samples": { "1": { "agreement": 8, "agreement_correct": 4, "agreement_precision": 0.5, "agreement_rate": 0.16, "any_round_hidden_exact": 6, "any_round_hidden_exact_rate": 0.12, "any_visible_verified_hidden_exact": 6, "any_visible_verified_hidden_exact_rate": 0.12, "both_direct_and_program": 4, "direct_accuracy": 0.38, "direct_exact": 19, "direct_only": 15, "direct_parse_ok": 46, "direct_parse_rate": 0.92, "final_hidden_exact": 6, "final_program_accuracy": 0.12, "final_visible_pass": 15, "final_visible_pass_rate": 0.3, "hybrid_accuracy": 0.4, "hybrid_exact": 20, "initial_accuracy": 0.08, "initial_hidden_exact": 4, "initial_visible_pass": 11, "initial_visible_pass_rate": 0.22, "mean_round_count": 3.18, "n": 50, "neither": 29, "oracle_union": 21, "oracle_union_rate": 0.42, "program_only": 2, "visible_false_pass": 9, "visible_false_pass_rate_among_visible": 0.6 }, "2": { "agreement": 9, "agreement_correct": 8, "agreement_precision": 0.8888888888888888, "agreement_rate": 0.18, "any_round_hidden_exact": 10, "any_round_hidden_exact_rate": 0.2, "any_visible_verified_hidden_exact": 10, "any_visible_verified_hidden_exact_rate": 0.2, "both_direct_and_program": 8, "direct_accuracy": 0.58, "direct_exact": 29, "direct_only": 21, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 10, "final_program_accuracy": 0.2, "final_visible_pass": 13, "final_visible_pass_rate": 0.26, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "initial_accuracy": 0.14, "initial_hidden_exact": 7, "initial_visible_pass": 8, "initial_visible_pass_rate": 0.16, "mean_round_count": 3.38, "n": 50, "neither": 19, "oracle_union": 31, "oracle_union_rate": 0.62, "program_only": 2, "visible_false_pass": 3, "visible_false_pass_rate_among_visible": 0.23076923076923078 }, "3": { "agreement": 16, "agreement_correct": 12, "agreement_precision": 0.75, "agreement_rate": 0.32, "any_round_hidden_exact": 18, "any_round_hidden_exact_rate": 0.36, "any_visible_verified_hidden_exact": 17, "any_visible_verified_hidden_exact_rate": 0.34, "both_direct_and_program": 12, "direct_accuracy": 0.6, "direct_exact": 30, "direct_only": 18, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 17, "final_program_accuracy": 0.34, "final_visible_pass": 19, "final_visible_pass_rate": 0.38, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "initial_accuracy": 0.22, "initial_hidden_exact": 11, "initial_visible_pass": 12, "initial_visible_pass_rate": 0.24, "mean_round_count": 3.08, "n": 50, "neither": 15, "oracle_union": 35, "oracle_union_rate": 0.7, "program_only": 5, "visible_false_pass": 2, "visible_false_pass_rate_among_visible": 0.10526315789473684 }, "4": { "agreement": 12, "agreement_correct": 11, "agreement_precision": 0.9166666666666666, "agreement_rate": 0.24, "any_round_hidden_exact": 16, "any_round_hidden_exact_rate": 0.32, "any_visible_verified_hidden_exact": 15, "any_visible_verified_hidden_exact_rate": 0.3, "both_direct_and_program": 11, "direct_accuracy": 0.62, "direct_exact": 31, "direct_only": 20, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 15, "final_program_accuracy": 0.3, "final_visible_pass": 17, "final_visible_pass_rate": 0.34, "hybrid_accuracy": 0.64, "hybrid_exact": 32, "initial_accuracy": 0.18, "initial_hidden_exact": 9, "initial_visible_pass": 10, "initial_visible_pass_rate": 0.2, "mean_round_count": 3.16, "n": 50, "neither": 15, "oracle_union": 35, "oracle_union_rate": 0.7, "program_only": 4, "visible_false_pass": 2, "visible_false_pass_rate_among_visible": 0.11764705882352941 }, "5": { "agreement": 10, "agreement_correct": 9, "agreement_precision": 0.9, "agreement_rate": 0.2, "any_round_hidden_exact": 14, "any_round_hidden_exact_rate": 0.28, "any_visible_verified_hidden_exact": 14, "any_visible_verified_hidden_exact_rate": 0.28, "both_direct_and_program": 9, "direct_accuracy": 0.58, "direct_exact": 29, "direct_only": 20, "direct_parse_ok": 47, "direct_parse_rate": 0.94, "final_hidden_exact": 14, "final_program_accuracy": 0.28, "final_visible_pass": 14, "final_visible_pass_rate": 0.28, "hybrid_accuracy": 0.6, "hybrid_exact": 30, "initial_accuracy": 0.18, "initial_hidden_exact": 9, "initial_visible_pass": 9, "initial_visible_pass_rate": 0.18, "mean_round_count": 3.3, "n": 50, "neither": 16, "oracle_union": 34, "oracle_union_rate": 0.68, "program_only": 5, "visible_false_pass": 0, "visible_false_pass_rate_among_visible": 0.0 } }, "overall": { "agreement": 55, "agreement_correct": 44, "agreement_precision": 0.8, "agreement_rate": 0.22, "any_round_hidden_exact": 64, "any_round_hidden_exact_rate": 0.256, "any_visible_verified_hidden_exact": 62, "any_visible_verified_hidden_exact_rate": 0.248, "both_direct_and_program": 44, "direct_accuracy": 0.552, "direct_exact": 138, "direct_only": 94, "direct_parse_ok": 234, "direct_parse_rate": 0.936, "final_hidden_exact": 62, "final_program_accuracy": 0.248, "final_visible_pass": 78, "final_visible_pass_rate": 0.312, "hybrid_accuracy": 0.568, "hybrid_exact": 142, "initial_accuracy": 0.16, "initial_hidden_exact": 40, "initial_visible_pass": 50, "initial_visible_pass_rate": 0.2, "mean_round_count": 3.22, "n": 250, "neither": 94, "oracle_union": 156, "oracle_union_rate": 0.624, "program_only": 18, "visible_false_pass": 16, "visible_false_pass_rate_among_visible": 0.20512820512820512 }, "round_stats": { "0": { "attempted": 250, "hidden_exact": 41, "visible_pass": 50, "visible_verified_hidden_exact": 40 }, "1": { "attempted": 200, "hidden_exact": 17, "visible_pass": 20, "visible_verified_hidden_exact": 16 }, "2": { "attempted": 180, "hidden_exact": 6, "visible_pass": 5, "visible_verified_hidden_exact": 5 }, "3": { "attempted": 175, "hidden_exact": 1, "visible_pass": 3, "visible_verified_hidden_exact": 1 } } }