{ "selection": { "candidate_variants": [ "verified_structural", "row_column_rule", "split_fold_unpivot", "header_aware", "cell_parser" ], "family_splits": { "dev": [ "potters_wheel_fold_2", "synthetic_10", "synthetic_17", "synthetic_24", "synthetic_30", "synthetic_37", "synthetic_40", "synthetic_45", "synthetic_46", "synthetic_51" ], "test": [ "agriculture", "crime_data_wrangler", "potters_wheel_merge_split", "potters_wheel_unfold", "potters_wheel_unfold2", "proactive_wrangling_fold", "synthetic_12", "synthetic_25", "synthetic_48", "synthetic_8" ], "train": [ "craigslist_data_wrangler", "potters_wheel_divide", "potters_wheel_fold", "potters_wheel_split_fold", "proactive_wrangling_complex", "reshape_table_structure_data_wrangler", "synthetic_11", "synthetic_13", "synthetic_15", "synthetic_18", "synthetic_19", "synthetic_2", "synthetic_22", "synthetic_26", "synthetic_27", "synthetic_28", "synthetic_29", "synthetic_3", "synthetic_33", "synthetic_34", "synthetic_36", "synthetic_4", "synthetic_41", "synthetic_43", "synthetic_44", "synthetic_47", "synthetic_49", "synthetic_5", "synthetic_6", "synthetic_7" ] }, "greedy_order": [ "verified_structural", "cell_parser", "row_column_rule", "header_aware", "split_fold_unpivot" ], "selected_k": 5, "selected_policy": "consensus_2", "selected_variants": [ "verified_structural", "cell_parser", "row_column_rule", "header_aware", "split_fold_unpivot" ], "selection_rule": "greedy ordered variants on train; choose prefix and selector maximizing dev exact accuracy with loss/precision/token tie-breaks" }, "split_metrics": { "dev": { "direct": { "accuracy": 0.4, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 0, "exact": 4, "n": 10, "oracle_union": 4, "oracle_union_accuracy": 0.4, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 5235 }, "selected": { "accuracy": 0.5, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "n": 10, "oracle_union": 5, "oracle_union_accuracy": 0.5, "oracle_visible_program": 3, "program_commit_correct": 2, "program_commit_precision": 1.0, "program_commits": 2, "total_forward_tokens": 87242 }, "selected_consensus_2": { "accuracy": 0.5, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "n": 10, "oracle_union": 5, "oracle_union_accuracy": 0.5, "oracle_visible_program": 3, "program_commit_correct": 2, "program_commit_precision": 1.0, "program_commits": 2, "total_forward_tokens": 87242 }, "selected_first_visible": { "accuracy": 0.4, "direct_accuracy": 0.4, "direct_correct_losses": 1, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 4, "n": 10, "oracle_union": 5, "oracle_union_accuracy": 0.5, "oracle_visible_program": 3, "program_commit_correct": 3, "program_commit_precision": 0.42857142857142855, "program_commits": 7, "total_forward_tokens": 87242 }, "selected_oracle": { "accuracy": 0.4, "direct_accuracy": 0.4, "direct_correct_losses": 1, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 4, "n": 10, "oracle_union": 5, "oracle_union_accuracy": 0.5, "oracle_visible_program": 3, "program_commit_correct": 3, "program_commit_precision": 0.42857142857142855, "program_commits": 7, "total_forward_tokens": 87242 } }, "test": { "direct": { "accuracy": 0.42, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 0, "exact": 21, "n": 50, "oracle_union": 21, "oracle_union_accuracy": 0.42, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 73911 }, "selected": { "accuracy": 0.48, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 24, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 11, "program_commit_precision": 0.7857142857142857, "program_commits": 14, "total_forward_tokens": 890030 }, "selected_consensus_2": { "accuracy": 0.48, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 24, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 11, "program_commit_precision": 0.7857142857142857, "program_commits": 14, "total_forward_tokens": 890030 }, "selected_first_visible": { "accuracy": 0.56, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 28, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 18, "program_commit_precision": 0.782608695652174, "program_commits": 23, "total_forward_tokens": 890030 }, "selected_oracle": { "accuracy": 0.56, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 28, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 18, "program_commit_precision": 0.782608695652174, "program_commits": 23, "total_forward_tokens": 890030 } }, "train": { "direct": { "accuracy": 0.3, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "n": 30, "oracle_union": 9, "oracle_union_accuracy": 0.3, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 23539 }, "selected": { "accuracy": 0.3333333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 1, "exact": 10, "n": 30, "oracle_union": 12, "oracle_union_accuracy": 0.4, "oracle_visible_program": 10, "program_commit_correct": 4, "program_commit_precision": 0.4, "program_commits": 10, "total_forward_tokens": 341451 }, "selected_consensus_2": { "accuracy": 0.3333333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 1, "exact": 10, "n": 30, "oracle_union": 12, "oracle_union_accuracy": 0.4, "oracle_visible_program": 10, "program_commit_correct": 4, "program_commit_precision": 0.4, "program_commits": 10, "total_forward_tokens": 341451 }, "selected_first_visible": { "accuracy": 0.36666666666666664, "direct_accuracy": 0.3, "direct_correct_losses": 1, "direct_exact": 9, "direct_miss_recoveries": 3, "exact": 11, "n": 30, "oracle_union": 12, "oracle_union_accuracy": 0.4, "oracle_visible_program": 10, "program_commit_correct": 10, "program_commit_precision": 0.45454545454545453, "program_commits": 22, "total_forward_tokens": 341451 }, "selected_oracle": { "accuracy": 0.36666666666666664, "direct_accuracy": 0.3, "direct_correct_losses": 1, "direct_exact": 9, "direct_miss_recoveries": 3, "exact": 11, "n": 30, "oracle_union": 12, "oracle_union_accuracy": 0.4, "oracle_visible_program": 10, "program_commit_correct": 10, "program_commit_precision": 0.45454545454545453, "program_commits": 22, "total_forward_tokens": 341451 } } }, "test_family_metrics": { "agriculture": { "accuracy": 0.6, "direct_accuracy": 0.8, "direct_correct_losses": 1, "direct_exact": 4, "direct_miss_recoveries": 0, "exact": 3, "n": 5, "oracle_union": 4, "oracle_union_accuracy": 0.8, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 64187 }, "crime_data_wrangler": { "accuracy": 0.0, "direct_accuracy": 0.0, "direct_correct_losses": 0, "direct_exact": 0, "direct_miss_recoveries": 0, "exact": 0, "n": 5, "oracle_union": 0, "oracle_union_accuracy": 0.0, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 233360 }, "potters_wheel_merge_split": { "accuracy": 0.6, "direct_accuracy": 0.0, "direct_correct_losses": 0, "direct_exact": 0, "direct_miss_recoveries": 3, "exact": 3, "n": 5, "oracle_union": 4, "oracle_union_accuracy": 0.8, "oracle_visible_program": 4, "program_commit_correct": 3, "program_commit_precision": 0.75, "program_commits": 4, "total_forward_tokens": 24387 }, "potters_wheel_unfold": { "accuracy": 0.4, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 2, "direct_miss_recoveries": 0, "exact": 2, "n": 5, "oracle_union": 2, "oracle_union_accuracy": 0.4, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 56662 }, "potters_wheel_unfold2": { "accuracy": 0.0, "direct_accuracy": 0.0, "direct_correct_losses": 0, "direct_exact": 0, "direct_miss_recoveries": 0, "exact": 0, "n": 5, "oracle_union": 0, "oracle_union_accuracy": 0.0, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 41700 }, "proactive_wrangling_fold": { "accuracy": 1.0, "direct_accuracy": 1.0, "direct_correct_losses": 0, "direct_exact": 5, "direct_miss_recoveries": 0, "exact": 5, "n": 5, "oracle_union": 5, "oracle_union_accuracy": 1.0, "oracle_visible_program": 4, "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 30184 }, "synthetic_12": { "accuracy": 1.0, "direct_accuracy": 1.0, "direct_correct_losses": 0, "direct_exact": 5, "direct_miss_recoveries": 0, "exact": 5, "n": 5, "oracle_union": 5, "oracle_union_accuracy": 1.0, "oracle_visible_program": 5, "program_commit_correct": 4, "program_commit_precision": 1.0, "program_commits": 4, "total_forward_tokens": 105112 }, "synthetic_25": { "accuracy": 0.8, "direct_accuracy": 0.8, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 0, "exact": 4, "n": 5, "oracle_union": 4, "oracle_union_accuracy": 0.8, "oracle_visible_program": 1, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 128627 }, "synthetic_48": { "accuracy": 0.2, "direct_accuracy": 0.2, "direct_correct_losses": 0, "direct_exact": 1, "direct_miss_recoveries": 0, "exact": 1, "n": 5, "oracle_union": 1, "oracle_union_accuracy": 0.2, "oracle_visible_program": 0, "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 54213 }, "synthetic_8": { "accuracy": 0.2, "direct_accuracy": 0.0, "direct_correct_losses": 0, "direct_exact": 0, "direct_miss_recoveries": 1, "exact": 1, "n": 5, "oracle_union": 4, "oracle_union_accuracy": 0.8, "oracle_visible_program": 4, "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 151598 } }, "test_policy_metrics": { "consensus_2": { "accuracy": 0.48, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 24, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 11, "program_commit_precision": 0.7857142857142857, "program_commits": 14, "total_forward_tokens": 890030 }, "consensus_3": { "accuracy": 0.48, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 3, "exact": 24, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 10, "program_commit_precision": 0.9090909090909091, "program_commits": 11, "total_forward_tokens": 890030 }, "direct": { "accuracy": 0.42, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 0, "exact": 21, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 890030 }, "first_visible_program": { "accuracy": 0.56, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 28, "n": 50, "oracle_union": 29, "oracle_union_accuracy": 0.58, "oracle_visible_program": 18, "program_commit_correct": 18, "program_commit_precision": 0.782608695652174, "program_commits": 23, "total_forward_tokens": 890030 } }, "variant_quality": [ { "initial_visible_pass": 20, "n": 90, "repair_added_visible": 3, "variant": "cell_parser", "visible_pass": 23, "visible_pass_rate": 0.25555555555555554, "visible_precision": 0.6521739130434783 }, { "initial_visible_pass": 6, "n": 90, "repair_added_visible": 9, "variant": "header_aware", "visible_pass": 15, "visible_pass_rate": 0.16666666666666666, "visible_precision": 0.4666666666666667 }, { "initial_visible_pass": 16, "n": 90, "repair_added_visible": 6, "variant": "row_column_rule", "visible_pass": 22, "visible_pass_rate": 0.24444444444444444, "visible_precision": 0.6363636363636364 }, { "initial_visible_pass": 21, "n": 90, "repair_added_visible": 7, "variant": "split_fold_unpivot", "visible_pass": 28, "visible_pass_rate": 0.3111111111111111, "visible_precision": 0.5714285714285714 }, { "initial_visible_pass": 29, "n": 90, "repair_added_visible": 3, "variant": "verified_structural", "visible_pass": 32, "visible_pass_rate": 0.35555555555555557, "visible_precision": 0.625 } ] }