{ "controller_selection": { "num_candidates_after_train_gate": 4468, "selected": { "dev": { "accuracy": 0.5, "avg_forward_tokens": 682.1, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "split_fold_unpivot": 3 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 6821 }, "policy": { "false_action": "direct_only", "feature": "direct_out_cols_less_new_in", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1253.6333333333334, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "split_fold_unpivot": 6 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 1, "program_commit_precision": 0.3333333333333333, "program_commits": 3, "total_forward_tokens": 37609 } }, "top_candidates": [ { "dev": { "accuracy": 0.5, "avg_forward_tokens": 682.1, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "split_fold_unpivot": 3 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 6821 }, "policy": { "false_action": "direct_only", "feature": "direct_out_cols_less_new_in", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1253.6333333333334, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "split_fold_unpivot": 6 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 1, "program_commit_precision": 0.3333333333333333, "program_commits": 3, "total_forward_tokens": 37609 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 993.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "split_fold_unpivot": 4 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 9939 }, "policy": { "false_action": "direct_only", "feature": "visible_out_less_cols", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1308.7, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "split_fold_unpivot": 7 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 1, "program_commit_precision": 0.3333333333333333, "program_commits": 3, "total_forward_tokens": 39261 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1133.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "row_column_rule": 4 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 11339 }, "policy": { "false_action": "single_row_column_rule", "feature": "visible_rows_change", "true_action": "direct_only" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1106.0333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "row_column_rule": 7 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 2, "total_forward_tokens": 33181 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1158.8, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "split_fold_unpivot": 5 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 11588 }, "policy": { "false_action": "direct_only", "feature": "visible_row_expansion_or_col_contraction", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1409.1666666666667, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "split_fold_unpivot": 10 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 3, "program_commit_precision": 0.6, "program_commits": 5, "total_forward_tokens": 42275 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1305.0, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 3, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 13050 }, "policy": { "false_action": "direct_only", "feature": "direct_out_cols_less_new_in", "true_action": "canary_split_fold_unpivot_disagree_escalate" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1478.9666666666667, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 6, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 44369 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1587.2, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 4, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 15872 }, "policy": { "false_action": "canary_split_fold_unpivot_disagree_escalate", "feature": "visible_rows_change", "true_action": "direct_only" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1067.1333333333334, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "split_fold_unpivot": 7 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 32014 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1601.6, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 2, "header_aware": 2, "row_column_rule": 3, "split_fold_unpivot": 2, "verified_structural": 2 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 2, "program_commit_precision": 1.0, "program_commits": 2, "total_forward_tokens": 16016 }, "policy": { "false_action": "direct_only", "feature": "direct_out_cols_less_new_in", "true_action": "canary_row_column_rule_disagree_escalate" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1576.9666666666667, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 6, "split_fold_unpivot": 1, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 47309 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1616.8, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 4, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 16168 }, "policy": { "false_action": "direct_only", "feature": "visible_out_less_cols", "true_action": "canary_split_fold_unpivot_disagree_escalate" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1534.0333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 7, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 46021 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1716.2, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 4, "split_fold_unpivot": 1, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 17162 }, "policy": { "false_action": "canary_row_column_rule_disagree_escalate", "feature": "visible_rows_change", "true_action": "direct_only" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1106.0333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "row_column_rule": 7 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 33181 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1781.7, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 5, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 17817 }, "policy": { "false_action": "direct_only", "feature": "visible_row_expansion_or_col_contraction", "true_action": "canary_split_fold_unpivot_disagree_escalate" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1634.5, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 10, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 49035 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1811.4, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 6, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 18114 }, "policy": { "false_action": "canary_split_fold_unpivot_disagree_escalate", "feature": "direct_out_rows_less_new_in", "true_action": "direct_only" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1434.8, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 12, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 43044 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1811.4, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 6, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 18114 }, "policy": { "false_action": "canary_split_fold_unpivot_disagree_escalate", "feature": "visible_col_expansion_or_row_contraction", "true_action": "direct_only" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1770.0333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 13, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 53101 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1811.4, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 6, "verified_structural": 1 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 18114 }, "policy": { "false_action": "canary_split_fold_unpivot_disagree_escalate", "feature": "visible_out_less_rows", "true_action": "direct_only" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1846.8, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 1, "split_fold_unpivot": 15, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 55404 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1865.5, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 7, "split_fold_unpivot": 3 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 18655 }, "policy": { "false_action": "single_cell_parser", "feature": "direct_out_cols_less_new_in", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 2610.5333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "cell_parser": 24, "split_fold_unpivot": 6 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 6, "program_commit_precision": 0.5, "program_commits": 12, "total_forward_tokens": 78316 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1865.5, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 7, "split_fold_unpivot": 3 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 18655 }, "policy": { "false_action": "canary_cell_parser_disagree_escalate", "feature": "direct_out_cols_less_new_in", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 3416.9, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "cell_parser": 24, "header_aware": 3, "row_column_rule": 3, "split_fold_unpivot": 9, "verified_structural": 3 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 3, "program_commit_precision": 0.5, "program_commits": 6, "total_forward_tokens": 102507 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1865.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 2, "header_aware": 2, "row_column_rule": 4, "split_fold_unpivot": 2, "verified_structural": 2 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 2, "program_commit_precision": 1.0, "program_commits": 2, "total_forward_tokens": 18659 }, "policy": { "false_action": "direct_only", "feature": "visible_out_less_cols", "true_action": "canary_row_column_rule_disagree_escalate" }, "train": { "accuracy": 0.3, "avg_forward_tokens": 1657.3666666666666, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 0, "exact": 9, "generated_variant_counts": { "cell_parser": 1, "header_aware": 1, "row_column_rule": 7, "split_fold_unpivot": 1, "verified_structural": 1 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 0, "program_commit_precision": 0.0, "program_commits": 1, "total_forward_tokens": 49721 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1927.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "row_column_rule": 4, "verified_structural": 6 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 2, "program_commit_precision": 1.0, "program_commits": 2, "total_forward_tokens": 19279 }, "policy": { "false_action": "single_row_column_rule", "feature": "visible_rows_change", "true_action": "single_verified_structural" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 2731.366666666667, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "row_column_rule": 7, "verified_structural": 23 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 4, "program_commit_precision": 0.5714285714285714, "program_commits": 7, "total_forward_tokens": 81941 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1927.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "row_column_rule": 4, "verified_structural": 6 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 2, "program_commit_precision": 1.0, "program_commits": 2, "total_forward_tokens": 19279 }, "policy": { "false_action": "single_row_column_rule", "feature": "visible_rows_change", "true_action": "prefix1_first_visible" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 2731.366666666667, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "row_column_rule": 7, "verified_structural": 23 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 4, "program_commit_precision": 0.5714285714285714, "program_commits": 7, "total_forward_tokens": 81941 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1927.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "row_column_rule": 4, "verified_structural": 6 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 19279 }, "policy": { "false_action": "single_row_column_rule", "feature": "visible_rows_change", "true_action": "canary_verified_structural_disagree_escalate" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 3145.9666666666667, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "cell_parser": 2, "header_aware": 2, "row_column_rule": 9, "split_fold_unpivot": 2, "verified_structural": 23 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 2, "program_commit_precision": 0.5, "program_commits": 4, "total_forward_tokens": 94379 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1997.0, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 6, "split_fold_unpivot": 4 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 19970 }, "policy": { "false_action": "single_cell_parser", "feature": "visible_out_less_cols", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 2585.0333333333333, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "cell_parser": 23, "split_fold_unpivot": 7 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 6, "program_commit_precision": 0.5, "program_commits": 12, "total_forward_tokens": 77551 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 1997.0, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 6, "split_fold_unpivot": 4 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 19970 }, "policy": { "false_action": "canary_cell_parser_disagree_escalate", "feature": "visible_out_less_cols", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 3391.4, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "cell_parser": 23, "header_aware": 3, "row_column_rule": 3, "split_fold_unpivot": 10, "verified_structural": 3 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 3, "program_commit_precision": 0.5, "program_commits": 6, "total_forward_tokens": 101742 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 2011.0, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 5, "split_fold_unpivot": 5 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 20110 }, "policy": { "false_action": "single_cell_parser", "feature": "visible_row_expansion_or_col_contraction", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 2623.0, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "cell_parser": 20, "split_fold_unpivot": 10 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 6, "program_commit_precision": 0.5, "program_commits": 12, "total_forward_tokens": 78690 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 2011.0, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 5, "split_fold_unpivot": 5 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 3, "program_commit_precision": 1.0, "program_commits": 3, "total_forward_tokens": 20110 }, "policy": { "false_action": "canary_cell_parser_disagree_escalate", "feature": "visible_row_expansion_or_col_contraction", "true_action": "single_split_fold_unpivot" }, "train": { "accuracy": 0.36666666666666664, "avg_forward_tokens": 3429.366666666667, "direct_accuracy": 0.3, "direct_correct_losses": 0, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 11, "generated_variant_counts": { "cell_parser": 20, "header_aware": 3, "row_column_rule": 3, "split_fold_unpivot": 13, "verified_structural": 3 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 5, "program_commit_precision": 0.625, "program_commits": 8, "total_forward_tokens": 102881 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 2069.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 6, "row_column_rule": 4 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 2, "program_commit_precision": 1.0, "program_commits": 2, "total_forward_tokens": 20699 }, "policy": { "false_action": "single_row_column_rule", "feature": "visible_rows_change", "true_action": "single_cell_parser" }, "train": { "accuracy": 0.3333333333333333, "avg_forward_tokens": 2730.233333333333, "direct_accuracy": 0.3, "direct_correct_losses": 1, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 10, "generated_variant_counts": { "cell_parser": 23, "row_column_rule": 7 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 6, "program_commit_precision": 0.5, "program_commits": 12, "total_forward_tokens": 81907 } }, { "dev": { "accuracy": 0.5, "avg_forward_tokens": 2069.9, "direct_accuracy": 0.4, "direct_correct_losses": 0, "direct_exact": 4, "direct_miss_recoveries": 1, "exact": 5, "generated_variant_counts": { "cell_parser": 6, "row_column_rule": 4 }, "n": 10, "policy": "candidate_dev", "program_commit_correct": 1, "program_commit_precision": 1.0, "program_commits": 1, "total_forward_tokens": 20699 }, "policy": { "false_action": "single_row_column_rule", "feature": "visible_rows_change", "true_action": "canary_cell_parser_disagree_escalate" }, "train": { "accuracy": 0.3333333333333333, "avg_forward_tokens": 3979.9333333333334, "direct_accuracy": 0.3, "direct_correct_losses": 1, "direct_exact": 9, "direct_miss_recoveries": 2, "exact": 10, "generated_variant_counts": { "cell_parser": 23, "header_aware": 5, "row_column_rule": 12, "split_fold_unpivot": 5, "verified_structural": 5 }, "n": 30, "policy": "candidate_train", "program_commit_correct": 2, "program_commit_precision": 0.2857142857142857, "program_commits": 7, "total_forward_tokens": 119398 } } ] }, "deployable_pareto_front": [ "direct_only", "learned_controller", "fixed_visible_out_less_cols_full_else_direct" ], "family_metrics": { "agriculture": { "accuracy": 0.8, "direct_accuracy": 0.8, "direct_exact": 4, "exact": 4, "losses": 0, "n": 5, "program_commits": 0, "recoveries": 0, "tokens": 5453 }, "crime_data_wrangler": { "accuracy": 0.0, "direct_accuracy": 0.0, "direct_exact": 0, "exact": 0, "losses": 0, "n": 5, "program_commits": 5, "recoveries": 0, "tokens": 66089 }, "potters_wheel_merge_split": { "accuracy": 0.4, "direct_accuracy": 0.0, "direct_exact": 0, "exact": 2, "losses": 0, "n": 5, "program_commits": 4, "recoveries": 2, "tokens": 5696 }, "potters_wheel_unfold": { "accuracy": 0.4, "direct_accuracy": 0.4, "direct_exact": 2, "exact": 2, "losses": 0, "n": 5, "program_commits": 1, "recoveries": 0, "tokens": 3987 }, "potters_wheel_unfold2": { "accuracy": 0.0, "direct_accuracy": 0.0, "direct_exact": 0, "exact": 0, "losses": 0, "n": 5, "program_commits": 0, "recoveries": 0, "tokens": 1409 }, "proactive_wrangling_fold": { "accuracy": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "exact": 5, "losses": 0, "n": 5, "program_commits": 0, "recoveries": 0, "tokens": 1708 }, "synthetic_12": { "accuracy": 1.0, "direct_accuracy": 1.0, "direct_exact": 5, "exact": 5, "losses": 0, "n": 5, "program_commits": 0, "recoveries": 0, "tokens": 11102 }, "synthetic_25": { "accuracy": 0.8, "direct_accuracy": 0.8, "direct_exact": 4, "exact": 4, "losses": 0, "n": 5, "program_commits": 0, "recoveries": 0, "tokens": 9942 }, "synthetic_48": { "accuracy": 0.2, "direct_accuracy": 0.2, "direct_exact": 1, "exact": 1, "losses": 0, "n": 5, "program_commits": 0, "recoveries": 0, "tokens": 2242 }, "synthetic_8": { "accuracy": 0.4, "direct_accuracy": 0.0, "direct_exact": 0, "exact": 2, "losses": 0, "n": 5, "program_commits": 5, "recoveries": 2, "tokens": 40823 } }, "selected_test_metrics": { "accuracy": 0.5, "avg_forward_tokens": 2969.02, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 25, "generated_variant_counts": { "split_fold_unpivot": 15 }, "n": 50, "policy": "learned_controller", "program_commit_correct": 4, "program_commit_precision": 1.0, "program_commits": 4, "total_forward_tokens": 148451 }, "selected_test_policy": "learned_controller", "test_metrics": [ { "accuracy": 0.58, "avg_forward_tokens": 3959.02, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 29, "generated_variant_counts": { "cell_parser": 8, "header_aware": 8, "row_column_rule": 8, "split_fold_unpivot": 8, "verified_structural": 8 }, "n": 50, "policy": "oracle_budget_full_only_when_helpful", "program_commit_correct": 8, "program_commit_precision": 1.0, "program_commits": 8, "total_forward_tokens": 197951 }, { "accuracy": 0.58, "avg_forward_tokens": 4627.3, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 29, "generated_variant_counts": { "cell_parser": 9, "header_aware": 9, "row_column_rule": 9, "split_fold_unpivot": 9, "verified_structural": 9 }, "n": 50, "policy": "fixed_visible_out_less_cols_full_else_direct", "program_commit_correct": 8, "program_commit_precision": 1.0, "program_commits": 8, "total_forward_tokens": 231365 }, { "accuracy": 0.58, "avg_forward_tokens": 9017.06, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 29, "generated_variant_counts": { "cell_parser": 15, "header_aware": 15, "row_column_rule": 15, "split_fold_unpivot": 15, "verified_structural": 15 }, "n": 50, "policy": "fixed_direct_out_cols_less_new_in_full_else_direct", "program_commit_correct": 8, "program_commit_precision": 1.0, "program_commits": 8, "total_forward_tokens": 450853 }, { "accuracy": 0.58, "avg_forward_tokens": 13984.16, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 29, "generated_variant_counts": { "cell_parser": 34, "header_aware": 34, "row_column_rule": 34, "split_fold_unpivot": 34, "verified_structural": 34 }, "n": 50, "policy": "fixed_visible_cols_change_full_else_direct", "program_commit_correct": 9, "program_commit_precision": 0.6923076923076923, "program_commits": 13, "total_forward_tokens": 699208 }, { "accuracy": 0.58, "avg_forward_tokens": 17800.6, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 29, "generated_variant_counts": { "cell_parser": 50, "header_aware": 50, "row_column_rule": 50, "split_fold_unpivot": 50, "verified_structural": 50 }, "n": 50, "policy": "oracle_best_available_full_budget", "program_commit_correct": 8, "program_commit_precision": 1.0, "program_commits": 8, "total_forward_tokens": 890030 }, { "accuracy": 0.56, "avg_forward_tokens": 17800.6, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 8, "exact": 28, "generated_variant_counts": { "cell_parser": 50, "header_aware": 50, "row_column_rule": 50, "split_fold_unpivot": 50, "verified_structural": 50 }, "n": 50, "policy": "prefix5_first_visible", "program_commit_correct": 18, "program_commit_precision": 0.782608695652174, "program_commits": 23, "total_forward_tokens": 890030 }, { "accuracy": 0.54, "avg_forward_tokens": 10777.5, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 7, "exact": 27, "generated_variant_counts": { "cell_parser": 50, "row_column_rule": 50, "verified_structural": 50 }, "n": 50, "policy": "prefix3_first_visible", "program_commit_correct": 17, "program_commit_precision": 0.85, "program_commits": 20, "total_forward_tokens": 538875 }, { "accuracy": 0.54, "avg_forward_tokens": 14513.14, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 7, "exact": 27, "generated_variant_counts": { "cell_parser": 50, "header_aware": 50, "row_column_rule": 50, "verified_structural": 50 }, "n": 50, "policy": "prefix4_first_visible", "program_commit_correct": 17, "program_commit_precision": 0.85, "program_commits": 20, "total_forward_tokens": 725657 }, { "accuracy": 0.52, "avg_forward_tokens": 7392.14, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 6, "exact": 26, "generated_variant_counts": { "cell_parser": 50, "verified_structural": 50 }, "n": 50, "policy": "prefix2_first_visible", "program_commit_correct": 16, "program_commit_precision": 0.8421052631578947, "program_commits": 19, "total_forward_tokens": 369607 }, { "accuracy": 0.5, "avg_forward_tokens": 2969.02, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 25, "generated_variant_counts": { "split_fold_unpivot": 15 }, "n": 50, "policy": "learned_controller", "program_commit_correct": 4, "program_commit_precision": 1.0, "program_commits": 4, "total_forward_tokens": 148451 }, { "accuracy": 0.5, "avg_forward_tokens": 4291.24, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 5, "exact": 25, "generated_variant_counts": { "verified_structural": 50 }, "n": 50, "policy": "single_verified_structural", "program_commit_correct": 14, "program_commit_precision": 0.8235294117647058, "program_commits": 17, "total_forward_tokens": 214562 }, { "accuracy": 0.5, "avg_forward_tokens": 4291.24, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 5, "exact": 25, "generated_variant_counts": { "verified_structural": 50 }, "n": 50, "policy": "prefix1_first_visible", "program_commit_correct": 14, "program_commit_precision": 0.8235294117647058, "program_commits": 17, "total_forward_tokens": 214562 }, { "accuracy": 0.5, "avg_forward_tokens": 4765.68, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 25, "generated_variant_counts": { "split_fold_unpivot": 50 }, "n": 50, "policy": "single_split_fold_unpivot", "program_commit_correct": 9, "program_commit_precision": 0.8181818181818182, "program_commits": 11, "total_forward_tokens": 238284 }, { "accuracy": 0.5, "avg_forward_tokens": 4863.58, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 25, "generated_variant_counts": { "row_column_rule": 50 }, "n": 50, "policy": "single_row_column_rule", "program_commit_correct": 10, "program_commit_precision": 0.8333333333333334, "program_commits": 12, "total_forward_tokens": 243179 }, { "accuracy": 0.5, "avg_forward_tokens": 5604.12, "direct_accuracy": 0.42, "direct_correct_losses": 1, "direct_exact": 21, "direct_miss_recoveries": 5, "exact": 25, "generated_variant_counts": { "cell_parser": 8, "header_aware": 8, "row_column_rule": 8, "split_fold_unpivot": 8, "verified_structural": 50 }, "n": 50, "policy": "canary_verified_structural_disagree_escalate", "program_commit_correct": 5, "program_commit_precision": 0.625, "program_commits": 8, "total_forward_tokens": 280206 }, { "accuracy": 0.5, "avg_forward_tokens": 5724.42, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 25, "generated_variant_counts": { "cell_parser": 6, "header_aware": 6, "row_column_rule": 50, "split_fold_unpivot": 6, "verified_structural": 6 }, "n": 50, "policy": "canary_row_column_rule_disagree_escalate", "program_commit_correct": 4, "program_commit_precision": 0.6666666666666666, "program_commits": 6, "total_forward_tokens": 286221 }, { "accuracy": 0.5, "avg_forward_tokens": 5982.86, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 4, "exact": 25, "generated_variant_counts": { "cell_parser": 5, "header_aware": 5, "row_column_rule": 5, "split_fold_unpivot": 50, "verified_structural": 5 }, "n": 50, "policy": "canary_split_fold_unpivot_disagree_escalate", "program_commit_correct": 4, "program_commit_precision": 0.8, "program_commits": 5, "total_forward_tokens": 299143 }, { "accuracy": 0.42, "avg_forward_tokens": 1478.22, "direct_accuracy": 0.42, "direct_correct_losses": 0, "direct_exact": 21, "direct_miss_recoveries": 0, "exact": 21, "generated_variant_counts": {}, "n": 50, "policy": "direct_only", "program_commit_correct": 0, "program_commit_precision": null, "program_commits": 0, "total_forward_tokens": 73911 } ], "variant_order": [ "verified_structural", "cell_parser", "row_column_rule", "header_aware", "split_fold_unpivot" ] }