{
  "run": "smoke_counterfactual_pref_v2",
  "args": {
    "run_name": "smoke_counterfactual_pref_v2",
    "model_name": "Qwen/Qwen3-4B",
    "seed": 131,
    "device": "",
    "seed_train_size": 16,
    "unlabeled_train_size": 32,
    "full_supervised_size": 32,
    "val_size": 16,
    "fresh_size": 16,
    "hard_size": 16,
    "max_arith_steps": 4,
    "qwen_batch_size": 8,
    "d_model": 128,
    "compiler_layers": 1,
    "preference_layers": 1,
    "heads": 4,
    "dropout": 0.1,
    "batch_size": 16,
    "group_batch_size": 4,
    "lr": 0.0005,
    "distill_lr": 0.00025,
    "preference_lr": 0.0003,
    "answer_weight": 0.2,
    "seed_epochs": 1,
    "distill_epochs": 1,
    "preference_epochs": 1,
    "full_supervised_epochs": 1,
    "preference_val_limit": 16,
    "search_topk": 2,
    "max_two_arg_pairs": 4,
    "max_candidates": 64,
    "hard_negative_count": 32,
    "preference_margin": 0.25,
    "base_margin": 0.5,
    "listwise_weight": 1.0,
    "pairwise_weight": 1.0,
    "base_contrast_weight": 0.5,
    "quality_loss_weight": 0.5,
    "valid_loss_weight": 0.1,
    "final_loss_weight": 0.1,
    "trace_top_loss_weight": 0.05,
    "trace_depth_loss_weight": 0.03
  },
  "results": [
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "seed_supervised",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3531669865642994,
      "mean_candidates": 65.125,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "seed_supervised",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.35797665369649806,
      "mean_candidates": 64.25,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "seed_supervised",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3524904214559387,
      "mean_candidates": 65.25,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "seed_supervised",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.35797665369649806,
      "mean_candidates": 64.25,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "seed_supervised",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.35384615384615387,
      "mean_candidates": 65.0,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "counterfactual_preference_selector",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": 0.0625,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.0625,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3531669865642994,
      "mean_candidates": 65.125,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "counterfactual_preference_selector",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": 0.0625,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.0625,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.35797665369649806,
      "mean_candidates": 64.25,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "counterfactual_preference_selector",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": 0.1875,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.1875,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3524904214559387,
      "mean_candidates": 65.25,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "counterfactual_preference_selector",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": 0.125,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.125,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.35797665369649806,
      "mean_candidates": 64.25,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "counterfactual_preference_selector",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": 0.375,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.375,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.35384615384615387,
      "mean_candidates": 65.0,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "answer_verified_distill",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.32798573975044565,
      "mean_candidates": 70.125,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "answer_verified_distill",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.323943661971831,
      "mean_candidates": 71.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "answer_verified_distill",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.33093525179856115,
      "mean_candidates": 69.5,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "answer_verified_distill",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3250883392226148,
      "mean_candidates": 70.75,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "answer_verified_distill",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.33153153153153153,
      "mean_candidates": 69.375,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "preference_selected_distill",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.875,
      "candidate_valid_rate": 0.2795138888888889,
      "mean_candidates": 72.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "preference_selected_distill",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0,
      "answer_search_accuracy": 0.0,
      "oracle_accuracy": 0.0,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.875,
      "candidate_valid_rate": 0.28097731239092494,
      "mean_candidates": 71.625,
      "found_rate": 0.0,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "preference_selected_distill",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.8125,
      "candidate_valid_rate": 0.26,
      "mean_candidates": 71.875,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "preference_selected_distill",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.6875,
      "candidate_valid_rate": 0.2227112676056338,
      "mean_candidates": 71.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "preference_selected_distill",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.6875,
      "candidate_valid_rate": 0.22389380530973452,
      "mean_candidates": 70.625,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "best_quality_distill",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3357664233576642,
      "mean_candidates": 68.5,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "best_quality_distill",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3339382940108893,
      "mean_candidates": 68.875,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "best_quality_distill",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3388581952117864,
      "mean_candidates": 67.875,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "best_quality_distill",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.33093525179856115,
      "mean_candidates": 69.5,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "best_quality_distill",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.34011090573012936,
      "mean_candidates": 67.625,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "full_supervised",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3194444444444444,
      "mean_candidates": 72.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "full_supervised",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3250883392226148,
      "mean_candidates": 70.75,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "full_supervised",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.32167832167832167,
      "mean_candidates": 71.5,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "full_supervised",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.32857142857142857,
      "mean_candidates": 70.0,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref_v2",
      "phase": "full_supervised",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3262411347517731,
      "mean_candidates": 70.5,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    }
  ]
}