{
  "run": "smoke_counterfactual_pref",
  "args": {
    "run_name": "smoke_counterfactual_pref",
    "model_name": "Qwen/Qwen3-4B",
    "seed": 131,
    "device": "",
    "seed_train_size": 16,
    "unlabeled_train_size": 32,
    "full_supervised_size": 32,
    "val_size": 16,
    "fresh_size": 16,
    "hard_size": 16,
    "max_arith_steps": 4,
    "qwen_batch_size": 8,
    "d_model": 128,
    "compiler_layers": 1,
    "preference_layers": 1,
    "heads": 4,
    "dropout": 0.1,
    "batch_size": 16,
    "group_batch_size": 4,
    "lr": 0.0005,
    "distill_lr": 0.00025,
    "preference_lr": 0.0003,
    "answer_weight": 0.2,
    "seed_epochs": 2,
    "distill_epochs": 1,
    "preference_epochs": 2,
    "full_supervised_epochs": 1,
    "preference_val_limit": 16,
    "search_topk": 2,
    "max_two_arg_pairs": 4,
    "max_candidates": 64,
    "hard_negative_count": 32,
    "preference_margin": 0.25,
    "base_margin": 0.5,
    "listwise_weight": 1.0,
    "pairwise_weight": 1.0,
    "base_contrast_weight": 0.5,
    "quality_loss_weight": 0.5,
    "valid_loss_weight": 0.1,
    "final_loss_weight": 0.1,
    "trace_top_loss_weight": 0.05,
    "trace_depth_loss_weight": 0.03
  },
  "results": [
    {
      "run": "smoke_counterfactual_pref",
      "phase": "seed_supervised",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3382352941176471,
      "mean_candidates": 68.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "seed_supervised",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.34328358208955223,
      "mean_candidates": 67.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "seed_supervised",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.33948339483394835,
      "mean_candidates": 67.75,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "seed_supervised",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.34328358208955223,
      "mean_candidates": 67.0,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "seed_supervised",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3445692883895131,
      "mean_candidates": 66.75,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "counterfactual_preference_selector",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": 0.0625,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.0625,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3382352941176471,
      "mean_candidates": 68.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "counterfactual_preference_selector",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": 0.0625,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.0625,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.34328358208955223,
      "mean_candidates": 67.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "counterfactual_preference_selector",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": 0.1875,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.1875,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.33948339483394835,
      "mean_candidates": 67.75,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "counterfactual_preference_selector",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": 0.125,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.125,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.34328358208955223,
      "mean_candidates": 67.0,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "counterfactual_preference_selector",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": 0.375,
      "preference_rerank_valid_rate": 1.0,
      "preference_rerank_program_exact": 0.0,
      "preference_rerank_quality": 1.375,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3445692883895131,
      "mean_candidates": 66.75,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "answer_verified_distill",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.33153153153153153,
      "mean_candidates": 69.375,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "answer_verified_distill",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.32798573975044565,
      "mean_candidates": 70.125,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "answer_verified_distill",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.33273056057866185,
      "mean_candidates": 69.125,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "answer_verified_distill",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3303411131059246,
      "mean_candidates": 69.625,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "answer_verified_distill",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3333333333333333,
      "mean_candidates": 69.0,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "preference_selected_distill",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3177892918825561,
      "mean_candidates": 72.375,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "preference_selected_distill",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3161512027491409,
      "mean_candidates": 72.75,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "preference_selected_distill",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.31669535283993117,
      "mean_candidates": 72.625,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "preference_selected_distill",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3161512027491409,
      "mean_candidates": 72.75,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "preference_selected_distill",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.31833910034602075,
      "mean_candidates": 72.25,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "best_quality_distill",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0,
      "answer_search_accuracy": 0.0,
      "oracle_accuracy": 0.0,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.0,
      "candidate_valid_rate": 0.0,
      "mean_candidates": 57.0,
      "found_rate": 0.0,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "best_quality_distill",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0,
      "answer_search_accuracy": 0.0,
      "oracle_accuracy": 0.0,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.0,
      "candidate_valid_rate": 0.0,
      "mean_candidates": 57.0,
      "found_rate": 0.0,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "best_quality_distill",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.0,
      "answer_search_accuracy": 0.0,
      "oracle_accuracy": 0.0,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.0,
      "candidate_valid_rate": 0.0,
      "mean_candidates": 57.0,
      "found_rate": 0.0,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "best_quality_distill",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.0,
      "answer_search_accuracy": 0.0,
      "oracle_accuracy": 0.0,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.0,
      "candidate_valid_rate": 0.0,
      "mean_candidates": 57.0,
      "found_rate": 0.0,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "best_quality_distill",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.0,
      "answer_search_accuracy": 0.0,
      "oracle_accuracy": 0.0,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 0.0,
      "candidate_valid_rate": 0.0,
      "mean_candidates": 57.0,
      "found_rate": 0.0,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "full_supervised",
      "split": "val_mixed",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.31724137931034485,
      "mean_candidates": 72.5,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "full_supervised",
      "split": "fresh_standard",
      "n": 16,
      "direct_accuracy": 0.0625,
      "answer_search_accuracy": 0.0625,
      "oracle_accuracy": 0.0625,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3150684931506849,
      "mean_candidates": 73.0,
      "found_rate": 0.0625,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "full_supervised",
      "split": "fresh_paraphrase",
      "n": 16,
      "direct_accuracy": 0.1875,
      "answer_search_accuracy": 0.1875,
      "oracle_accuracy": 0.1875,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3150684931506849,
      "mean_candidates": 73.0,
      "found_rate": 0.1875,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "full_supervised",
      "split": "fresh_paired",
      "n": 16,
      "direct_accuracy": 0.125,
      "answer_search_accuracy": 0.125,
      "oracle_accuracy": 0.125,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3150684931506849,
      "mean_candidates": 73.0,
      "found_rate": 0.125,
      "program_exact": 0.0,
      "gap_recovered": null
    },
    {
      "run": "smoke_counterfactual_pref",
      "phase": "full_supervised",
      "split": "hard_composition",
      "n": 16,
      "direct_accuracy": 0.375,
      "answer_search_accuracy": 0.375,
      "oracle_accuracy": 0.375,
      "preference_rerank_accuracy": null,
      "preference_rerank_valid_rate": null,
      "preference_rerank_program_exact": null,
      "preference_rerank_quality": null,
      "direct_valid_rate": 1.0,
      "candidate_valid_rate": 0.3156089193825043,
      "mean_candidates": 72.875,
      "found_rate": 0.375,
      "program_exact": 0.0,
      "gap_recovered": null
    }
  ]
}