{
  "run": "pilot_vm_echo_s96",
  "args": {
    "run_name": "pilot_vm_echo_s96",
    "model_name": "Qwen/Qwen3-4B",
    "seed": 41,
    "device": "",
    "seed_train_size": 96,
    "unlabeled_train_size": 256,
    "full_supervised_size": 256,
    "val_size": 64,
    "fresh_size": 64,
    "hard_size": 64,
    "max_arith_steps": 4,
    "qwen_batch_size": 16,
    "d_model": 256,
    "layers": 2,
    "heads": 4,
    "dropout": 0.1,
    "batch_size": 32,
    "lr": 0.0005,
    "expert_lr": 0.00025,
    "answer_weight": 0.2,
    "arms": [
      "baseline",
      "vm_echo"
    ],
    "echo_weight": 0.35,
    "echo_trace_weight": 1.0,
    "echo_depth_weight": 0.4,
    "echo_final_weight": 0.5,
    "echo_valid_weight": 0.2,
    "seed_epochs": 6,
    "expert_rounds": 1,
    "expert_epochs": 4,
    "full_supervised_epochs": 8,
    "search_topk": 3,
    "max_two_arg_pairs": 6
  },
  "results": [
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_seed_supervised",
      "split": "val_mixed",
      "n": 64,
      "direct_accuracy": 0.046875,
      "search_accuracy": 0.421875,
      "oracle_accuracy": 0.421875,
      "program_exact": 0.0,
      "valid_rate": 0.5327380952380952,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.421875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.008797653958944282,
      "echo_trace_depth_accuracy": 0.008797653958944282
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_seed_supervised",
      "split": "fresh_standard",
      "n": 64,
      "direct_accuracy": 0.125,
      "search_accuracy": 0.46875,
      "oracle_accuracy": 0.46875,
      "program_exact": 0.0,
      "valid_rate": 0.5334821428571429,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.46875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.015625,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.02608695652173913,
      "echo_trace_depth_accuracy": 0.0
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_seed_supervised",
      "split": "fresh_paraphrase",
      "n": 64,
      "direct_accuracy": 0.0625,
      "search_accuracy": 0.4375,
      "oracle_accuracy": 0.4375,
      "program_exact": 0.0,
      "valid_rate": 0.5359623015873016,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.4375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.015625,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.025210084033613446,
      "echo_trace_depth_accuracy": 0.01680672268907563
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_seed_supervised",
      "split": "fresh_paired",
      "n": 64,
      "direct_accuracy": 0.078125,
      "search_accuracy": 0.390625,
      "oracle_accuracy": 0.390625,
      "program_exact": 0.0,
      "valid_rate": 0.5364583333333334,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.390625,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.005291005291005291,
      "echo_trace_depth_accuracy": 0.007936507936507936
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_seed_supervised",
      "split": "hard_composition",
      "n": 64,
      "direct_accuracy": 0.09375,
      "search_accuracy": 0.359375,
      "oracle_accuracy": 0.359375,
      "program_exact": 0.0,
      "valid_rate": 0.535218253968254,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.359375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.021686746987951807,
      "echo_trace_depth_accuracy": 0.007228915662650603
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_expert_round_1",
      "split": "val_mixed",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.46875,
      "oracle_accuracy": 0.46875,
      "program_exact": 0.015625,
      "valid_rate": 0.5443948412698413,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.46875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.011730205278592375,
      "echo_trace_depth_accuracy": 0.0
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_expert_round_1",
      "split": "fresh_standard",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.4375,
      "oracle_accuracy": 0.4375,
      "program_exact": 0.0,
      "valid_rate": 0.5520833333333334,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.4375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.014492753623188406,
      "echo_trace_depth_accuracy": 0.0
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_expert_round_1",
      "split": "fresh_paraphrase",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.59375,
      "oracle_accuracy": 0.59375,
      "program_exact": 0.0,
      "valid_rate": 0.5486111111111112,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.59375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.0196078431372549,
      "echo_trace_depth_accuracy": 0.0
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_expert_round_1",
      "split": "fresh_paired",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.328125,
      "oracle_accuracy": 0.328125,
      "program_exact": 0.0,
      "valid_rate": 0.5602678571428571,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.328125,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.015873015873015872,
      "echo_trace_depth_accuracy": 0.0
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_expert_round_1",
      "split": "hard_composition",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.46875,
      "oracle_accuracy": 0.46875,
      "program_exact": 0.015625,
      "valid_rate": 0.5510912698412699,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.46875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.016867469879518072,
      "echo_trace_depth_accuracy": 0.0
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_full_supervised",
      "split": "val_mixed",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.4375,
      "oracle_accuracy": 0.4375,
      "program_exact": 0.0,
      "valid_rate": 0.5287698412698413,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.4375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.03125,
      "echo_trace_top_accuracy": 0.017595307917888565,
      "echo_trace_depth_accuracy": 0.07917888563049853
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_full_supervised",
      "split": "fresh_standard",
      "n": 64,
      "direct_accuracy": 0.140625,
      "search_accuracy": 0.421875,
      "oracle_accuracy": 0.421875,
      "program_exact": 0.015625,
      "valid_rate": 0.5300099206349206,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.421875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.0,
      "echo_trace_top_accuracy": 0.014492753623188406,
      "echo_trace_depth_accuracy": 0.07246376811594203
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_full_supervised",
      "split": "fresh_paraphrase",
      "n": 64,
      "direct_accuracy": 0.140625,
      "search_accuracy": 0.421875,
      "oracle_accuracy": 0.421875,
      "program_exact": 0.0,
      "valid_rate": 0.533234126984127,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.421875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.0,
      "echo_valid_accuracy": 0.125,
      "echo_trace_top_accuracy": 0.01680672268907563,
      "echo_trace_depth_accuracy": 0.06442577030812324
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_full_supervised",
      "split": "fresh_paired",
      "n": 64,
      "direct_accuracy": 0.09375,
      "search_accuracy": 0.421875,
      "oracle_accuracy": 0.421875,
      "program_exact": 0.0,
      "valid_rate": 0.5391865079365079,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.421875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.03125,
      "echo_valid_accuracy": 0.0625,
      "echo_trace_top_accuracy": 0.0026455026455026454,
      "echo_trace_depth_accuracy": 0.06349206349206349
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "baseline",
      "phase": "baseline_full_supervised",
      "split": "hard_composition",
      "n": 64,
      "direct_accuracy": 0.0625,
      "search_accuracy": 0.484375,
      "oracle_accuracy": 0.484375,
      "program_exact": 0.0,
      "valid_rate": 0.5370691792710142,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.046875,
      "found_rate": 0.484375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.03125,
      "echo_valid_accuracy": 0.046875,
      "echo_trace_top_accuracy": 0.0024096385542168677,
      "echo_trace_depth_accuracy": 0.10843373493975904
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_seed_supervised",
      "split": "val_mixed",
      "n": 64,
      "direct_accuracy": 0.0,
      "search_accuracy": 0.21875,
      "oracle_accuracy": 0.21875,
      "program_exact": 0.0,
      "valid_rate": 0.5079365079365079,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.21875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.09375,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.10850439882697947,
      "echo_trace_depth_accuracy": 0.8093841642228738
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_seed_supervised",
      "split": "fresh_standard",
      "n": 64,
      "direct_accuracy": 0.0,
      "search_accuracy": 0.109375,
      "oracle_accuracy": 0.109375,
      "program_exact": 0.0,
      "valid_rate": 0.5079365079365079,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.109375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.171875,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.10144927536231885,
      "echo_trace_depth_accuracy": 0.7971014492753623
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_seed_supervised",
      "split": "fresh_paraphrase",
      "n": 64,
      "direct_accuracy": 0.015625,
      "search_accuracy": 0.296875,
      "oracle_accuracy": 0.296875,
      "program_exact": 0.0,
      "valid_rate": 0.5079365079365079,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.296875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.109375,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.10644257703081232,
      "echo_trace_depth_accuracy": 0.8207282913165266
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_seed_supervised",
      "split": "fresh_paired",
      "n": 64,
      "direct_accuracy": 0.0,
      "search_accuracy": 0.25,
      "oracle_accuracy": 0.25,
      "program_exact": 0.0,
      "valid_rate": 0.5081845238095238,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.25,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.09375,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.10846560846560846,
      "echo_trace_depth_accuracy": 0.8148148148148148
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_seed_supervised",
      "split": "hard_composition",
      "n": 64,
      "direct_accuracy": 0.015625,
      "search_accuracy": 0.25,
      "oracle_accuracy": 0.25,
      "program_exact": 0.0,
      "valid_rate": 0.5081845238095238,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.25,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.109375,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.07951807228915662,
      "echo_trace_depth_accuracy": 0.7614457831325301
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_expert_round_1",
      "split": "val_mixed",
      "n": 64,
      "direct_accuracy": 0.09375,
      "search_accuracy": 0.28125,
      "oracle_accuracy": 0.28125,
      "program_exact": 0.0,
      "valid_rate": 0.5228174603174603,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.28125,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.125,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.06744868035190615,
      "echo_trace_depth_accuracy": 0.9120234604105572
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_expert_round_1",
      "split": "fresh_standard",
      "n": 64,
      "direct_accuracy": 0.03125,
      "search_accuracy": 0.34375,
      "oracle_accuracy": 0.34375,
      "program_exact": 0.0,
      "valid_rate": 0.5317460317460317,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.34375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.09375,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.06086956521739131,
      "echo_trace_depth_accuracy": 0.9536231884057971
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_expert_round_1",
      "split": "fresh_paraphrase",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.421875,
      "oracle_accuracy": 0.421875,
      "program_exact": 0.0,
      "valid_rate": 0.5235615079365079,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.421875,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.203125,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.09803921568627451,
      "echo_trace_depth_accuracy": 0.9103641456582633
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_expert_round_1",
      "split": "fresh_paired",
      "n": 64,
      "direct_accuracy": 0.078125,
      "search_accuracy": 0.359375,
      "oracle_accuracy": 0.359375,
      "program_exact": 0.0,
      "valid_rate": 0.5378130424001983,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.046875,
      "found_rate": 0.359375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.125,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.06878306878306878,
      "echo_trace_depth_accuracy": 0.9126984126984127
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_expert_round_1",
      "split": "hard_composition",
      "n": 64,
      "direct_accuracy": 0.03125,
      "search_accuracy": 0.328125,
      "oracle_accuracy": 0.328125,
      "program_exact": 0.0,
      "valid_rate": 0.5295138888888888,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.328125,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.09375,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.03614457831325301,
      "echo_trace_depth_accuracy": 0.8843373493975903
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_full_supervised",
      "split": "val_mixed",
      "n": 64,
      "direct_accuracy": 0.171875,
      "search_accuracy": 0.4375,
      "oracle_accuracy": 0.4375,
      "program_exact": 0.015625,
      "valid_rate": 0.5344742063492064,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.4375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.125,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.10557184750733138,
      "echo_trace_depth_accuracy": 0.9970674486803519
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_full_supervised",
      "split": "fresh_standard",
      "n": 64,
      "direct_accuracy": 0.203125,
      "search_accuracy": 0.453125,
      "oracle_accuracy": 0.453125,
      "program_exact": 0.03125,
      "valid_rate": 0.5357142857142857,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.453125,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.15625,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.07246376811594203,
      "echo_trace_depth_accuracy": 0.9942028985507246
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_full_supervised",
      "split": "fresh_paraphrase",
      "n": 64,
      "direct_accuracy": 0.171875,
      "search_accuracy": 0.453125,
      "oracle_accuracy": 0.453125,
      "program_exact": 0.03125,
      "valid_rate": 0.5394345238095238,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.453125,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.171875,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.14565826330532214,
      "echo_trace_depth_accuracy": 1.0
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_full_supervised",
      "split": "fresh_paired",
      "n": 64,
      "direct_accuracy": 0.109375,
      "search_accuracy": 0.3125,
      "oracle_accuracy": 0.3125,
      "program_exact": 0.0,
      "valid_rate": 0.5434027777777778,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.3125,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.09375,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.07671957671957672,
      "echo_trace_depth_accuracy": 0.9947089947089947
    },
    {
      "run": "pilot_vm_echo_s96",
      "arm": "vm_echo",
      "phase": "vm_echo_full_supervised",
      "split": "hard_composition",
      "n": 64,
      "direct_accuracy": 0.09375,
      "search_accuracy": 0.4375,
      "oracle_accuracy": 0.4375,
      "program_exact": 0.0,
      "valid_rate": 0.5431547619047619,
      "direct_valid_rate": 1.0,
      "mean_candidates": 189.0,
      "found_rate": 0.4375,
      "gap_recovered": 1.0,
      "echo_final_accuracy": 0.125,
      "echo_valid_accuracy": 1.0,
      "echo_trace_top_accuracy": 0.07228915662650602,
      "echo_trace_depth_accuracy": 0.9493975903614458
    }
  ]
}