{
  "experiment": "qwen35_4b_live_tool_dagger",
  "learned_rule": {
    "results": {
      "dev": {
        "accuracy": 0.5833333333333334,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 7,
        "n": 12,
        "policy": "learned_rule",
        "program_commits": 4,
        "program_correct": 4,
        "program_generations": 12,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 2,
        "n": 12,
        "policy": "learned_rule",
        "program_commits": 2,
        "program_correct": 2,
        "program_generations": 12,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "test"
      },
      "train": {
        "accuracy": 0.2222222222222222,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 8,
        "n": 36,
        "policy": "learned_rule",
        "program_commits": 6,
        "program_correct": 2,
        "program_generations": 36,
        "program_precision": 0.3333333333333333,
        "repair_actions": 0,
        "split": "train"
      }
    },
    "selected": {
      "dev": {
        "accuracy": 0.5833333333333334,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 7,
        "n": 12,
        "policy": "rule",
        "program_commits": 4,
        "program_correct": 4,
        "program_generations": 12,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "dev"
      },
      "false": "WRITE",
      "feature": "program_disagrees_direct",
      "true": "PROGRAM"
    },
    "top_candidates": [
      {
        "dev": {
          "accuracy": 0.5833333333333334,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 4,
          "exact": 7,
          "n": 12,
          "policy": "rule",
          "program_commits": 4,
          "program_correct": 4,
          "program_generations": 12,
          "program_precision": 1.0,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "program_disagrees_direct",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.5833333333333334,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 4,
          "exact": 7,
          "n": 12,
          "policy": "rule",
          "program_commits": 6,
          "program_correct": 5,
          "program_generations": 12,
          "program_precision": 0.8333333333333334,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "round_zero",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.5833333333333334,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 4,
          "exact": 7,
          "n": 12,
          "policy": "rule",
          "program_commits": 6,
          "program_correct": 5,
          "program_generations": 12,
          "program_precision": 0.8333333333333334,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "stage_round",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.5833333333333334,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 4,
          "exact": 7,
          "n": 12,
          "policy": "rule",
          "program_commits": 6,
          "program_correct": 5,
          "program_generations": 12,
          "program_precision": 0.8333333333333334,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "PROGRAM",
        "feature": "stage_start",
        "true": "WRITE"
      },
      {
        "dev": {
          "accuracy": 0.5833333333333334,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 4,
          "exact": 7,
          "n": 12,
          "policy": "rule",
          "program_commits": 6,
          "program_correct": 5,
          "program_generations": 12,
          "program_precision": 0.8333333333333334,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "visible_exec_ok",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.5833333333333334,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 4,
          "exact": 7,
          "n": 12,
          "policy": "rule",
          "program_commits": 6,
          "program_correct": 5,
          "program_generations": 12,
          "program_precision": 0.8333333333333334,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "visible_pass",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.3333333333333333,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 1,
          "exact": 4,
          "n": 12,
          "policy": "rule",
          "program_commits": 3,
          "program_correct": 2,
          "program_generations": 12,
          "program_precision": 0.6666666666666666,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "program_char_close_direct",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.3333333333333333,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 1,
          "exact": 4,
          "n": 12,
          "policy": "rule",
          "program_commits": 3,
          "program_correct": 2,
          "program_generations": 12,
          "program_precision": 0.6666666666666666,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "program_direct_same_shape",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.3333333333333333,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 1,
          "exact": 4,
          "n": 12,
          "policy": "rule",
          "program_commits": 3,
          "program_correct": 2,
          "program_generations": 12,
          "program_precision": 0.6666666666666666,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "program_hidden_shape_matches_direct",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 7,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "direct_char_close_visible",
        "true": "DIRECT"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 0,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "FIX",
        "feature": "direct_char_close_visible",
        "true": "DIRECT"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 0,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "PROGRAM",
        "feature": "direct_char_close_visible",
        "true": "DIRECT"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 5,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "DIRECT",
        "feature": "direct_char_close_visible",
        "true": "WRITE"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 5,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "FIX",
        "feature": "direct_char_close_visible",
        "true": "WRITE"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 5,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "PROGRAM",
        "feature": "direct_char_close_visible",
        "true": "WRITE"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 0,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "DIRECT",
        "feature": "direct_char_close_visible",
        "true": "FIX"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 7,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "direct_char_close_visible",
        "true": "FIX"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 0,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "PROGRAM",
        "feature": "direct_char_close_visible",
        "true": "FIX"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 0,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "DIRECT",
        "feature": "direct_char_close_visible",
        "true": "PROGRAM"
      },
      {
        "dev": {
          "accuracy": 0.25,
          "direct_correct_losses": 0,
          "direct_miss_recoveries": 0,
          "exact": 3,
          "n": 12,
          "policy": "rule",
          "program_commits": 0,
          "program_correct": 0,
          "program_generations": 7,
          "program_precision": null,
          "repair_actions": 0,
          "split": "dev"
        },
        "false": "WRITE",
        "feature": "direct_char_close_visible",
        "true": "PROGRAM"
      }
    ]
  },
  "lora": {
    "adapter_dir": "/workspace/experiments/qwen35_4b_live_tool_dagger/reports/adapters/lora_seq_policy",
    "losses": [
      4.552603721618652,
      4.3091535568237305,
      0.13554275035858154,
      4.1220855712890625,
      0.12548206746578217,
      0.5192352533340454,
      0.131893128156662,
      0.5669736266136169,
      0.08364348113536835,
      0.4361639618873596,
      0.5317410826683044,
      0.10381060093641281,
      0.16736292839050293,
      0.1739228367805481,
      0.03661981597542763,
      0.041602615267038345,
      0.013173554092645645,
      0.010240478441119194,
      0.0046274675987660885,
      0.19941329956054688,
      0.01275654323399067,
      0.09120283275842667,
      0.1857789158821106,
      0.41154390573501587,
      0.20518971979618073,
      0.03726516291499138,
      0.4256483018398285,
      0.02286791428923607,
      0.6268458366394043,
      0.17589432001113892,
      0.015690037980675697,
      0.06582826375961304,
      0.012067671865224838,
      0.07876944541931152,
      0.23160956799983978,
      0.3600897192955017,
      0.05817200988531113,
      0.06957320868968964,
      0.6606329679489136,
      0.1189231276512146,
      0.7554565072059631,
      0.0034829117357730865,
      0.028579797595739365,
      0.04386873543262482,
      0.010879961773753166,
      0.05726607143878937,
      0.43046337366104126,
      0.22795119881629944,
      0.2885868549346924,
      0.3226395547389984,
      0.07357446849346161,
      0.053542375564575195,
      0.06547041237354279,
      0.1294330358505249,
      0.02236418053507805,
      0.010622951202094555,
      0.4816814959049225,
      0.016333693638443947,
      0.0589015930891037,
      0.42135101556777954
    ],
    "name": "lora_seq_policy",
    "num_examples": 472,
    "num_tokenized": 472,
    "shuffle": false
  },
  "lora_shuffled": {
    "adapter_dir": "/workspace/experiments/qwen35_4b_live_tool_dagger/reports/adapters/lora_shuffled_seq",
    "losses": [
      5.98736047744751,
      3.61386775970459,
      0.40403375029563904,
      1.6176822185516357,
      0.33792543411254883,
      0.612591028213501,
      0.2562498450279236,
      0.6004236936569214,
      0.7027271389961243,
      0.24262264370918274,
      0.23796787858009338,
      0.4898225963115692,
      0.5542962551116943,
      0.4273870587348938,
      0.5374972820281982,
      0.4340919852256775,
      1.005829095840454,
      0.41283947229385376,
      0.7791346311569214,
      0.4211442470550537,
      0.3397350311279297,
      1.107345461845398,
      0.38835904002189636,
      0.5309644341468811,
      0.6325112581253052,
      0.8960577249526978,
      0.5121994018554688,
      0.5254725813865662,
      0.9888128638267517,
      0.2754490375518799,
      0.953143298625946,
      0.29794931411743164,
      0.9600629210472107,
      0.4448034167289734,
      0.6994848251342773,
      0.31755179166793823,
      0.3572532534599304,
      0.22302262485027313,
      0.4121244251728058,
      0.9374489188194275,
      0.16532158851623535,
      0.5247519016265869,
      0.7846110463142395,
      0.1692488044500351,
      0.20932486653327942,
      0.6659720540046692,
      0.29607194662094116,
      0.3075041174888611,
      0.5040891766548157,
      0.49308153986930847,
      0.7955588102340698,
      0.30777180194854736,
      0.3151448667049408,
      0.4014561176300049,
      0.29424336552619934,
      0.36411264538764954,
      0.16922667622566223,
      0.21472857892513275,
      0.13311588764190674,
      0.37640491127967834
    ],
    "name": "lora_shuffled_seq",
    "num_examples": 472,
    "num_tokenized": 472,
    "shuffle": true
  },
  "n_records": 60,
  "policy_results": {
    "always_tool_visible": {
      "dev": {
        "accuracy": 0.5833333333333334,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 7,
        "n": 12,
        "policy": "always_tool_visible",
        "program_commits": 6,
        "program_correct": 5,
        "program_generations": 12,
        "program_precision": 0.8333333333333334,
        "repair_actions": 12,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 2,
        "n": 12,
        "policy": "always_tool_visible",
        "program_commits": 3,
        "program_correct": 2,
        "program_generations": 12,
        "program_precision": 0.6666666666666666,
        "repair_actions": 18,
        "split": "test"
      },
      "train": {
        "accuracy": 0.2777777777777778,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 10,
        "n": 36,
        "policy": "always_tool_visible",
        "program_commits": 9,
        "program_correct": 5,
        "program_generations": 36,
        "program_precision": 0.5555555555555556,
        "repair_actions": 56,
        "split": "train"
      }
    },
    "direct_only": {
      "dev": {
        "accuracy": 0.25,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 0,
        "exact": 3,
        "n": 12,
        "policy": "direct_only",
        "program_commits": 0,
        "program_correct": 0,
        "program_generations": 0,
        "program_precision": null,
        "repair_actions": 0,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.0,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 0,
        "exact": 0,
        "n": 12,
        "policy": "direct_only",
        "program_commits": 0,
        "program_correct": 0,
        "program_generations": 0,
        "program_precision": null,
        "repair_actions": 0,
        "split": "test"
      },
      "train": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 0,
        "exact": 6,
        "n": 36,
        "policy": "direct_only",
        "program_commits": 0,
        "program_correct": 0,
        "program_generations": 0,
        "program_precision": null,
        "repair_actions": 0,
        "split": "train"
      }
    },
    "learned_rule": {
      "dev": {
        "accuracy": 0.5833333333333334,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 7,
        "n": 12,
        "policy": "learned_rule",
        "program_commits": 4,
        "program_correct": 4,
        "program_generations": 12,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 2,
        "n": 12,
        "policy": "learned_rule",
        "program_commits": 2,
        "program_correct": 2,
        "program_generations": 12,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "test"
      },
      "train": {
        "accuracy": 0.2222222222222222,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 8,
        "n": 36,
        "policy": "learned_rule",
        "program_commits": 6,
        "program_correct": 2,
        "program_generations": 36,
        "program_precision": 0.3333333333333333,
        "repair_actions": 0,
        "split": "train"
      }
    },
    "lora_seq_policy": {
      "dev": {
        "accuracy": 0.3333333333333333,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 1,
        "exact": 4,
        "n": 12,
        "policy": "lora_seq_policy",
        "program_commits": 1,
        "program_correct": 1,
        "program_generations": 4,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 2,
        "n": 12,
        "policy": "lora_seq_policy",
        "program_commits": 2,
        "program_correct": 2,
        "program_generations": 6,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "test"
      },
      "train": {
        "accuracy": 0.25,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 3,
        "exact": 9,
        "n": 36,
        "policy": "lora_seq_policy",
        "program_commits": 3,
        "program_correct": 3,
        "program_generations": 13,
        "program_precision": 1.0,
        "repair_actions": 3,
        "split": "train"
      }
    },
    "lora_shuffled_seq": {
      "dev": {
        "accuracy": 0.25,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 0,
        "exact": 3,
        "n": 12,
        "policy": "lora_shuffled_seq",
        "program_commits": 0,
        "program_correct": 0,
        "program_generations": 0,
        "program_precision": null,
        "repair_actions": 0,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.0,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 0,
        "exact": 0,
        "n": 12,
        "policy": "lora_shuffled_seq",
        "program_commits": 0,
        "program_correct": 0,
        "program_generations": 0,
        "program_precision": null,
        "repair_actions": 0,
        "split": "test"
      },
      "train": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 0,
        "exact": 6,
        "n": 36,
        "policy": "lora_shuffled_seq",
        "program_commits": 0,
        "program_correct": 0,
        "program_generations": 0,
        "program_precision": null,
        "repair_actions": 0,
        "split": "train"
      }
    },
    "oracle_seq": {
      "dev": {
        "accuracy": 0.5833333333333334,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 7,
        "n": 12,
        "policy": "oracle_seq",
        "program_commits": 4,
        "program_correct": 4,
        "program_generations": 4,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 2,
        "n": 12,
        "policy": "oracle_seq",
        "program_commits": 2,
        "program_correct": 2,
        "program_generations": 2,
        "program_precision": 1.0,
        "repair_actions": 0,
        "split": "test"
      },
      "train": {
        "accuracy": 0.2777777777777778,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 10,
        "n": 36,
        "policy": "oracle_seq",
        "program_commits": 4,
        "program_correct": 4,
        "program_generations": 4,
        "program_precision": 1.0,
        "repair_actions": 2,
        "split": "train"
      }
    },
    "visible_disagree_rule": {
      "dev": {
        "accuracy": 0.5833333333333334,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 7,
        "n": 12,
        "policy": "visible_disagree_rule",
        "program_commits": 4,
        "program_correct": 4,
        "program_generations": 12,
        "program_precision": 1.0,
        "repair_actions": 12,
        "split": "dev"
      },
      "test": {
        "accuracy": 0.16666666666666666,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 2,
        "exact": 2,
        "n": 12,
        "policy": "visible_disagree_rule",
        "program_commits": 2,
        "program_correct": 2,
        "program_generations": 12,
        "program_precision": 1.0,
        "repair_actions": 18,
        "split": "test"
      },
      "train": {
        "accuracy": 0.2777777777777778,
        "direct_correct_losses": 0,
        "direct_miss_recoveries": 4,
        "exact": 10,
        "n": 36,
        "policy": "visible_disagree_rule",
        "program_commits": 8,
        "program_correct": 4,
        "program_generations": 36,
        "program_precision": 0.5,
        "repair_actions": 56,
        "split": "train"
      }
    }
  },
  "splits": {
    "dev": {
      "direct_exact": 3,
      "families": [
        "potters_wheel_merge_split",
        "proactive_wrangling_fold",
        "synthetic_22",
        "synthetic_51"
      ],
      "n": 12,
      "oracle_union": 7,
      "program_exact": 5
    },
    "test": {
      "direct_exact": 0,
      "families": [
        "potters_wheel_divide",
        "reshape_table_structure_data_wrangler",
        "synthetic_13",
        "synthetic_45"
      ],
      "n": 12,
      "oracle_union": 2,
      "program_exact": 2
    },
    "train": {
      "direct_exact": 6,
      "families": [
        "agriculture",
        "craigslist_data_wrangler",
        "crime_data_wrangler",
        "potters_wheel_fold",
        "potters_wheel_fold_2",
        "potters_wheel_split_fold",
        "potters_wheel_unfold",
        "potters_wheel_unfold2",
        "proactive_wrangling_complex",
        "synthetic_11",
        "synthetic_5",
        "synthetic_8"
      ],
      "n": 36,
      "oracle_union": 10,
      "program_exact": 5
    }
  },
  "trace_file": "/workspace/experiments/qwen35_4b_live_tool_dagger/reports/fresh_traces_split2.jsonl"
}