{ "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "args": { "run_name": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "model_id": "Qwen/Qwen3-4B", "seed": 123, "modulus": 97, "stage_max_steps": "24,24,24", "stage_min_lengths": "1,1,8", "stage_train_max_lengths": "8,16,24", "stage_steps": "300,150,300", "train_examples": 512, "train_template_mode": "mixed", "eval_lengths": "8,16,24", "eval_examples": 64, "paired_eval_pairs": 32, "batch_size": 8, "grad_accum": 1, "eval_batch_size": 8, "max_length": 2048, "head_width": 512, "compiler_layers": 1, "compiler_heads": 4, "compiler_dropout": 0.05, "expansion_noise": 0.005, "expansion_mode": "copy_last", "trace_loss_weight": 1.0, "executor_loss_weight": 1.0, "state_loss_weight": 1.0, "init_trace_loss_weight": 4.0, "op_trace_loss_weight": 1.0, "arg_trace_loss_weight": 4.0, "direct_head_weight": 0.0, "lr": 0.0002, "weight_decay": 0.0, "optimizer": "paged_adamw_8bit", "max_grad_norm": 1.0, "torch_dtype": "bf16", "load_in_4bit": true, "device_map": "auto", "use_lora": true, "gradient_checkpointing": true, "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "lora_target_modules": "all-linear", "log_interval": 50, "save_checkpoints": true, "checkpoint_dir": "", "selection_split": "paired_L24", "selection_metric": "executor_pair_both_correct" }, "metadata": { "python": "3.12.3", "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "torch": "2.8.0+cu128", "cuda_available": true, "transformers_loader": "AutoModelForCausalLM", "peft_installed": true, "use_lora": true, "lora_r": 8, "model_id": "Qwen/Qwen3-4B", "gpu_name": "NVIDIA RTX 6000 Ada Generation", "gpu_vram_gb": 47.383 }, "stages": [ { "name": "stage1_max24", "max_steps": 24, "train_min_len": 1, "train_max_len": 8, "steps": 300 }, { "name": "stage2_max24", "max_steps": 24, "train_min_len": 1, "train_max_len": 16, "steps": 150 }, { "name": "stage3_max24", "max_steps": 24, "train_min_len": 8, "train_max_len": 24, "steps": 300 } ], "train_log": [ { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage1_max24", "stage_idx": 1, "local_step": 1, "global_step": 1, "compiler_max_steps": 24, "elapsed_sec": 5.153, "expansion_event": "init", "init_loss": 4.647781848907471, "op_loss": 1.1190651655197144, "arg_loss": 4.537424087524414, "state_loss": 4.579545497894287, "state_train_accuracy": 0.0, "executor_loss": 4.579961776733398, "loss": 47.01939392089844 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage1_max24", "stage_idx": 1, "local_step": 50, "global_step": 50, "compiler_max_steps": 24, "elapsed_sec": 138.901, "expansion_event": "init", "init_loss": 3.8246278762817383, "op_loss": 1.122908353805542, "arg_loss": 3.4875152111053467, "state_loss": 4.566431522369385, "state_train_accuracy": 0.0, "executor_loss": 4.5567097663879395, "loss": 39.49462127685547 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage1_max24", "stage_idx": 1, "local_step": 100, "global_step": 100, "compiler_max_steps": 24, "elapsed_sec": 277.436, "expansion_event": "init", "init_loss": 1.2005528211593628, "op_loss": 0.05156470090150833, "arg_loss": 1.254866600036621, "state_loss": 3.7301313877105713, "state_train_accuracy": 0.3243243396282196, "executor_loss": 3.4903926849365234, "loss": 17.093767166137695 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage1_max24", "stage_idx": 1, "local_step": 150, "global_step": 150, "compiler_max_steps": 24, "elapsed_sec": 415.749, "expansion_event": "init", "init_loss": 0.14358550310134888, "op_loss": 0.0012570362305268645, "arg_loss": 0.02752610854804516, "state_loss": 0.23781833052635193, "state_train_accuracy": 1.0, "executor_loss": 0.2761993408203125, "loss": 1.1997212171554565 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage1_max24", "stage_idx": 1, "local_step": 200, "global_step": 200, "compiler_max_steps": 24, "elapsed_sec": 554.487, "expansion_event": "init", "init_loss": 0.06243208050727844, "op_loss": 2.7690964998328127e-05, "arg_loss": 0.0022468753159046173, "state_loss": 0.05400078371167183, "state_train_accuracy": 0.9999999403953552, "executor_loss": 0.07408291101455688, "loss": 0.3868272304534912 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage1_max24", "stage_idx": 1, "local_step": 250, "global_step": 250, "compiler_max_steps": 24, "elapsed_sec": 693.214, "expansion_event": "init", "init_loss": 0.007394655141979456, "op_loss": 5.212682026467519e-06, "arg_loss": 0.0003286467108409852, "state_loss": 0.013247201219201088, "state_train_accuracy": 1.0, "executor_loss": 0.008312193676829338, "loss": 0.052457816898822784 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage1_max24", "stage_idx": 1, "local_step": 300, "global_step": 300, "compiler_max_steps": 24, "elapsed_sec": 831.622, "expansion_event": "init", "init_loss": 0.0010708232875913382, "op_loss": 2.1168652892811224e-06, "arg_loss": 0.00012856854300480336, "state_loss": 0.0015114182606339455, "state_train_accuracy": 1.0, "executor_loss": 0.001609724247828126, "loss": 0.0079208267852664 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage2_max24", "stage_idx": 2, "local_step": 1, "global_step": 301, "compiler_max_steps": 24, "elapsed_sec": 965.062, "expansion_event": "same", "init_loss": 0.0024942816235125065, "op_loss": 5.638898073812015e-06, "arg_loss": 0.00012212821457069367, "state_loss": 0.002114485949277878, "state_train_accuracy": 1.0, "executor_loss": 0.003548369277268648, "loss": 0.01613413356244564 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage2_max24", "stage_idx": 2, "local_step": 50, "global_step": 350, "compiler_max_steps": 24, "elapsed_sec": 1115.341, "expansion_event": "same", "init_loss": 0.0006049536750651896, "op_loss": 4.6490913518937305e-06, "arg_loss": 9.929277439368889e-05, "state_loss": 0.0009920328157022595, "state_train_accuracy": 1.0, "executor_loss": 0.0011116618989035487, "loss": 0.004925329703837633 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage2_max24", "stage_idx": 2, "local_step": 100, "global_step": 400, "compiler_max_steps": 24, "elapsed_sec": 1268.824, "expansion_event": "same", "init_loss": 0.00022104554227553308, "op_loss": 1.035629111356684e-06, "arg_loss": 3.116693551419303e-05, "state_loss": 0.0003692987665999681, "state_train_accuracy": 1.0, "executor_loss": 0.00047871991409920156, "loss": 0.001857904251664877 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage2_max24", "stage_idx": 2, "local_step": 150, "global_step": 450, "compiler_max_steps": 24, "elapsed_sec": 1423.261, "expansion_event": "same", "init_loss": 7.147801079554483e-05, "op_loss": 5.195650828682119e-06, "arg_loss": 2.751454485405702e-05, "state_loss": 0.00021545460913330317, "state_train_accuracy": 1.0, "executor_loss": 0.0002797225315589458, "loss": 0.0008963430300354958 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage3_max24", "stage_idx": 3, "local_step": 1, "global_step": 451, "compiler_max_steps": 24, "elapsed_sec": 1556.876, "expansion_event": "same", "init_loss": 0.0001399355533067137, "op_loss": 0.27560773491859436, "arg_loss": 0.4337005019187927, "state_loss": 0.8530460000038147, "state_train_accuracy": 0.8960000276565552, "executor_loss": 3.3817319869995117, "loss": 6.245747089385986 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage3_max24", "stage_idx": 3, "local_step": 50, "global_step": 500, "compiler_max_steps": 24, "elapsed_sec": 1726.886, "expansion_event": "same", "init_loss": 0.00020995869999751449, "op_loss": 9.32340117287822e-06, "arg_loss": 0.021592486649751663, "state_loss": 0.09150592237710953, "state_train_accuracy": 0.9674796462059021, "executor_loss": 0.3321746289730072, "loss": 0.5108996629714966 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage3_max24", "stage_idx": 3, "local_step": 100, "global_step": 550, "compiler_max_steps": 24, "elapsed_sec": 1900.76, "expansion_event": "same", "init_loss": 7.995565829332918e-05, "op_loss": 1.0273369298374746e-05, "arg_loss": 4.995226481696591e-05, "state_loss": 0.0005501610576175153, "state_train_accuracy": 0.9999999403953552, "executor_loss": 0.000998541247099638, "loss": 0.0020786074455827475 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage3_max24", "stage_idx": 3, "local_step": 150, "global_step": 600, "compiler_max_steps": 24, "elapsed_sec": 2074.171, "expansion_event": "same", "init_loss": 5.505765147972852e-05, "op_loss": 3.1411573218065314e-06, "arg_loss": 2.7410780603531748e-05, "state_loss": 0.00034634987241588533, "state_train_accuracy": 1.0, "executor_loss": 0.0005874945200048387, "loss": 0.0012668592389672995 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage3_max24", "stage_idx": 3, "local_step": 200, "global_step": 650, "compiler_max_steps": 24, "elapsed_sec": 2247.512, "expansion_event": "same", "init_loss": 2.177030182792805e-05, "op_loss": 2.415495146124158e-06, "arg_loss": 4.662830542656593e-05, "state_loss": 0.0004528893041424453, "state_train_accuracy": 1.0, "executor_loss": 0.0008614071412011981, "loss": 0.0015903064049780369 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage3_max24", "stage_idx": 3, "local_step": 250, "global_step": 700, "compiler_max_steps": 24, "elapsed_sec": 2421.117, "expansion_event": "same", "init_loss": 1.0922484761977103e-05, "op_loss": 1.3834270475854282e-06, "arg_loss": 1.0591507816570811e-05, "state_loss": 0.00011264691420365125, "state_train_accuracy": 1.0, "executor_loss": 0.00018907476624008268, "loss": 0.00038916105404496193 }, { "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "stage": "stage3_max24", "stage_idx": 3, "local_step": 300, "global_step": 750, "compiler_max_steps": 24, "elapsed_sec": 2594.673, "expansion_event": "same", "init_loss": 1.4439100596064236e-05, "op_loss": 7.095787850630586e-07, "arg_loss": 5.2281602620496415e-06, "state_loss": 6.363669672282413e-05, "state_train_accuracy": 1.0, "executor_loss": 0.00010796535934787244, "loss": 0.00025098066544160247 } ], "metrics": [ { "stage": "stage1_max24", "split": "standard_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9975859150290489, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paraphrase_L8", "n": 64, "max_steps": 24, "executor_accuracy": 0.984375, "executor_target_mass": 0.9790687188506126, "direct_accuracy": NaN, "init_accuracy": 0.984375, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.984375, "state_accuracy": 0.984375, "state_all_exact": 0.984375, "state_prefix_fraction": 0.984375, "state_mean_correct_prefix": 7.875, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "heldout_L8", "n": 64, "max_steps": 24, "executor_accuracy": 0.875, "executor_target_mass": 0.8477062284946442, "direct_accuracy": NaN, "init_accuracy": 0.96875, "op_accuracy": 0.986328125, "arg_accuracy": 0.994140625, "program_exact": 0.875, "state_accuracy": 0.9140625, "state_all_exact": 0.875, "state_prefix_fraction": 0.9140625, "state_mean_correct_prefix": 7.3125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paired_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9969310984015465, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "executor_pair_answer_consistency": 1.0, "executor_pair_both_correct": 1.0, "compiler_pair_program_consistency": 1.0, "compiler_pair_state_consistency": 1.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paired_heldout_L8", "n": 64, "max_steps": 24, "executor_accuracy": 0.90625, "executor_target_mass": 0.8648611083626747, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.994140625, "arg_accuracy": 0.982421875, "program_exact": 0.90625, "state_accuracy": 0.974609375, "state_all_exact": 0.90625, "state_prefix_fraction": 0.974609375, "state_mean_correct_prefix": 7.796875, "executor_pair_answer_consistency": 0.8125, "executor_pair_both_correct": 0.8125, "compiler_pair_program_consistency": 0.8125, "compiler_pair_state_consistency": 0.8125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "standard_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.484375, "executor_target_mass": 0.4918026551604271, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.97265625, "arg_accuracy": 0.9482421875, "program_exact": 0.484375, "state_accuracy": 0.9130859375, "state_all_exact": 0.484375, "state_prefix_fraction": 0.912109375, "state_mean_correct_prefix": 14.59375, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paraphrase_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.359375, "executor_target_mass": 0.3638630360364914, "direct_accuracy": NaN, "init_accuracy": 0.984375, "op_accuracy": 0.9658203125, "arg_accuracy": 0.9306640625, "program_exact": 0.34375, "state_accuracy": 0.8720703125, "state_all_exact": 0.34375, "state_prefix_fraction": 0.8701171875, "state_mean_correct_prefix": 13.921875, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "heldout_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.15625, "executor_target_mass": 0.11565884626179468, "direct_accuracy": NaN, "init_accuracy": 0.984375, "op_accuracy": 0.849609375, "arg_accuracy": 0.7822265625, "program_exact": 0.15625, "state_accuracy": 0.7041015625, "state_all_exact": 0.15625, "state_prefix_fraction": 0.7041015625, "state_mean_correct_prefix": 11.265625, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paired_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.53125, "executor_target_mass": 0.5371711887419224, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.96484375, "arg_accuracy": 0.953125, "program_exact": 0.53125, "state_accuracy": 0.9072265625, "state_all_exact": 0.53125, "state_prefix_fraction": 0.9072265625, "state_mean_correct_prefix": 14.515625, "executor_pair_answer_consistency": 0.40625, "executor_pair_both_correct": 0.40625, "compiler_pair_program_consistency": 0.40625, "compiler_pair_state_consistency": 0.40625, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paired_heldout_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.296875, "executor_target_mass": 0.30182516761124134, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.9150390625, "arg_accuracy": 0.8720703125, "program_exact": 0.265625, "state_accuracy": 0.7978515625, "state_all_exact": 0.265625, "state_prefix_fraction": 0.791015625, "state_mean_correct_prefix": 12.65625, "executor_pair_answer_consistency": 0.0625, "executor_pair_both_correct": 0.03125, "compiler_pair_program_consistency": 0.0625, "compiler_pair_state_consistency": 0.0625, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "standard_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.0, "executor_target_mass": 0.003270446104579605, "direct_accuracy": NaN, "init_accuracy": 0.984375, "op_accuracy": 0.8626302083333334, "arg_accuracy": 0.763671875, "program_exact": 0.0, "state_accuracy": 0.6165364583333334, "state_all_exact": 0.0, "state_prefix_fraction": 0.61328125, "state_mean_correct_prefix": 14.71875, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paraphrase_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.0, "executor_target_mass": 0.004630769952200353, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.81640625, "arg_accuracy": 0.7291666666666666, "program_exact": 0.0, "state_accuracy": 0.6328125, "state_all_exact": 0.0, "state_prefix_fraction": 0.6276041666666666, "state_mean_correct_prefix": 15.0625, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "heldout_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.0, "executor_target_mass": 0.010112411109730601, "direct_accuracy": NaN, "init_accuracy": 0.96875, "op_accuracy": 0.6686197916666666, "arg_accuracy": 0.525390625, "program_exact": 0.0, "state_accuracy": 0.4309895833333333, "state_all_exact": 0.0, "state_prefix_fraction": 0.4264322916666667, "state_mean_correct_prefix": 10.234375, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paired_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.015625, "executor_target_mass": 0.019107205924228765, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.8274739583333334, "arg_accuracy": 0.7259114583333334, "program_exact": 0.015625, "state_accuracy": 0.62109375, "state_all_exact": 0.015625, "state_prefix_fraction": 0.6145833333333334, "state_mean_correct_prefix": 14.75, "executor_pair_answer_consistency": 0.0, "executor_pair_both_correct": 0.0, "compiler_pair_program_consistency": 0.0, "compiler_pair_state_consistency": 0.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage1_max24", "split": "paired_heldout_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.015625, "executor_target_mass": 0.015002730651758611, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.7682291666666666, "arg_accuracy": 0.6575520833333334, "program_exact": 0.0, "state_accuracy": 0.5625, "state_all_exact": 0.0, "state_prefix_fraction": 0.5579427083333334, "state_mean_correct_prefix": 13.390625, "executor_pair_answer_consistency": 0.03125, "executor_pair_both_correct": 0.0, "compiler_pair_program_consistency": 0.0, "compiler_pair_state_consistency": 0.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 300, "stage_idx": 1 }, { "stage": "stage2_max24", "split": "standard_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9983531609177589, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paraphrase_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9994442984461784, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "heldout_L8", "n": 64, "max_steps": 24, "executor_accuracy": 0.796875, "executor_target_mass": 0.784334734082222, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.935546875, "arg_accuracy": 0.931640625, "program_exact": 0.796875, "state_accuracy": 0.869140625, "state_all_exact": 0.796875, "state_prefix_fraction": 0.869140625, "state_mean_correct_prefix": 6.953125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paired_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9995968639850616, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "executor_pair_answer_consistency": 1.0, "executor_pair_both_correct": 1.0, "compiler_pair_program_consistency": 1.0, "compiler_pair_state_consistency": 1.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paired_heldout_L8", "n": 64, "max_steps": 24, "executor_accuracy": 0.953125, "executor_target_mass": 0.9370190650224686, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.984375, "arg_accuracy": 0.974609375, "program_exact": 0.953125, "state_accuracy": 0.966796875, "state_all_exact": 0.953125, "state_prefix_fraction": 0.966796875, "state_mean_correct_prefix": 7.734375, "executor_pair_answer_consistency": 0.90625, "executor_pair_both_correct": 0.90625, "compiler_pair_program_consistency": 0.90625, "compiler_pair_state_consistency": 0.90625, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "standard_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.96875, "executor_target_mass": 0.9598270207643509, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 0.998046875, "program_exact": 0.96875, "state_accuracy": 0.99609375, "state_all_exact": 0.96875, "state_prefix_fraction": 0.99609375, "state_mean_correct_prefix": 15.9375, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paraphrase_L16", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9987776055932045, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 16.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "heldout_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.78125, "executor_target_mass": 0.7407534271478653, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.9501953125, "arg_accuracy": 0.9404296875, "program_exact": 0.78125, "state_accuracy": 0.8544921875, "state_all_exact": 0.78125, "state_prefix_fraction": 0.8544921875, "state_mean_correct_prefix": 13.671875, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paired_L16", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9987620860338211, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 16.0, "executor_pair_answer_consistency": 1.0, "executor_pair_both_correct": 1.0, "compiler_pair_program_consistency": 1.0, "compiler_pair_state_consistency": 1.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paired_heldout_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.796875, "executor_target_mass": 0.7721416726708412, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.931640625, "arg_accuracy": 0.90234375, "program_exact": 0.796875, "state_accuracy": 0.87109375, "state_all_exact": 0.796875, "state_prefix_fraction": 0.869140625, "state_mean_correct_prefix": 13.90625, "executor_pair_answer_consistency": 0.59375, "executor_pair_both_correct": 0.59375, "compiler_pair_program_consistency": 0.59375, "compiler_pair_state_consistency": 0.59375, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "standard_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.03125, "executor_target_mass": 0.028288151671404194, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.791015625, "arg_accuracy": 0.724609375, "program_exact": 0.0, "state_accuracy": 0.6770833333333334, "state_all_exact": 0.0, "state_prefix_fraction": 0.6725260416666666, "state_mean_correct_prefix": 16.140625, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paraphrase_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.0, "executor_target_mass": 0.0052719806426466675, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.794921875, "arg_accuracy": 0.7298177083333334, "program_exact": 0.0, "state_accuracy": 0.65625, "state_all_exact": 0.0, "state_prefix_fraction": 0.6555989583333334, "state_mean_correct_prefix": 15.734375, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "heldout_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.0, "executor_target_mass": 0.0023022345194476657, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.7682291666666666, "arg_accuracy": 0.6946614583333334, "program_exact": 0.0, "state_accuracy": 0.5592447916666666, "state_all_exact": 0.0, "state_prefix_fraction": 0.5559895833333334, "state_mean_correct_prefix": 13.34375, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paired_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.0, "executor_target_mass": 0.004756739002459653, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.7962239583333334, "arg_accuracy": 0.720703125, "program_exact": 0.0, "state_accuracy": 0.6627604166666666, "state_all_exact": 0.0, "state_prefix_fraction": 0.662109375, "state_mean_correct_prefix": 15.890625, "executor_pair_answer_consistency": 0.03125, "executor_pair_both_correct": 0.0, "compiler_pair_program_consistency": 0.03125, "compiler_pair_state_consistency": 0.03125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage2_max24", "split": "paired_heldout_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.0, "executor_target_mass": 0.0090635117412603, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.8001302083333334, "arg_accuracy": 0.7239583333333334, "program_exact": 0.0, "state_accuracy": 0.6217447916666666, "state_all_exact": 0.0, "state_prefix_fraction": 0.6184895833333334, "state_mean_correct_prefix": 14.84375, "executor_pair_answer_consistency": 0.03125, "executor_pair_both_correct": 0.0, "compiler_pair_program_consistency": 0.03125, "compiler_pair_state_consistency": 0.03125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 450, "stage_idx": 2 }, { "stage": "stage3_max24", "split": "standard_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.999921016395092, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paraphrase_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9999374449253082, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "heldout_L8", "n": 64, "max_steps": 24, "executor_accuracy": 0.875, "executor_target_mass": 0.8776780441403389, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.953125, "arg_accuracy": 0.93359375, "program_exact": 0.859375, "state_accuracy": 0.908203125, "state_all_exact": 0.859375, "state_prefix_fraction": 0.90625, "state_mean_correct_prefix": 7.25, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paired_L8", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9999413937330246, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 8.0, "executor_pair_answer_consistency": 1.0, "executor_pair_both_correct": 1.0, "compiler_pair_program_consistency": 1.0, "compiler_pair_state_consistency": 1.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paired_heldout_L8", "n": 64, "max_steps": 24, "executor_accuracy": 0.890625, "executor_target_mass": 0.8786064386367798, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.9609375, "arg_accuracy": 0.9609375, "program_exact": 0.890625, "state_accuracy": 0.931640625, "state_all_exact": 0.890625, "state_prefix_fraction": 0.931640625, "state_mean_correct_prefix": 7.453125, "executor_pair_answer_consistency": 0.78125, "executor_pair_both_correct": 0.78125, "compiler_pair_program_consistency": 0.78125, "compiler_pair_state_consistency": 0.78125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "standard_L16", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9998754113912582, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 16.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paraphrase_L16", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9998693093657494, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 16.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "heldout_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.828125, "executor_target_mass": 0.8274371847510338, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.9287109375, "arg_accuracy": 0.9130859375, "program_exact": 0.828125, "state_accuracy": 0.87109375, "state_all_exact": 0.828125, "state_prefix_fraction": 0.8701171875, "state_mean_correct_prefix": 13.921875, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paired_L16", "n": 64, "max_steps": 24, "executor_accuracy": 1.0, "executor_target_mass": 0.9998904392123222, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 1.0, "state_accuracy": 1.0, "state_all_exact": 1.0, "state_prefix_fraction": 1.0, "state_mean_correct_prefix": 16.0, "executor_pair_answer_consistency": 1.0, "executor_pair_both_correct": 1.0, "compiler_pair_program_consistency": 1.0, "compiler_pair_state_consistency": 1.0, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paired_heldout_L16", "n": 64, "max_steps": 24, "executor_accuracy": 0.90625, "executor_target_mass": 0.8903933465480804, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.9599609375, "arg_accuracy": 0.935546875, "program_exact": 0.90625, "state_accuracy": 0.931640625, "state_all_exact": 0.90625, "state_prefix_fraction": 0.9306640625, "state_mean_correct_prefix": 14.890625, "executor_pair_answer_consistency": 0.8125, "executor_pair_both_correct": 0.8125, "compiler_pair_program_consistency": 0.8125, "compiler_pair_state_consistency": 0.8125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "standard_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.234375, "executor_target_mass": 0.22930023130902555, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.93359375, "arg_accuracy": 0.9049479166666666, "program_exact": 0.171875, "state_accuracy": 0.8828125, "state_all_exact": 0.171875, "state_prefix_fraction": 0.87890625, "state_mean_correct_prefix": 21.09375, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paraphrase_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.921875, "executor_target_mass": 0.9219244420528412, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.9967447916666666, "arg_accuracy": 0.99609375, "program_exact": 0.90625, "state_accuracy": 0.9928385416666666, "state_all_exact": 0.921875, "state_prefix_fraction": 0.9928385416666666, "state_mean_correct_prefix": 23.828125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "heldout_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.234375, "executor_target_mass": 0.23053055258696986, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.888671875, "arg_accuracy": 0.837890625, "program_exact": 0.21875, "state_accuracy": 0.748046875, "state_all_exact": 0.21875, "state_prefix_fraction": 0.7467447916666666, "state_mean_correct_prefix": 17.921875, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paired_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.59375, "executor_target_mass": 0.5956403762102127, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.9602864583333334, "arg_accuracy": 0.9524739583333334, "program_exact": 0.5625, "state_accuracy": 0.9388020833333334, "state_all_exact": 0.5625, "state_prefix_fraction": 0.9368489583333334, "state_mean_correct_prefix": 22.484375, "executor_pair_answer_consistency": 0.25, "executor_pair_both_correct": 0.25, "compiler_pair_program_consistency": 0.1875, "compiler_pair_state_consistency": 0.1875, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 }, { "stage": "stage3_max24", "split": "paired_heldout_L24", "n": 64, "max_steps": 24, "executor_accuracy": 0.21875, "executor_target_mass": 0.20325435511767864, "direct_accuracy": NaN, "init_accuracy": 1.0, "op_accuracy": 0.90625, "arg_accuracy": 0.869140625, "program_exact": 0.21875, "state_accuracy": 0.8196614583333334, "state_all_exact": 0.21875, "state_prefix_fraction": 0.818359375, "state_mean_correct_prefix": 19.640625, "executor_pair_answer_consistency": 0.0625, "executor_pair_both_correct": 0.03125, "compiler_pair_program_consistency": 0.03125, "compiler_pair_state_consistency": 0.03125, "run": "main_max24_curriculum_seed123", "arm": "max24_curriculum", "seed": 123, "global_step": 750, "stage_idx": 3 } ], "checkpoints": [ { "run": "main_max24_curriculum_seed123", "seed": 123, "tag": "stage1_max24_step00300", "stage": "stage1_max24", "global_step": 300, "compiler_max_steps": 24, "checkpoint_dir": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage1_max24_step00300", "model_checkpoint": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage1_max24_step00300/adapter", "heads_checkpoint": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage1_max24_step00300/heads.pt", "selection_split": "paired_L24", "selection_metric": "executor_pair_both_correct", "selection_value": 0.0, "executor_accuracy": 0.015625, "program_exact": 0.015625, "state_prefix_fraction": 0.6145833333333334 }, { "run": "main_max24_curriculum_seed123", "seed": 123, "tag": "stage2_max24_step00450", "stage": "stage2_max24", "global_step": 450, "compiler_max_steps": 24, "checkpoint_dir": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage2_max24_step00450", "model_checkpoint": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage2_max24_step00450/adapter", "heads_checkpoint": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage2_max24_step00450/heads.pt", "selection_split": "paired_L24", "selection_metric": "executor_pair_both_correct", "selection_value": 0.0, "executor_accuracy": 0.0, "program_exact": 0.0, "state_prefix_fraction": 0.662109375 }, { "run": "main_max24_curriculum_seed123", "seed": 123, "tag": "stage3_max24_step00750", "stage": "stage3_max24", "global_step": 750, "compiler_max_steps": 24, "checkpoint_dir": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage3_max24_step00750", "model_checkpoint": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage3_max24_step00750/adapter", "heads_checkpoint": "/workspace/large_artifacts/qwen_compiler_multiseed_reattribution/checkpoints/main_max24_curriculum_seed123/stage3_max24_step00750/heads.pt", "selection_split": "paired_L24", "selection_metric": "executor_pair_both_correct", "selection_value": 0.25, "executor_accuracy": 0.59375, "program_exact": 0.5625, "state_prefix_fraction": 0.9368489583333334 } ], "elapsed_sec": 2722.069 }