{ "args": { "model_id": "hf-internal-testing/tiny-random-LlamaForCausalLM", "load_in_4bit": false, "torch_dtype": "fp32", "device_map": "auto", "use_lora": false, "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "lora_target_modules": "all-linear", "gradient_checkpointing": false, "optimizer": "adamw", "modulus": 17, "max_steps": 6, "train_min_len": 1, "train_max_len": 12, "curriculum_stages": "short:1:2:1,long:3:4:1", "answer_train_min_len": 1, "answer_train_max_len": 4, "eval_lengths": "2,4", "train_template_mode": "mixed", "paired_train": true, "paired_batches": true, "paired_template_modes": "standard,paraphrase", "pair_size": 2, "answer_train_template_mode": "mixed", "eval_template_modes": "standard,paraphrase", "paired_eval": true, "paired_eval_template_modes": "standard,paraphrase", "train_size": 4, "answer_train_size": 8, "eval_size": 4, "train_batch_size": 4, "eval_batch_size": 4, "bootstrap_steps": 0, "answer_steps": 0, "eval_every": 200, "stage_eval_every": 200, "lr": 0.001, "answer_lr_scale": 0.1, "weight_decay": 0.0, "grad_clip": 1.0, "head_width": 64, "rank_temperature": 1.0, "arg_reader_mode": "after_op", "arg_window": 8, "arg_distance_temperature": 2.0, "trace_loss_weight": 1.0, "init_trace_loss_weight": 2.0, "op_trace_loss_weight": 1.0, "arg_trace_loss_weight": 2.0, "selection_loss_weight": 1.0, "role_pos_weight": 20.0, "init_selection_loss_weight": 1.0, "op_selection_loss_weight": 1.0, "arg_selection_loss_weight": 2.0, "executor_loss_weight": 1.0, "state_loss_weight": 0.0, "variants": "copy_trace", "max_length": 128, "seed": 7, "train_seed": -1, "output_dir": "experiments/qwen_state_ladder_compiler/runs/smoke_tiny_no_state_ladder", "checkpoint_dir": "" }, "metadata": { "python": "3.12.3", "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "torch": "2.8.0+cu128", "cuda_available": true, "transformers_loader": "AutoModelForCausalLM", "peft_installed": true, "use_lora": false, "lora_r": 8, "gpu_name": "NVIDIA RTX 6000 Ada Generation", "gpu_vram_gb": 47.383 }, "hidden_dim": 16, "dataset": { "train_size": 4, "train_examples": 16, "curriculum_stages": [ { "name": "short", "min_len": 1, "max_len": 2, "steps": 1 }, { "name": "long", "min_len": 3, "max_len": 4, "steps": 1 } ], "answer_train_size": 8, "eval_size": 4, "train_lengths": [ 1, 12 ], "answer_train_lengths": [ 1, 4 ], "eval_lengths": [ 2, 4 ], "train_template_mode": "mixed", "paired_train": true, "paired_batches": true, "paired_template_modes": [ "standard", "paraphrase" ], "answer_train_template_mode": "mixed", "eval_template_modes": [ "standard", "paraphrase" ], "paired_eval": true, "paired_eval_template_modes": [ "standard", "paraphrase" ] }, "variants": { "copy_trace": { "variant": "copy_trace", "train_seconds": 0.5112645626068115, "checkpoints": [ "large_artifacts/qwen_state_ladder_compiler/checkpoints/smoke_tiny_no_state_ladder/copy_trace/frozen_backbone.txt", "large_artifacts/qwen_state_ladder_compiler/checkpoints/smoke_tiny_no_state_ladder/copy_trace/heads.pt" ], "train_log": [ { "variant": "copy_trace", "stage": "trace_short", "local_step": 1, "step": 1, "loss": 24.303245544433594, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 1.1673901081085205, "init_loss": 0.7878621220588684, "op_loss": 0.0, "arg_loss": 1.7154699563980103, "init_attn_loss": 3.9999570846557617, "op_attn_loss": 3.966416120529175, "arg_attn_loss": 4.350911617279053, "role_loss": 1.4609923362731934, "standard_len2_direct_accuracy": NaN, "standard_len2_direct_target_mass": NaN, "standard_len2_executor_accuracy": 0.0, "standard_len2_executor_target_mass": 0.12301875650882721, "standard_len2_init_accuracy": 0.5, "standard_len2_init_pos_accuracy": 0.0, "standard_len2_op_accuracy": 0.625, "standard_len2_arg_accuracy": 0.25, "standard_len2_op_pos_accuracy": 0.0, "standard_len2_arg_pos_accuracy": 0.0, "standard_len2_program_exact": 0.0, "standard_len2_state_accuracy": 0.25, "standard_len2_state_all_exact": 0.0, "standard_len2_state_prefix_fraction": 0.25, "standard_len2_state_mean_correct_prefix": 0.5, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.25, "standard_len4_executor_target_mass": 0.05972013622522354, "standard_len4_init_accuracy": 0.5, "standard_len4_init_pos_accuracy": 0.0, "standard_len4_op_accuracy": 0.8125, "standard_len4_arg_accuracy": 0.3125, "standard_len4_op_pos_accuracy": 0.125, "standard_len4_arg_pos_accuracy": 0.0, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.125, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.0, "standard_len4_state_mean_correct_prefix": 0.0, "paraphrase_len2_direct_accuracy": NaN, "paraphrase_len2_direct_target_mass": NaN, "paraphrase_len2_executor_accuracy": 0.25, "paraphrase_len2_executor_target_mass": 0.15703503787517548, "paraphrase_len2_init_accuracy": 0.75, "paraphrase_len2_init_pos_accuracy": 0.0, "paraphrase_len2_op_accuracy": 0.625, "paraphrase_len2_arg_accuracy": 0.125, "paraphrase_len2_op_pos_accuracy": 0.0, "paraphrase_len2_arg_pos_accuracy": 0.0, "paraphrase_len2_program_exact": 0.0, "paraphrase_len2_state_accuracy": 0.25, "paraphrase_len2_state_all_exact": 0.0, "paraphrase_len2_state_prefix_fraction": 0.125, "paraphrase_len2_state_mean_correct_prefix": 0.25, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.05925239995121956, "paraphrase_len4_init_accuracy": 0.25, "paraphrase_len4_init_pos_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.625, "paraphrase_len4_arg_accuracy": 0.0625, "paraphrase_len4_op_pos_accuracy": 0.0, "paraphrase_len4_arg_pos_accuracy": 0.0, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.0, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.0, "paraphrase_len4_state_mean_correct_prefix": 0.0, "paired_len2_direct_accuracy": NaN, "paired_len2_direct_target_mass": NaN, "paired_len2_executor_accuracy": 0.0, "paired_len2_executor_target_mass": 0.07686382159590721, "paired_len2_init_accuracy": 0.25, "paired_len2_init_pos_accuracy": 0.0, "paired_len2_op_accuracy": 0.5, "paired_len2_arg_accuracy": 0.0625, "paired_len2_op_pos_accuracy": 0.0, "paired_len2_arg_pos_accuracy": 0.0, "paired_len2_program_exact": 0.0, "paired_len2_state_accuracy": 0.25, "paired_len2_state_all_exact": 0.0, "paired_len2_state_prefix_fraction": 0.25, "paired_len2_state_mean_correct_prefix": 0.5, "paired_len2_pair_true_answer_consistency": 1.0, "paired_len2_pair_true_length_consistency": 1.0, "paired_len2_executor_pair_answer_consistency": 0.75, "paired_len2_executor_pair_both_correct": 0.0, "paired_len2_compiler_pair_program_consistency": 0.75, "paired_len2_compiler_pair_state_consistency": 0.75, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.125, "paired_len4_executor_target_mass": 0.08915121108293533, "paired_len4_init_accuracy": 0.0, "paired_len4_init_pos_accuracy": 0.0, "paired_len4_op_accuracy": 0.875, "paired_len4_arg_accuracy": 0.28125, "paired_len4_op_pos_accuracy": 0.03125, "paired_len4_arg_pos_accuracy": 0.0625, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.125, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0625, "paired_len4_state_mean_correct_prefix": 0.25, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_pair_true_length_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.5, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.25, "paired_len4_compiler_pair_state_consistency": 0.25 }, { "variant": "copy_trace", "stage": "trace_long", "local_step": 1, "step": 2, "loss": 18.11187171936035, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 0.8885325789451599, "init_loss": 0.0, "op_loss": 0.2967061400413513, "arg_loss": 0.0, "init_attn_loss": 4.216744899749756, "op_attn_loss": 3.4238510131835938, "arg_attn_loss": 3.6281633377075195, "role_loss": 2.0297110080718994, "standard_len2_direct_accuracy": NaN, "standard_len2_direct_target_mass": NaN, "standard_len2_executor_accuracy": 0.0, "standard_len2_executor_target_mass": 0.12558454275131226, "standard_len2_init_accuracy": 0.5, "standard_len2_init_pos_accuracy": 0.0, "standard_len2_op_accuracy": 0.625, "standard_len2_arg_accuracy": 0.25, "standard_len2_op_pos_accuracy": 0.0, "standard_len2_arg_pos_accuracy": 0.0, "standard_len2_program_exact": 0.0, "standard_len2_state_accuracy": 0.25, "standard_len2_state_all_exact": 0.0, "standard_len2_state_prefix_fraction": 0.25, "standard_len2_state_mean_correct_prefix": 0.5, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.25, "standard_len4_executor_target_mass": 0.05961025133728981, "standard_len4_init_accuracy": 0.5, "standard_len4_init_pos_accuracy": 0.0, "standard_len4_op_accuracy": 0.9375, "standard_len4_arg_accuracy": 0.375, "standard_len4_op_pos_accuracy": 0.1875, "standard_len4_arg_pos_accuracy": 0.0625, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.1875, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.0, "standard_len4_state_mean_correct_prefix": 0.0, "paraphrase_len2_direct_accuracy": NaN, "paraphrase_len2_direct_target_mass": NaN, "paraphrase_len2_executor_accuracy": 0.25, "paraphrase_len2_executor_target_mass": 0.15745002031326294, "paraphrase_len2_init_accuracy": 0.75, "paraphrase_len2_init_pos_accuracy": 0.0, "paraphrase_len2_op_accuracy": 0.625, "paraphrase_len2_arg_accuracy": 0.125, "paraphrase_len2_op_pos_accuracy": 0.0, "paraphrase_len2_arg_pos_accuracy": 0.0, "paraphrase_len2_program_exact": 0.0, "paraphrase_len2_state_accuracy": 0.25, "paraphrase_len2_state_all_exact": 0.0, "paraphrase_len2_state_prefix_fraction": 0.125, "paraphrase_len2_state_mean_correct_prefix": 0.25, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.05954747647047043, "paraphrase_len4_init_accuracy": 0.25, "paraphrase_len4_init_pos_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.625, "paraphrase_len4_arg_accuracy": 0.125, "paraphrase_len4_op_pos_accuracy": 0.0, "paraphrase_len4_arg_pos_accuracy": 0.0, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.0, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.0, "paraphrase_len4_state_mean_correct_prefix": 0.0, "paired_len2_direct_accuracy": NaN, "paired_len2_direct_target_mass": NaN, "paired_len2_executor_accuracy": 0.0, "paired_len2_executor_target_mass": 0.0775954071432352, "paired_len2_init_accuracy": 0.25, "paired_len2_init_pos_accuracy": 0.0, "paired_len2_op_accuracy": 0.5, "paired_len2_arg_accuracy": 0.0625, "paired_len2_op_pos_accuracy": 0.0, "paired_len2_arg_pos_accuracy": 0.0, "paired_len2_program_exact": 0.0, "paired_len2_state_accuracy": 0.25, "paired_len2_state_all_exact": 0.0, "paired_len2_state_prefix_fraction": 0.25, "paired_len2_state_mean_correct_prefix": 0.5, "paired_len2_pair_true_answer_consistency": 1.0, "paired_len2_pair_true_length_consistency": 1.0, "paired_len2_executor_pair_answer_consistency": 0.75, "paired_len2_executor_pair_both_correct": 0.0, "paired_len2_compiler_pair_program_consistency": 0.75, "paired_len2_compiler_pair_state_consistency": 0.75, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.08894691988825798, "paired_len4_init_accuracy": 0.0, "paired_len4_init_pos_accuracy": 0.0, "paired_len4_op_accuracy": 0.90625, "paired_len4_arg_accuracy": 0.34375, "paired_len4_op_pos_accuracy": 0.09375, "paired_len4_arg_pos_accuracy": 0.0625, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.09375, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0625, "paired_len4_state_mean_correct_prefix": 0.25, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_pair_true_length_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.25, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.0, "paired_len4_compiler_pair_state_consistency": 0.0 } ], "final_metrics": { "standard_len2": { "variant": "copy_trace", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.12558454275131226, "init_accuracy": 0.5, "init_pos_accuracy": 0.0, "op_accuracy": 0.625, "arg_accuracy": 0.25, "op_pos_accuracy": 0.0, "arg_pos_accuracy": 0.0, "program_exact": 0.0, "state_accuracy": 0.25, "state_all_exact": 0.0, "state_prefix_fraction": 0.25, "state_mean_correct_prefix": 0.5 }, "standard_len4": { "variant": "copy_trace", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.25, "executor_target_mass": 0.05961025133728981, "init_accuracy": 0.5, "init_pos_accuracy": 0.0, "op_accuracy": 0.9375, "arg_accuracy": 0.375, "op_pos_accuracy": 0.1875, "arg_pos_accuracy": 0.0625, "program_exact": 0.0, "state_accuracy": 0.1875, "state_all_exact": 0.0, "state_prefix_fraction": 0.0, "state_mean_correct_prefix": 0.0 }, "paraphrase_len2": { "variant": "copy_trace", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.25, "executor_target_mass": 0.15745002031326294, "init_accuracy": 0.75, "init_pos_accuracy": 0.0, "op_accuracy": 0.625, "arg_accuracy": 0.125, "op_pos_accuracy": 0.0, "arg_pos_accuracy": 0.0, "program_exact": 0.0, "state_accuracy": 0.25, "state_all_exact": 0.0, "state_prefix_fraction": 0.125, "state_mean_correct_prefix": 0.25 }, "paraphrase_len4": { "variant": "copy_trace", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.05954747647047043, "init_accuracy": 0.25, "init_pos_accuracy": 0.0, "op_accuracy": 0.625, "arg_accuracy": 0.125, "op_pos_accuracy": 0.0, "arg_pos_accuracy": 0.0, "program_exact": 0.0, "state_accuracy": 0.0, "state_all_exact": 0.0, "state_prefix_fraction": 0.0, "state_mean_correct_prefix": 0.0 }, "paired_len2": { "variant": "copy_trace", "n": 8.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.0775954071432352, "init_accuracy": 0.25, "init_pos_accuracy": 0.0, "op_accuracy": 0.5, "arg_accuracy": 0.0625, "op_pos_accuracy": 0.0, "arg_pos_accuracy": 0.0, "program_exact": 0.0, "state_accuracy": 0.25, "state_all_exact": 0.0, "state_prefix_fraction": 0.25, "state_mean_correct_prefix": 0.5, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.75, "executor_pair_both_correct": 0.0, "compiler_pair_program_consistency": 0.75, "compiler_pair_state_consistency": 0.75 }, "paired_len4": { "variant": "copy_trace", "n": 8.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.08894691988825798, "init_accuracy": 0.0, "init_pos_accuracy": 0.0, "op_accuracy": 0.90625, "arg_accuracy": 0.34375, "op_pos_accuracy": 0.09375, "arg_pos_accuracy": 0.0625, "program_exact": 0.0, "state_accuracy": 0.09375, "state_all_exact": 0.0, "state_prefix_fraction": 0.0625, "state_mean_correct_prefix": 0.25, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.25, "executor_pair_both_correct": 0.0, "compiler_pair_program_consistency": 0.0, "compiler_pair_state_consistency": 0.0 } } } } }