{ "checkpoint": { "adapter_path": "large_artifacts/qwen_hidden_vm_mixed_domains/checkpoints/smoke_hidden_vm_mixed/adapter", "heads_path": "large_artifacts/qwen_hidden_vm_mixed_domains/checkpoints/smoke_hidden_vm_mixed/heads.pt", "run": "smoke_hidden_vm_mixed" }, "metadata": { "args": { "arg_distance_temperature": 4.0, "arg_selection_loss_weight": 4.0, "arg_trace_loss_weight": 4.0, "arg_window": 16, "device_map": "auto", "direct_answer_loss_weight": 0.0, "domain_eval_examples": 2, "domains": "arithmetic,calendar,unit,list,boolean,lookup", "eval_batch_size": 2, "eval_every": 0, "eval_examples": 6, "eval_length": 4, "eval_pairs": 4, "eval_seed": 92001, "executor_loss_weight": 0.2, "grad_clip": 1.0, "gradient_checkpointing": true, "hard_length": 5, "head_width": 512, "init_selection_loss_weight": 1.0, "init_trace_loss_weight": 4.0, "load_in_4bit": true, "log_every": 1, "lora_alpha": 16, "lora_dropout": 0.05, "lora_r": 8, "lora_target_modules": "all-linear", "lr": 5e-05, "max_length": 384, "max_steps": 5, "model_id": "Qwen/Qwen3-4B", "op_selection_loss_weight": 1.0, "op_trace_loss_weight": 1.0, "rank_temperature": 1.0, "role_pos_weight": 20.0, "run_name": "smoke_hidden_vm_mixed", "seed": 92, "selection_loss_weight": 1.0, "state_loss_weight": 0.05, "torch_dtype": "bf16", "trace_loss_weight": 1.0, "train_batch_size": 1, "train_examples": 12, "train_lora": true, "train_max_len": 4, "train_min_len": 1, "train_steps": 2, "train_template_mode": "mixed", "val_examples": 6, "value_modulus": 97, "variant": "trace", "weight_decay": 0.0 }, "cuda_available": true, "gpu_name": "NVIDIA RTX 6000 Ada Generation", "hidden_dim": 2560, "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "python": "3.12.3", "torch": "2.8.0+cu128", "train_seconds": 16.028564929962158 }, "metrics": { "domain_arithmetic": { "arg_accuracy": 0.5, "direct_accuracy": 0.0, "domain_arithmetic_direct_accuracy": 0.0, "domain_arithmetic_executor_accuracy": 0.0, "domain_arithmetic_n": 2.0, "executor_accuracy": 0.0, "init_accuracy": 0.0, "n": 2.0, "op_accuracy": 0.625, "program_exact": 0.0, "state_accuracy": 0.0, "state_prefix_fraction": 0.0 }, "domain_boolean": { "arg_accuracy": 0.75, "direct_accuracy": 0.0, "domain_boolean_direct_accuracy": 0.0, "domain_boolean_executor_accuracy": 0.5, "domain_boolean_n": 2.0, "executor_accuracy": 0.5, "init_accuracy": 0.5, "n": 2.0, "op_accuracy": 1.0, "program_exact": 0.0, "state_accuracy": 0.125, "state_prefix_fraction": 0.0 }, "domain_calendar": { "arg_accuracy": 0.375, "direct_accuracy": 0.0, "domain_calendar_direct_accuracy": 0.0, "domain_calendar_executor_accuracy": 0.5, "domain_calendar_n": 2.0, "executor_accuracy": 0.5, "init_accuracy": 0.0, "n": 2.0, "op_accuracy": 0.625, "program_exact": 0.0, "state_accuracy": 0.375, "state_prefix_fraction": 0.125 }, "domain_list": { "arg_accuracy": 0.5, "direct_accuracy": 0.0, "domain_list_direct_accuracy": 0.0, "domain_list_executor_accuracy": 0.5, "domain_list_n": 2.0, "executor_accuracy": 0.5, "init_accuracy": 0.0, "n": 2.0, "op_accuracy": 1.0, "program_exact": 0.0, "state_accuracy": 0.125, "state_prefix_fraction": 0.0 }, "domain_lookup": { "arg_accuracy": 0.125, "direct_accuracy": 0.0, "domain_lookup_direct_accuracy": 0.0, "domain_lookup_executor_accuracy": 0.0, "domain_lookup_n": 2.0, "executor_accuracy": 0.0, "init_accuracy": 0.0, "n": 2.0, "op_accuracy": 0.375, "program_exact": 0.0, "state_accuracy": 0.125, "state_prefix_fraction": 0.125 }, "domain_unit": { "arg_accuracy": 0.5, "direct_accuracy": 0.0, "domain_unit_direct_accuracy": 0.0, "domain_unit_executor_accuracy": 0.0, "domain_unit_n": 2.0, "executor_accuracy": 0.0, "init_accuracy": 0.0, "n": 2.0, "op_accuracy": 0.875, "program_exact": 0.0, "state_accuracy": 0.0, "state_prefix_fraction": 0.0 }, "fresh_paired_mixed": { "arg_accuracy": 0.40625, "compiler_pair_program_consistency": 0.25, "compiler_pair_state_consistency": 0.5, "direct_accuracy": 0.25, "domain_boolean_direct_accuracy": 1.0, "domain_boolean_executor_accuracy": 1.0, "domain_boolean_n": 2.0, "domain_calendar_direct_accuracy": 0.0, "domain_calendar_executor_accuracy": 0.0, "domain_calendar_n": 4.0, "domain_list_direct_accuracy": 0.0, "domain_list_executor_accuracy": 1.0, "domain_list_n": 2.0, "executor_accuracy": 0.5, "executor_pair_answer_consistency": 0.75, "executor_pair_both_correct": 0.5, "init_accuracy": 0.125, "n": 8.0, "op_accuracy": 0.78125, "program_exact": 0.0, "state_accuracy": 0.125, "state_prefix_fraction": 0.0 }, "fresh_paraphrase_mixed": { "arg_accuracy": 0.625, "direct_accuracy": 0.16666666666666666, "domain_arithmetic_direct_accuracy": 0.0, "domain_arithmetic_executor_accuracy": 0.0, "domain_arithmetic_n": 2.0, "domain_calendar_direct_accuracy": 1.0, "domain_calendar_executor_accuracy": 0.0, "domain_calendar_n": 1.0, "domain_list_direct_accuracy": 0.0, "domain_list_executor_accuracy": 1.0, "domain_list_n": 1.0, "domain_unit_direct_accuracy": 0.0, "domain_unit_executor_accuracy": 0.0, "domain_unit_n": 2.0, "executor_accuracy": 0.16666666666666666, "init_accuracy": 0.0, "n": 6.0, "op_accuracy": 0.6666666666666666, "program_exact": 0.0, "state_accuracy": 0.041666666666666664, "state_prefix_fraction": 0.0 }, "fresh_standard_mixed": { "arg_accuracy": 0.2916666666666667, "direct_accuracy": 0.0, "domain_calendar_direct_accuracy": 0.0, "domain_calendar_executor_accuracy": 0.0, "domain_calendar_n": 1.0, "domain_list_direct_accuracy": 0.0, "domain_list_executor_accuracy": 0.6666666666666666, "domain_list_n": 3.0, "domain_lookup_direct_accuracy": 0.0, "domain_lookup_executor_accuracy": 0.0, "domain_lookup_n": 2.0, "executor_accuracy": 0.3333333333333333, "init_accuracy": 0.0, "n": 6.0, "op_accuracy": 0.7083333333333334, "program_exact": 0.0, "state_accuracy": 0.25, "state_prefix_fraction": 0.16666666666666666 }, "hard_paraphrase_mixed": { "arg_accuracy": 0.36666666666666664, "direct_accuracy": 0.16666666666666666, "domain_arithmetic_direct_accuracy": 0.0, "domain_arithmetic_executor_accuracy": 0.0, "domain_arithmetic_n": 1.0, "domain_boolean_direct_accuracy": 1.0, "domain_boolean_executor_accuracy": 0.0, "domain_boolean_n": 1.0, "domain_list_direct_accuracy": 0.0, "domain_list_executor_accuracy": 0.5, "domain_list_n": 2.0, "domain_lookup_direct_accuracy": 0.0, "domain_lookup_executor_accuracy": 0.0, "domain_lookup_n": 1.0, "domain_unit_direct_accuracy": 0.0, "domain_unit_executor_accuracy": 0.0, "domain_unit_n": 1.0, "executor_accuracy": 0.16666666666666666, "init_accuracy": 0.16666666666666666, "n": 6.0, "op_accuracy": 0.7666666666666667, "program_exact": 0.0, "state_accuracy": 0.13333333333333333, "state_prefix_fraction": 0.1 }, "hard_standard_mixed": { "arg_accuracy": 0.43333333333333335, "direct_accuracy": 0.16666666666666666, "domain_arithmetic_direct_accuracy": 0.0, "domain_arithmetic_executor_accuracy": 0.0, "domain_arithmetic_n": 2.0, "domain_boolean_direct_accuracy": 0.0, "domain_boolean_executor_accuracy": 1.0, "domain_boolean_n": 1.0, "domain_calendar_direct_accuracy": 1.0, "domain_calendar_executor_accuracy": 0.0, "domain_calendar_n": 1.0, "domain_unit_direct_accuracy": 0.0, "domain_unit_executor_accuracy": 0.0, "domain_unit_n": 2.0, "executor_accuracy": 0.16666666666666666, "init_accuracy": 0.16666666666666666, "n": 6.0, "op_accuracy": 0.7666666666666667, "program_exact": 0.0, "state_accuracy": 0.03333333333333333, "state_prefix_fraction": 0.0 }, "val_mixed": { "arg_accuracy": 0.4583333333333333, "direct_accuracy": 0.16666666666666666, "domain_arithmetic_direct_accuracy": 0.0, "domain_arithmetic_executor_accuracy": 0.0, "domain_arithmetic_n": 1.0, "domain_boolean_direct_accuracy": 1.0, "domain_boolean_executor_accuracy": 1.0, "domain_boolean_n": 1.0, "domain_lookup_direct_accuracy": 0.0, "domain_lookup_executor_accuracy": 0.0, "domain_lookup_n": 2.0, "domain_unit_direct_accuracy": 0.0, "domain_unit_executor_accuracy": 0.0, "domain_unit_n": 2.0, "executor_accuracy": 0.16666666666666666, "init_accuracy": 0.0, "n": 6.0, "op_accuracy": 0.7083333333333334, "program_exact": 0.0, "state_accuracy": 0.08333333333333333, "state_prefix_fraction": 0.041666666666666664 } }, "train_log": [ { "domain_arithmetic_arg_accuracy": 0.625, "domain_arithmetic_direct_accuracy": 0.0, "domain_arithmetic_domain_arithmetic_direct_accuracy": 0.0, "domain_arithmetic_domain_arithmetic_executor_accuracy": 0.0, "domain_arithmetic_domain_arithmetic_n": 2.0, "domain_arithmetic_executor_accuracy": 0.0, "domain_arithmetic_init_accuracy": 0.0, "domain_arithmetic_op_accuracy": 0.875, "domain_arithmetic_program_exact": 0.0, "domain_arithmetic_state_accuracy": 0.0, "domain_arithmetic_state_prefix_fraction": 0.0, "domain_boolean_arg_accuracy": 0.625, "domain_boolean_direct_accuracy": 0.0, "domain_boolean_domain_boolean_direct_accuracy": 0.0, "domain_boolean_domain_boolean_executor_accuracy": 0.5, "domain_boolean_domain_boolean_n": 2.0, "domain_boolean_executor_accuracy": 0.5, "domain_boolean_init_accuracy": 0.0, "domain_boolean_op_accuracy": 1.0, "domain_boolean_program_exact": 0.0, "domain_boolean_state_accuracy": 0.125, "domain_boolean_state_prefix_fraction": 0.0, "domain_calendar_arg_accuracy": 0.125, "domain_calendar_direct_accuracy": 0.0, "domain_calendar_domain_calendar_direct_accuracy": 0.0, "domain_calendar_domain_calendar_executor_accuracy": 0.0, "domain_calendar_domain_calendar_n": 2.0, "domain_calendar_executor_accuracy": 0.0, "domain_calendar_init_accuracy": 0.0, "domain_calendar_op_accuracy": 0.625, "domain_calendar_program_exact": 0.0, "domain_calendar_state_accuracy": 0.25, "domain_calendar_state_prefix_fraction": 0.125, "domain_list_arg_accuracy": 0.125, "domain_list_direct_accuracy": 0.0, "domain_list_domain_list_direct_accuracy": 0.0, "domain_list_domain_list_executor_accuracy": 1.0, "domain_list_domain_list_n": 2.0, "domain_list_executor_accuracy": 1.0, "domain_list_init_accuracy": 0.0, "domain_list_op_accuracy": 1.0, "domain_list_program_exact": 0.0, "domain_list_state_accuracy": 0.625, "domain_list_state_prefix_fraction": 0.0, "domain_lookup_arg_accuracy": 0.0, "domain_lookup_direct_accuracy": 0.0, "domain_lookup_domain_lookup_direct_accuracy": 0.0, "domain_lookup_domain_lookup_executor_accuracy": 0.0, "domain_lookup_domain_lookup_n": 2.0, "domain_lookup_executor_accuracy": 0.0, "domain_lookup_init_accuracy": 0.0, "domain_lookup_op_accuracy": 0.25, "domain_lookup_program_exact": 0.0, "domain_lookup_state_accuracy": 0.0, "domain_lookup_state_prefix_fraction": 0.0, "domain_unit_arg_accuracy": 0.375, "domain_unit_direct_accuracy": 0.0, "domain_unit_domain_unit_direct_accuracy": 0.0, "domain_unit_domain_unit_executor_accuracy": 0.0, "domain_unit_domain_unit_n": 2.0, "domain_unit_executor_accuracy": 0.0, "domain_unit_init_accuracy": 0.0, "domain_unit_op_accuracy": 0.75, "domain_unit_program_exact": 0.0, "domain_unit_state_accuracy": 0.0, "domain_unit_state_prefix_fraction": 0.0, "fresh_paired_mixed_arg_accuracy": 0.09375, "fresh_paired_mixed_compiler_pair_program_consistency": 0.5, "fresh_paired_mixed_compiler_pair_state_consistency": 0.5, "fresh_paired_mixed_direct_accuracy": 0.25, "fresh_paired_mixed_domain_boolean_direct_accuracy": 1.0, "fresh_paired_mixed_domain_boolean_executor_accuracy": 1.0, "fresh_paired_mixed_domain_boolean_n": 2.0, "fresh_paired_mixed_domain_calendar_direct_accuracy": 0.0, "fresh_paired_mixed_domain_calendar_executor_accuracy": 0.0, "fresh_paired_mixed_domain_calendar_n": 4.0, "fresh_paired_mixed_domain_list_direct_accuracy": 0.0, "fresh_paired_mixed_domain_list_executor_accuracy": 1.0, "fresh_paired_mixed_domain_list_n": 2.0, "fresh_paired_mixed_executor_accuracy": 0.5, "fresh_paired_mixed_executor_pair_answer_consistency": 0.75, "fresh_paired_mixed_executor_pair_both_correct": 0.5, "fresh_paired_mixed_init_accuracy": 0.125, "fresh_paired_mixed_op_accuracy": 0.71875, "fresh_paired_mixed_program_exact": 0.0, "fresh_paired_mixed_state_accuracy": 0.125, "fresh_paired_mixed_state_prefix_fraction": 0.0, "fresh_paraphrase_mixed_arg_accuracy": 0.25, "fresh_paraphrase_mixed_direct_accuracy": 0.16666666666666666, "fresh_paraphrase_mixed_domain_arithmetic_direct_accuracy": 0.0, "fresh_paraphrase_mixed_domain_arithmetic_executor_accuracy": 0.0, "fresh_paraphrase_mixed_domain_arithmetic_n": 2.0, "fresh_paraphrase_mixed_domain_calendar_direct_accuracy": 1.0, "fresh_paraphrase_mixed_domain_calendar_executor_accuracy": 0.0, "fresh_paraphrase_mixed_domain_calendar_n": 1.0, "fresh_paraphrase_mixed_domain_list_direct_accuracy": 0.0, "fresh_paraphrase_mixed_domain_list_executor_accuracy": 1.0, "fresh_paraphrase_mixed_domain_list_n": 1.0, "fresh_paraphrase_mixed_domain_unit_direct_accuracy": 0.0, "fresh_paraphrase_mixed_domain_unit_executor_accuracy": 0.0, "fresh_paraphrase_mixed_domain_unit_n": 2.0, "fresh_paraphrase_mixed_executor_accuracy": 0.16666666666666666, "fresh_paraphrase_mixed_init_accuracy": 0.0, "fresh_paraphrase_mixed_op_accuracy": 0.7083333333333334, "fresh_paraphrase_mixed_program_exact": 0.0, "fresh_paraphrase_mixed_state_accuracy": 0.041666666666666664, "fresh_paraphrase_mixed_state_prefix_fraction": 0.0, "fresh_standard_mixed_arg_accuracy": 0.041666666666666664, "fresh_standard_mixed_direct_accuracy": 0.0, "fresh_standard_mixed_domain_calendar_direct_accuracy": 0.0, "fresh_standard_mixed_domain_calendar_executor_accuracy": 0.0, "fresh_standard_mixed_domain_calendar_n": 1.0, "fresh_standard_mixed_domain_list_direct_accuracy": 0.0, "fresh_standard_mixed_domain_list_executor_accuracy": 0.3333333333333333, "fresh_standard_mixed_domain_list_n": 3.0, "fresh_standard_mixed_domain_lookup_direct_accuracy": 0.0, "fresh_standard_mixed_domain_lookup_executor_accuracy": 0.0, "fresh_standard_mixed_domain_lookup_n": 2.0, "fresh_standard_mixed_executor_accuracy": 0.16666666666666666, "fresh_standard_mixed_init_accuracy": 0.0, "fresh_standard_mixed_op_accuracy": 0.5416666666666666, "fresh_standard_mixed_program_exact": 0.0, "fresh_standard_mixed_state_accuracy": 0.125, "fresh_standard_mixed_state_prefix_fraction": 0.08333333333333333, "hard_paraphrase_mixed_arg_accuracy": 0.4666666666666667, "hard_paraphrase_mixed_direct_accuracy": 0.16666666666666666, "hard_paraphrase_mixed_domain_arithmetic_direct_accuracy": 0.0, "hard_paraphrase_mixed_domain_arithmetic_executor_accuracy": 0.0, "hard_paraphrase_mixed_domain_arithmetic_n": 1.0, "hard_paraphrase_mixed_domain_boolean_direct_accuracy": 1.0, "hard_paraphrase_mixed_domain_boolean_executor_accuracy": 1.0, "hard_paraphrase_mixed_domain_boolean_n": 1.0, "hard_paraphrase_mixed_domain_list_direct_accuracy": 0.0, "hard_paraphrase_mixed_domain_list_executor_accuracy": 0.0, "hard_paraphrase_mixed_domain_list_n": 2.0, "hard_paraphrase_mixed_domain_lookup_direct_accuracy": 0.0, "hard_paraphrase_mixed_domain_lookup_executor_accuracy": 0.0, "hard_paraphrase_mixed_domain_lookup_n": 1.0, "hard_paraphrase_mixed_domain_unit_direct_accuracy": 0.0, "hard_paraphrase_mixed_domain_unit_executor_accuracy": 0.0, "hard_paraphrase_mixed_domain_unit_n": 1.0, "hard_paraphrase_mixed_executor_accuracy": 0.16666666666666666, "hard_paraphrase_mixed_init_accuracy": 0.16666666666666666, "hard_paraphrase_mixed_op_accuracy": 0.7333333333333333, "hard_paraphrase_mixed_program_exact": 0.0, "hard_paraphrase_mixed_state_accuracy": 0.13333333333333333, "hard_paraphrase_mixed_state_prefix_fraction": 0.1, "hard_standard_mixed_arg_accuracy": 0.43333333333333335, "hard_standard_mixed_direct_accuracy": 0.16666666666666666, "hard_standard_mixed_domain_arithmetic_direct_accuracy": 0.0, "hard_standard_mixed_domain_arithmetic_executor_accuracy": 0.0, "hard_standard_mixed_domain_arithmetic_n": 2.0, "hard_standard_mixed_domain_boolean_direct_accuracy": 0.0, "hard_standard_mixed_domain_boolean_executor_accuracy": 1.0, "hard_standard_mixed_domain_boolean_n": 1.0, "hard_standard_mixed_domain_calendar_direct_accuracy": 1.0, "hard_standard_mixed_domain_calendar_executor_accuracy": 0.0, "hard_standard_mixed_domain_calendar_n": 1.0, "hard_standard_mixed_domain_unit_direct_accuracy": 0.0, "hard_standard_mixed_domain_unit_executor_accuracy": 0.0, "hard_standard_mixed_domain_unit_n": 2.0, "hard_standard_mixed_executor_accuracy": 0.16666666666666666, "hard_standard_mixed_init_accuracy": 0.0, "hard_standard_mixed_op_accuracy": 0.7666666666666667, "hard_standard_mixed_program_exact": 0.0, "hard_standard_mixed_state_accuracy": 0.03333333333333333, "hard_standard_mixed_state_prefix_fraction": 0.0, "phase": "baseline", "step": 0, "val_mixed_arg_accuracy": 0.25, "val_mixed_direct_accuracy": 0.16666666666666666, "val_mixed_domain_arithmetic_direct_accuracy": 0.0, "val_mixed_domain_arithmetic_executor_accuracy": 0.0, "val_mixed_domain_arithmetic_n": 1.0, "val_mixed_domain_boolean_direct_accuracy": 1.0, "val_mixed_domain_boolean_executor_accuracy": 1.0, "val_mixed_domain_boolean_n": 1.0, "val_mixed_domain_lookup_direct_accuracy": 0.0, "val_mixed_domain_lookup_executor_accuracy": 0.0, "val_mixed_domain_lookup_n": 2.0, "val_mixed_domain_unit_direct_accuracy": 0.0, "val_mixed_domain_unit_executor_accuracy": 0.0, "val_mixed_domain_unit_n": 2.0, "val_mixed_executor_accuracy": 0.16666666666666666, "val_mixed_init_accuracy": 0.0, "val_mixed_op_accuracy": 0.5833333333333334, "val_mixed_program_exact": 0.0, "val_mixed_state_accuracy": 0.16666666666666666, "val_mixed_state_prefix_fraction": 0.08333333333333333 }, { "arg_attn_loss": 4.0197834968566895, "arg_loss": 1.460241675376892, "executor_loss": 1.4475880861282349, "init_attn_loss": 4.118611812591553, "init_loss": 1.41404128074646, "loss": 38.25275802612305, "op_attn_loss": 3.779315233230591, "op_loss": 0.37243854999542236, "phase": "train", "role_loss": 2.0626916885375977, "state_loss": 1.0783696174621582, "state_train_accuracy": 0.6666666865348816, "step": 1 }, { "arg_attn_loss": 4.7996826171875, "arg_loss": 1.9338560104370117, "executor_loss": 2.016059398651123, "init_attn_loss": 4.377261161804199, "init_loss": 1.969316005706787, "loss": 46.85683059692383, "op_attn_loss": 4.587967395782471, "op_loss": 0.5084096789360046, "phase": "train", "role_loss": 2.0810060501098633, "state_loss": 1.7510943412780762, "state_train_accuracy": 0.5, "step": 2 } ] }