{ "args": { "model_id": "Qwen/Qwen3-4B", "load_in_4bit": true, "torch_dtype": "bf16", "device_map": "auto", "use_lora": true, "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "lora_target_modules": "all-linear", "gradient_checkpointing": true, "optimizer": "paged_adamw_8bit", "modulus": 97, "max_steps": 24, "train_min_len": 1, "train_max_len": 12, "curriculum_stages": "answer:1:24:300", "answer_train_min_len": 1, "answer_train_max_len": 24, "eval_lengths": "4,8,12,24", "train_template_mode": "mixed", "paired_train": true, "paired_batches": true, "paired_template_modes": "standard,paraphrase", "pair_size": 2, "answer_train_template_mode": "mixed", "eval_template_modes": "standard,paraphrase", "paired_eval": true, "paired_eval_template_modes": "standard,paraphrase", "train_size": 128, "answer_train_size": 512, "eval_size": 64, "train_batch_size": 4, "eval_batch_size": 8, "bootstrap_steps": 0, "answer_steps": 0, "eval_every": 100, "stage_eval_every": 100, "lr": 0.0002, "answer_lr_scale": 0.1, "weight_decay": 0.0, "grad_clip": 1.0, "head_width": 512, "register_width": 512, "register_layers": 1, "register_heads": 4, "register_dropout": 0.05, "register_style": "bare", "trace_loss_weight": 1.0, "init_trace_loss_weight": 4.0, "op_trace_loss_weight": 1.0, "arg_trace_loss_weight": 4.0, "executor_loss_weight": 1.0, "state_loss_weight": 0.25, "variants": "direct", "max_length": 768, "seed": 7, "train_seed": -1, "output_dir": "experiments/qwen_register_token_latent_compiler/runs/control_direct_answer_s300", "checkpoint_dir": "large_artifacts/qwen_register_token_latent_compiler/checkpoints/control_direct_answer_s300" }, "metadata": { "python": "3.12.3", "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "torch": "2.8.0+cu128", "cuda_available": true, "transformers_loader": "AutoModelForCausalLM", "peft_installed": true, "use_lora": true, "lora_r": 8, "gpu_name": "NVIDIA RTX 6000 Ada Generation", "gpu_vram_gb": 47.383 }, "hidden_dim": 2560, "register_count": 49, "dataset": { "train_size": 128, "train_examples": 256, "curriculum_stages": [ { "name": "answer", "min_len": 1, "max_len": 24, "steps": 300 } ], "answer_train_size": 512, "eval_size": 64, "eval_lengths": [ 4, 8, 12, 24 ], "train_template_mode": "mixed", "paired_train": true, "paired_template_modes": [ "standard", "paraphrase" ], "answer_train_template_mode": "mixed", "eval_template_modes": [ "standard", "paraphrase" ], "paired_eval": true, "paired_eval_template_modes": [ "standard", "paraphrase" ], "register_style": "bare" }, "variants": { "direct": { "variant": "direct", "train_seconds": 845.8741309642792, "checkpoints": [ "large_artifacts/qwen_register_token_latent_compiler/checkpoints/control_direct_answer_s300/direct/adapter", "large_artifacts/qwen_register_token_latent_compiler/checkpoints/control_direct_answer_s300/direct/heads.pt" ], "train_log": [ { "variant": "direct", "stage": "answer_only", "local_step": 1, "step": 1, "loss": 4.548712730407715, "trace_loss_active": false, "executor_loss_active": false, "state_loss_active": false, "lr_scale": 1.0, "direct_loss": 4.548712730407715, "standard_len4_direct_accuracy": 0.0, "standard_len4_direct_target_mass": 0.009991438477300107, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len4_state_accuracy": NaN, "standard_len4_state_all_exact": NaN, "standard_len4_state_prefix_fraction": NaN, "standard_len8_direct_accuracy": 0.015625, "standard_len8_direct_target_mass": 0.010375249199569225, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len8_state_accuracy": NaN, "standard_len8_state_all_exact": NaN, "standard_len8_state_prefix_fraction": NaN, "standard_len12_direct_accuracy": 0.0, "standard_len12_direct_target_mass": 0.00990655121859163, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len12_state_accuracy": NaN, "standard_len12_state_all_exact": NaN, "standard_len12_state_prefix_fraction": NaN, "standard_len24_direct_accuracy": 0.015625, "standard_len24_direct_target_mass": 0.010577832232229412, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "standard_len24_state_accuracy": NaN, "standard_len24_state_all_exact": NaN, "standard_len24_state_prefix_fraction": NaN, "paraphrase_len4_direct_accuracy": 0.0, "paraphrase_len4_direct_target_mass": 0.010066985269077122, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len4_state_accuracy": NaN, "paraphrase_len4_state_all_exact": NaN, "paraphrase_len4_state_prefix_fraction": NaN, "paraphrase_len8_direct_accuracy": 0.0, "paraphrase_len8_direct_target_mass": 0.01072024879977107, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len8_state_accuracy": NaN, "paraphrase_len8_state_all_exact": NaN, "paraphrase_len8_state_prefix_fraction": NaN, "paraphrase_len12_direct_accuracy": 0.0, "paraphrase_len12_direct_target_mass": 0.010127400164492428, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len12_state_accuracy": NaN, "paraphrase_len12_state_all_exact": NaN, "paraphrase_len12_state_prefix_fraction": NaN, "paraphrase_len24_direct_accuracy": 0.015625, "paraphrase_len24_direct_target_mass": 0.010251763858832419, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN, "paraphrase_len24_state_accuracy": NaN, "paraphrase_len24_state_all_exact": NaN, "paraphrase_len24_state_prefix_fraction": NaN, "paired_len4_direct_accuracy": 0.0, "paired_len4_direct_target_mass": 0.010528439248446375, "paired_len4_executor_accuracy": NaN, "paired_len4_executor_target_mass": NaN, "paired_len4_init_accuracy": NaN, "paired_len4_op_accuracy": NaN, "paired_len4_arg_accuracy": NaN, "paired_len4_program_exact": NaN, "paired_len4_state_accuracy": NaN, "paired_len4_state_all_exact": NaN, "paired_len4_state_prefix_fraction": NaN, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_direct_pair_answer_consistency": 0.75, "paired_len4_direct_pair_both_correct": 0.0, "paired_len8_direct_accuracy": 0.0234375, "paired_len8_direct_target_mass": 0.010385037050582469, "paired_len8_executor_accuracy": NaN, "paired_len8_executor_target_mass": NaN, "paired_len8_init_accuracy": NaN, "paired_len8_op_accuracy": NaN, "paired_len8_arg_accuracy": NaN, "paired_len8_program_exact": NaN, "paired_len8_state_accuracy": NaN, "paired_len8_state_all_exact": NaN, "paired_len8_state_prefix_fraction": NaN, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_direct_pair_answer_consistency": 0.734375, "paired_len8_direct_pair_both_correct": 0.015625, "paired_len12_direct_accuracy": 0.0078125, "paired_len12_direct_target_mass": 0.010441710939630866, "paired_len12_executor_accuracy": NaN, "paired_len12_executor_target_mass": NaN, "paired_len12_init_accuracy": NaN, "paired_len12_op_accuracy": NaN, "paired_len12_arg_accuracy": NaN, "paired_len12_program_exact": NaN, "paired_len12_state_accuracy": NaN, "paired_len12_state_all_exact": NaN, "paired_len12_state_prefix_fraction": NaN, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_direct_pair_answer_consistency": 0.609375, "paired_len12_direct_pair_both_correct": 0.0, "paired_len24_direct_accuracy": 0.0234375, "paired_len24_direct_target_mass": 0.010658582847099751, "paired_len24_executor_accuracy": NaN, "paired_len24_executor_target_mass": NaN, "paired_len24_init_accuracy": NaN, "paired_len24_op_accuracy": NaN, "paired_len24_arg_accuracy": NaN, "paired_len24_program_exact": NaN, "paired_len24_state_accuracy": NaN, "paired_len24_state_all_exact": NaN, "paired_len24_state_prefix_fraction": NaN, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_direct_pair_answer_consistency": 0.6875, "paired_len24_direct_pair_both_correct": 0.015625 }, { "variant": "direct", "stage": "answer_only", "local_step": 100, "step": 100, "loss": 5.15815544128418, "trace_loss_active": false, "executor_loss_active": false, "state_loss_active": false, "lr_scale": 1.0, "direct_loss": 5.15815544128418, "standard_len4_direct_accuracy": 0.015625, "standard_len4_direct_target_mass": 0.010356259299442172, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len4_state_accuracy": NaN, "standard_len4_state_all_exact": NaN, "standard_len4_state_prefix_fraction": NaN, "standard_len8_direct_accuracy": 0.046875, "standard_len8_direct_target_mass": 0.011637821095064282, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len8_state_accuracy": NaN, "standard_len8_state_all_exact": NaN, "standard_len8_state_prefix_fraction": NaN, "standard_len12_direct_accuracy": 0.0, "standard_len12_direct_target_mass": 0.009690985432825983, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len12_state_accuracy": NaN, "standard_len12_state_all_exact": NaN, "standard_len12_state_prefix_fraction": NaN, "standard_len24_direct_accuracy": 0.015625, "standard_len24_direct_target_mass": 0.011303443112410605, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "standard_len24_state_accuracy": NaN, "standard_len24_state_all_exact": NaN, "standard_len24_state_prefix_fraction": NaN, "paraphrase_len4_direct_accuracy": 0.0, "paraphrase_len4_direct_target_mass": 0.009024781233165413, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len4_state_accuracy": NaN, "paraphrase_len4_state_all_exact": NaN, "paraphrase_len4_state_prefix_fraction": NaN, "paraphrase_len8_direct_accuracy": 0.0, "paraphrase_len8_direct_target_mass": 0.00999791722279042, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len8_state_accuracy": NaN, "paraphrase_len8_state_all_exact": NaN, "paraphrase_len8_state_prefix_fraction": NaN, "paraphrase_len12_direct_accuracy": 0.015625, "paraphrase_len12_direct_target_mass": 0.010176196752581745, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len12_state_accuracy": NaN, "paraphrase_len12_state_all_exact": NaN, "paraphrase_len12_state_prefix_fraction": NaN, "paraphrase_len24_direct_accuracy": 0.015625, "paraphrase_len24_direct_target_mass": 0.010937099403236061, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN, "paraphrase_len24_state_accuracy": NaN, "paraphrase_len24_state_all_exact": NaN, "paraphrase_len24_state_prefix_fraction": NaN, "paired_len4_direct_accuracy": 0.015625, "paired_len4_direct_target_mass": 0.012035822961479425, "paired_len4_executor_accuracy": NaN, "paired_len4_executor_target_mass": NaN, "paired_len4_init_accuracy": NaN, "paired_len4_op_accuracy": NaN, "paired_len4_arg_accuracy": NaN, "paired_len4_program_exact": NaN, "paired_len4_state_accuracy": NaN, "paired_len4_state_all_exact": NaN, "paired_len4_state_prefix_fraction": NaN, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_direct_pair_answer_consistency": 0.21875, "paired_len4_direct_pair_both_correct": 0.0, "paired_len8_direct_accuracy": 0.0234375, "paired_len8_direct_target_mass": 0.011016997508704662, "paired_len8_executor_accuracy": NaN, "paired_len8_executor_target_mass": NaN, "paired_len8_init_accuracy": NaN, "paired_len8_op_accuracy": NaN, "paired_len8_arg_accuracy": NaN, "paired_len8_program_exact": NaN, "paired_len8_state_accuracy": NaN, "paired_len8_state_all_exact": NaN, "paired_len8_state_prefix_fraction": NaN, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_direct_pair_answer_consistency": 0.21875, "paired_len8_direct_pair_both_correct": 0.0, "paired_len12_direct_accuracy": 0.0, "paired_len12_direct_target_mass": 0.010981235624058172, "paired_len12_executor_accuracy": NaN, "paired_len12_executor_target_mass": NaN, "paired_len12_init_accuracy": NaN, "paired_len12_op_accuracy": NaN, "paired_len12_arg_accuracy": NaN, "paired_len12_program_exact": NaN, "paired_len12_state_accuracy": NaN, "paired_len12_state_all_exact": NaN, "paired_len12_state_prefix_fraction": NaN, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_direct_pair_answer_consistency": 0.328125, "paired_len12_direct_pair_both_correct": 0.0, "paired_len24_direct_accuracy": 0.0078125, "paired_len24_direct_target_mass": 0.009453960112296045, "paired_len24_executor_accuracy": NaN, "paired_len24_executor_target_mass": NaN, "paired_len24_init_accuracy": NaN, "paired_len24_op_accuracy": NaN, "paired_len24_arg_accuracy": NaN, "paired_len24_program_exact": NaN, "paired_len24_state_accuracy": NaN, "paired_len24_state_all_exact": NaN, "paired_len24_state_prefix_fraction": NaN, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_direct_pair_answer_consistency": 0.1875, "paired_len24_direct_pair_both_correct": 0.0 }, { "variant": "direct", "stage": "answer_only", "local_step": 200, "step": 200, "loss": 4.489184379577637, "trace_loss_active": false, "executor_loss_active": false, "state_loss_active": false, "lr_scale": 1.0, "direct_loss": 4.489184379577637, "standard_len4_direct_accuracy": 0.0, "standard_len4_direct_target_mass": 0.011436699714977294, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len4_state_accuracy": NaN, "standard_len4_state_all_exact": NaN, "standard_len4_state_prefix_fraction": NaN, "standard_len8_direct_accuracy": 0.0, "standard_len8_direct_target_mass": 0.010816301917657256, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len8_state_accuracy": NaN, "standard_len8_state_all_exact": NaN, "standard_len8_state_prefix_fraction": NaN, "standard_len12_direct_accuracy": 0.015625, "standard_len12_direct_target_mass": 0.008875600178726017, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len12_state_accuracy": NaN, "standard_len12_state_all_exact": NaN, "standard_len12_state_prefix_fraction": NaN, "standard_len24_direct_accuracy": 0.03125, "standard_len24_direct_target_mass": 0.012563865166157484, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "standard_len24_state_accuracy": NaN, "standard_len24_state_all_exact": NaN, "standard_len24_state_prefix_fraction": NaN, "paraphrase_len4_direct_accuracy": 0.03125, "paraphrase_len4_direct_target_mass": 0.0094730380224064, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len4_state_accuracy": NaN, "paraphrase_len4_state_all_exact": NaN, "paraphrase_len4_state_prefix_fraction": NaN, "paraphrase_len8_direct_accuracy": 0.0, "paraphrase_len8_direct_target_mass": 0.009604912949725986, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len8_state_accuracy": NaN, "paraphrase_len8_state_all_exact": NaN, "paraphrase_len8_state_prefix_fraction": NaN, "paraphrase_len12_direct_accuracy": 0.015625, "paraphrase_len12_direct_target_mass": 0.010256750218104571, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len12_state_accuracy": NaN, "paraphrase_len12_state_all_exact": NaN, "paraphrase_len12_state_prefix_fraction": NaN, "paraphrase_len24_direct_accuracy": 0.0, "paraphrase_len24_direct_target_mass": 0.010861504706554115, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN, "paraphrase_len24_state_accuracy": NaN, "paraphrase_len24_state_all_exact": NaN, "paraphrase_len24_state_prefix_fraction": NaN, "paired_len4_direct_accuracy": 0.0078125, "paired_len4_direct_target_mass": 0.011512628785567358, "paired_len4_executor_accuracy": NaN, "paired_len4_executor_target_mass": NaN, "paired_len4_init_accuracy": NaN, "paired_len4_op_accuracy": NaN, "paired_len4_arg_accuracy": NaN, "paired_len4_program_exact": NaN, "paired_len4_state_accuracy": NaN, "paired_len4_state_all_exact": NaN, "paired_len4_state_prefix_fraction": NaN, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_direct_pair_answer_consistency": 0.0, "paired_len4_direct_pair_both_correct": 0.0, "paired_len8_direct_accuracy": 0.015625, "paired_len8_direct_target_mass": 0.01012100515072234, "paired_len8_executor_accuracy": NaN, "paired_len8_executor_target_mass": NaN, "paired_len8_init_accuracy": NaN, "paired_len8_op_accuracy": NaN, "paired_len8_arg_accuracy": NaN, "paired_len8_program_exact": NaN, "paired_len8_state_accuracy": NaN, "paired_len8_state_all_exact": NaN, "paired_len8_state_prefix_fraction": NaN, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_direct_pair_answer_consistency": 0.0, "paired_len8_direct_pair_both_correct": 0.0, "paired_len12_direct_accuracy": 0.015625, "paired_len12_direct_target_mass": 0.011790893171564676, "paired_len12_executor_accuracy": NaN, "paired_len12_executor_target_mass": NaN, "paired_len12_init_accuracy": NaN, "paired_len12_op_accuracy": NaN, "paired_len12_arg_accuracy": NaN, "paired_len12_program_exact": NaN, "paired_len12_state_accuracy": NaN, "paired_len12_state_all_exact": NaN, "paired_len12_state_prefix_fraction": NaN, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_direct_pair_answer_consistency": 0.0, "paired_len12_direct_pair_both_correct": 0.0, "paired_len24_direct_accuracy": 0.0078125, "paired_len24_direct_target_mass": 0.009576360636856407, "paired_len24_executor_accuracy": NaN, "paired_len24_executor_target_mass": NaN, "paired_len24_init_accuracy": NaN, "paired_len24_op_accuracy": NaN, "paired_len24_arg_accuracy": NaN, "paired_len24_program_exact": NaN, "paired_len24_state_accuracy": NaN, "paired_len24_state_all_exact": NaN, "paired_len24_state_prefix_fraction": NaN, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_direct_pair_answer_consistency": 0.0, "paired_len24_direct_pair_both_correct": 0.0 }, { "variant": "direct", "stage": "answer_only", "local_step": 300, "step": 300, "loss": 3.871479034423828, "trace_loss_active": false, "executor_loss_active": false, "state_loss_active": false, "lr_scale": 1.0, "direct_loss": 3.871479034423828, "standard_len4_direct_accuracy": 0.015625, "standard_len4_direct_target_mass": 0.009979097987525165, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len4_state_accuracy": NaN, "standard_len4_state_all_exact": NaN, "standard_len4_state_prefix_fraction": NaN, "standard_len8_direct_accuracy": 0.015625, "standard_len8_direct_target_mass": 0.010282560833729804, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len8_state_accuracy": NaN, "standard_len8_state_all_exact": NaN, "standard_len8_state_prefix_fraction": NaN, "standard_len12_direct_accuracy": 0.015625, "standard_len12_direct_target_mass": 0.01085659972159192, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len12_state_accuracy": NaN, "standard_len12_state_all_exact": NaN, "standard_len12_state_prefix_fraction": NaN, "standard_len24_direct_accuracy": 0.03125, "standard_len24_direct_target_mass": 0.012475729861762375, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "standard_len24_state_accuracy": NaN, "standard_len24_state_all_exact": NaN, "standard_len24_state_prefix_fraction": NaN, "paraphrase_len4_direct_accuracy": 0.0, "paraphrase_len4_direct_target_mass": 0.009030730987433344, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len4_state_accuracy": NaN, "paraphrase_len4_state_all_exact": NaN, "paraphrase_len4_state_prefix_fraction": NaN, "paraphrase_len8_direct_accuracy": 0.0, "paraphrase_len8_direct_target_mass": 0.009098326787352562, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len8_state_accuracy": NaN, "paraphrase_len8_state_all_exact": NaN, "paraphrase_len8_state_prefix_fraction": NaN, "paraphrase_len12_direct_accuracy": 0.0, "paraphrase_len12_direct_target_mass": 0.009421012888196856, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len12_state_accuracy": NaN, "paraphrase_len12_state_all_exact": NaN, "paraphrase_len12_state_prefix_fraction": NaN, "paraphrase_len24_direct_accuracy": 0.0, "paraphrase_len24_direct_target_mass": 0.009803309803828597, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN, "paraphrase_len24_state_accuracy": NaN, "paraphrase_len24_state_all_exact": NaN, "paraphrase_len24_state_prefix_fraction": NaN, "paired_len4_direct_accuracy": 0.0078125, "paired_len4_direct_target_mass": 0.011186226387508214, "paired_len4_executor_accuracy": NaN, "paired_len4_executor_target_mass": NaN, "paired_len4_init_accuracy": NaN, "paired_len4_op_accuracy": NaN, "paired_len4_arg_accuracy": NaN, "paired_len4_program_exact": NaN, "paired_len4_state_accuracy": NaN, "paired_len4_state_all_exact": NaN, "paired_len4_state_prefix_fraction": NaN, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_direct_pair_answer_consistency": 0.0, "paired_len4_direct_pair_both_correct": 0.0, "paired_len8_direct_accuracy": 0.0, "paired_len8_direct_target_mass": 0.0106004418048542, "paired_len8_executor_accuracy": NaN, "paired_len8_executor_target_mass": NaN, "paired_len8_init_accuracy": NaN, "paired_len8_op_accuracy": NaN, "paired_len8_arg_accuracy": NaN, "paired_len8_program_exact": NaN, "paired_len8_state_accuracy": NaN, "paired_len8_state_all_exact": NaN, "paired_len8_state_prefix_fraction": NaN, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_direct_pair_answer_consistency": 0.0, "paired_len8_direct_pair_both_correct": 0.0, "paired_len12_direct_accuracy": 0.015625, "paired_len12_direct_target_mass": 0.0123308481706772, "paired_len12_executor_accuracy": NaN, "paired_len12_executor_target_mass": NaN, "paired_len12_init_accuracy": NaN, "paired_len12_op_accuracy": NaN, "paired_len12_arg_accuracy": NaN, "paired_len12_program_exact": NaN, "paired_len12_state_accuracy": NaN, "paired_len12_state_all_exact": NaN, "paired_len12_state_prefix_fraction": NaN, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_direct_pair_answer_consistency": 0.0, "paired_len12_direct_pair_both_correct": 0.0, "paired_len24_direct_accuracy": 0.015625, "paired_len24_direct_target_mass": 0.00973344044177793, "paired_len24_executor_accuracy": NaN, "paired_len24_executor_target_mass": NaN, "paired_len24_init_accuracy": NaN, "paired_len24_op_accuracy": NaN, "paired_len24_arg_accuracy": NaN, "paired_len24_program_exact": NaN, "paired_len24_state_accuracy": NaN, "paired_len24_state_all_exact": NaN, "paired_len24_state_prefix_fraction": NaN, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_direct_pair_answer_consistency": 0.0, "paired_len24_direct_pair_both_correct": 0.0 } ], "final_metrics": { "standard_len4": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.015625, "direct_target_mass": 0.009979097987525165, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "standard_len8": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.015625, "direct_target_mass": 0.010282560833729804, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "standard_len12": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.015625, "direct_target_mass": 0.01085659972159192, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "standard_len24": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.03125, "direct_target_mass": 0.012475729861762375, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "paraphrase_len4": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.0, "direct_target_mass": 0.009030730987433344, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "paraphrase_len8": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.0, "direct_target_mass": 0.009098326787352562, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "paraphrase_len12": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.0, "direct_target_mass": 0.009421012888196856, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "paraphrase_len24": { "variant": "direct", "n": 64.0, "direct_accuracy": 0.0, "direct_target_mass": 0.009803309803828597, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN }, "paired_len4": { "variant": "direct", "n": 128.0, "direct_accuracy": 0.0078125, "direct_target_mass": 0.011186226387508214, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN, "pair_true_answer_consistency": 1.0, "direct_pair_answer_consistency": 0.0, "direct_pair_both_correct": 0.0 }, "paired_len8": { "variant": "direct", "n": 128.0, "direct_accuracy": 0.0, "direct_target_mass": 0.0106004418048542, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN, "pair_true_answer_consistency": 1.0, "direct_pair_answer_consistency": 0.0, "direct_pair_both_correct": 0.0 }, "paired_len12": { "variant": "direct", "n": 128.0, "direct_accuracy": 0.015625, "direct_target_mass": 0.0123308481706772, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN, "pair_true_answer_consistency": 1.0, "direct_pair_answer_consistency": 0.0, "direct_pair_both_correct": 0.0 }, "paired_len24": { "variant": "direct", "n": 128.0, "direct_accuracy": 0.015625, "direct_target_mass": 0.00973344044177793, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN, "state_accuracy": NaN, "state_all_exact": NaN, "state_prefix_fraction": NaN, "pair_true_answer_consistency": 1.0, "direct_pair_answer_consistency": 0.0, "direct_pair_both_correct": 0.0 } } } } }