{ "args": { "model_id": "Qwen/Qwen3-4B", "load_in_4bit": true, "torch_dtype": "bf16", "device_map": "auto", "use_lora": true, "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "lora_target_modules": "all-linear", "gradient_checkpointing": true, "optimizer": "paged_adamw_8bit", "modulus": 97, "max_steps": 24, "train_min_len": 1, "train_max_len": 12, "curriculum_stages": "short:1:4:150,medium:1:8:150,train:1:12:150,long:8:24:150", "answer_train_min_len": 1, "answer_train_max_len": 24, "eval_lengths": "4,8,12,24", "train_template_mode": "mixed", "paired_train": true, "paired_batches": true, "paired_template_modes": "standard,paraphrase", "pair_size": 2, "answer_train_template_mode": "mixed", "eval_template_modes": "standard,paraphrase", "paired_eval": true, "paired_eval_template_modes": "standard,paraphrase", "train_size": 512, "answer_train_size": 512, "eval_size": 128, "train_batch_size": 4, "eval_batch_size": 8, "bootstrap_steps": 0, "answer_steps": 0, "eval_every": 150, "stage_eval_every": 150, "lr": 0.0002, "answer_lr_scale": 0.1, "weight_decay": 0.0, "grad_clip": 1.0, "head_width": 512, "register_width": 512, "register_layers": 1, "register_heads": 4, "register_dropout": 0.05, "register_style": "bare", "trace_loss_weight": 1.0, "init_trace_loss_weight": 4.0, "op_trace_loss_weight": 1.0, "arg_trace_loss_weight": 4.0, "executor_loss_weight": 1.0, "state_loss_weight": 0.25, "variants": "register_trace", "max_length": 768, "seed": 7, "train_seed": -1, "output_dir": "experiments/qwen_register_token_latent_compiler/runs/main_register_trace_s600", "checkpoint_dir": "large_artifacts/qwen_register_token_latent_compiler/checkpoints/main_register_trace_s600" }, "metadata": { "python": "3.12.3", "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "torch": "2.8.0+cu128", "cuda_available": true, "transformers_loader": "AutoModelForCausalLM", "peft_installed": true, "use_lora": true, "lora_r": 8, "gpu_name": "NVIDIA RTX 6000 Ada Generation", "gpu_vram_gb": 47.383 }, "hidden_dim": 2560, "register_count": 49, "dataset": { "train_size": 512, "train_examples": 4096, "curriculum_stages": [ { "name": "short", "min_len": 1, "max_len": 4, "steps": 150 }, { "name": "medium", "min_len": 1, "max_len": 8, "steps": 150 }, { "name": "train", "min_len": 1, "max_len": 12, "steps": 150 }, { "name": "long", "min_len": 8, "max_len": 24, "steps": 150 } ], "answer_train_size": 512, "eval_size": 128, "eval_lengths": [ 4, 8, 12, 24 ], "train_template_mode": "mixed", "paired_train": true, "paired_template_modes": [ "standard", "paraphrase" ], "answer_train_template_mode": "mixed", "eval_template_modes": [ "standard", "paraphrase" ], "paired_eval": true, "paired_eval_template_modes": [ "standard", "paraphrase" ], "register_style": "bare" }, "variants": { "register_trace": { "variant": "register_trace", "train_seconds": 2387.9672203063965, "checkpoints": [ "large_artifacts/qwen_register_token_latent_compiler/checkpoints/main_register_trace_s600/register_trace/adapter", "large_artifacts/qwen_register_token_latent_compiler/checkpoints/main_register_trace_s600/register_trace/heads.pt" ], "train_log": [ { "variant": "register_trace", "stage": "trace_short", "local_step": 1, "step": 1, "loss": 42.18625259399414, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.579524040222168, "init_loss": 4.5355401039123535, "op_loss": 1.0820157527923584, "arg_loss": 4.595637798309326, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0234375, "standard_len4_executor_target_mass": 0.010299774701707065, "standard_len4_init_accuracy": 0.0, "standard_len4_op_accuracy": 0.30859375, "standard_len4_arg_accuracy": 0.01953125, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.021484375, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.00390625, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0078125, "standard_len8_executor_target_mass": 0.010329711018130183, "standard_len8_init_accuracy": 0.0, "standard_len8_op_accuracy": 0.3310546875, "standard_len8_arg_accuracy": 0.01953125, "standard_len8_program_exact": 0.0, "standard_len8_state_accuracy": 0.0087890625, "standard_len8_state_all_exact": 0.0, "standard_len8_state_prefix_fraction": 0.0009765625, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.015625, "standard_len12_executor_target_mass": 0.010269791586324573, "standard_len12_init_accuracy": 0.0, "standard_len12_op_accuracy": 0.3255208333333333, "standard_len12_arg_accuracy": 0.018229166666666668, "standard_len12_program_exact": 0.0, "standard_len12_state_accuracy": 0.01171875, "standard_len12_state_all_exact": 0.0, "standard_len12_state_prefix_fraction": 0.0013020833333333333, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.03125, "standard_len24_executor_target_mass": 0.010274004249367863, "standard_len24_init_accuracy": 0.0078125, "standard_len24_op_accuracy": 0.3401692708333333, "standard_len24_arg_accuracy": 0.020182291666666668, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.017578125, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.0006510416666666666, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.015625, "paraphrase_len4_executor_target_mass": 0.010270505910739303, "paraphrase_len4_init_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.369140625, "paraphrase_len4_arg_accuracy": 0.013671875, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.01171875, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.005859375, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.015625, "paraphrase_len8_executor_target_mass": 0.010328435397241265, "paraphrase_len8_init_accuracy": 0.0, "paraphrase_len8_op_accuracy": 0.3408203125, "paraphrase_len8_arg_accuracy": 0.0185546875, "paraphrase_len8_program_exact": 0.0, "paraphrase_len8_state_accuracy": 0.01171875, "paraphrase_len8_state_all_exact": 0.0, "paraphrase_len8_state_prefix_fraction": 0.0029296875, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0, "paraphrase_len12_executor_target_mass": 0.010358243016526103, "paraphrase_len12_init_accuracy": 0.0078125, "paraphrase_len12_op_accuracy": 0.3287760416666667, "paraphrase_len12_arg_accuracy": 0.018880208333333332, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.014973958333333334, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.001953125, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0078125, "paraphrase_len24_executor_target_mass": 0.010301095608156174, "paraphrase_len24_init_accuracy": 0.0078125, "paraphrase_len24_op_accuracy": 0.3388671875, "paraphrase_len24_arg_accuracy": 0.015950520833333332, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.0107421875, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.0006510416666666666, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.01171875, "paired_len4_executor_target_mass": 0.01027072046417743, "paired_len4_init_accuracy": 0.0078125, "paired_len4_op_accuracy": 0.3271484375, "paired_len4_arg_accuracy": 0.0166015625, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.0048828125, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.1171875, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.0859375, "paired_len4_compiler_pair_state_consistency": 0.0859375, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.01171875, "paired_len8_executor_target_mass": 0.01030003686901182, "paired_len8_init_accuracy": 0.0, "paired_len8_op_accuracy": 0.32177734375, "paired_len8_arg_accuracy": 0.01318359375, "paired_len8_program_exact": 0.0, "paired_len8_state_accuracy": 0.01123046875, "paired_len8_state_all_exact": 0.0, "paired_len8_state_prefix_fraction": 0.0029296875, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.03125, "paired_len8_executor_pair_both_correct": 0.0, "paired_len8_compiler_pair_program_consistency": 0.0, "paired_len8_compiler_pair_state_consistency": 0.0, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.01171875, "paired_len12_executor_target_mass": 0.010358882951550186, "paired_len12_init_accuracy": 0.0078125, "paired_len12_op_accuracy": 0.3466796875, "paired_len12_arg_accuracy": 0.018229166666666668, "paired_len12_program_exact": 0.0, "paired_len12_state_accuracy": 0.008463541666666666, "paired_len12_state_all_exact": 0.0, "paired_len12_state_prefix_fraction": 0.0013020833333333333, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.0234375, "paired_len12_executor_pair_both_correct": 0.0, "paired_len12_compiler_pair_program_consistency": 0.0, "paired_len12_compiler_pair_state_consistency": 0.0, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.01953125, "paired_len24_executor_target_mass": 0.010299441666575149, "paired_len24_init_accuracy": 0.015625, "paired_len24_op_accuracy": 0.33349609375, "paired_len24_arg_accuracy": 0.015787760416666668, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.01123046875, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.0, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "register_trace", "stage": "trace_short", "local_step": 150, "step": 150, "loss": 31.4908447265625, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.587450981140137, "init_loss": 4.909481048583984, "op_loss": 0.006313145160675049, "arg_loss": 1.8147892951965332, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.010281934635713696, "standard_len4_init_accuracy": 0.015625, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.638671875, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.00390625, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.001953125, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0078125, "standard_len8_executor_target_mass": 0.01030551182338968, "standard_len8_init_accuracy": 0.0078125, "standard_len8_op_accuracy": 0.8974609375, "standard_len8_arg_accuracy": 0.5048828125, "standard_len8_program_exact": 0.0, "standard_len8_state_accuracy": 0.0068359375, "standard_len8_state_all_exact": 0.0, "standard_len8_state_prefix_fraction": 0.001953125, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0, "standard_len12_executor_target_mass": 0.010307014104910195, "standard_len12_init_accuracy": 0.0, "standard_len12_op_accuracy": 0.7838541666666666, "standard_len12_arg_accuracy": 0.3776041666666667, "standard_len12_program_exact": 0.0, "standard_len12_state_accuracy": 0.013020833333333334, "standard_len12_state_all_exact": 0.0, "standard_len12_state_prefix_fraction": 0.001953125, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.015625, "standard_len24_executor_target_mass": 0.010308440017979592, "standard_len24_init_accuracy": 0.0078125, "standard_len24_op_accuracy": 0.5846354166666666, "standard_len24_arg_accuracy": 0.18717447916666666, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.0107421875, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.0003255208333333333, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.010258465772494674, "paraphrase_len4_init_accuracy": 0.0078125, "paraphrase_len4_op_accuracy": 0.986328125, "paraphrase_len4_arg_accuracy": 0.599609375, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.009765625, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.00390625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0078125, "paraphrase_len8_executor_target_mass": 0.010324982926249504, "paraphrase_len8_init_accuracy": 0.015625, "paraphrase_len8_op_accuracy": 0.775390625, "paraphrase_len8_arg_accuracy": 0.427734375, "paraphrase_len8_program_exact": 0.0, "paraphrase_len8_state_accuracy": 0.013671875, "paraphrase_len8_state_all_exact": 0.0, "paraphrase_len8_state_prefix_fraction": 0.0009765625, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0, "paraphrase_len12_executor_target_mass": 0.010310667741578072, "paraphrase_len12_init_accuracy": 0.0, "paraphrase_len12_op_accuracy": 0.6510416666666666, "paraphrase_len12_arg_accuracy": 0.2884114583333333, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.006510416666666667, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.015625, "paraphrase_len24_executor_target_mass": 0.01030957413604483, "paraphrase_len24_init_accuracy": 0.015625, "paraphrase_len24_op_accuracy": 0.4921875, "paraphrase_len24_arg_accuracy": 0.169921875, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.012044270833333334, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.0022786458333333335, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.00390625, "paired_len4_executor_target_mass": 0.010267013072734699, "paired_len4_init_accuracy": 0.0, "paired_len4_op_accuracy": 0.9912109375, "paired_len4_arg_accuracy": 0.609375, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.0048828125, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.828125, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.8203125, "paired_len4_compiler_pair_state_consistency": 0.828125, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.00390625, "paired_len8_executor_target_mass": 0.010307849617674947, "paired_len8_init_accuracy": 0.0, "paired_len8_op_accuracy": 0.84619140625, "paired_len8_arg_accuracy": 0.478515625, "paired_len8_program_exact": 0.0, "paired_len8_state_accuracy": 0.0087890625, "paired_len8_state_all_exact": 0.0, "paired_len8_state_prefix_fraction": 0.00439453125, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.0546875, "paired_len8_executor_pair_both_correct": 0.0, "paired_len8_compiler_pair_program_consistency": 0.03125, "paired_len8_compiler_pair_state_consistency": 0.0390625, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.015625, "paired_len12_executor_target_mass": 0.01031262899050489, "paired_len12_init_accuracy": 0.015625, "paired_len12_op_accuracy": 0.7083333333333334, "paired_len12_arg_accuracy": 0.3411458333333333, "paired_len12_program_exact": 0.0, "paired_len12_state_accuracy": 0.008463541666666666, "paired_len12_state_all_exact": 0.0, "paired_len12_state_prefix_fraction": 0.0026041666666666665, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.0078125, "paired_len12_executor_pair_both_correct": 0.0, "paired_len12_compiler_pair_program_consistency": 0.0, "paired_len12_compiler_pair_state_consistency": 0.0, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.01171875, "paired_len24_executor_target_mass": 0.010308486293070018, "paired_len24_init_accuracy": 0.015625, "paired_len24_op_accuracy": 0.5442708333333334, "paired_len24_arg_accuracy": 0.18196614583333334, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.010904947916666666, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.0006510416666666666, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0078125, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "register_trace", "stage": "trace_medium", "local_step": 1, "step": 151, "loss": 32.91754913330078, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.587376117706299, "init_loss": 4.9424519538879395, "op_loss": 0.5202432870864868, "arg_loss": 2.01003098487854, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.010282635281328112, "standard_len4_init_accuracy": 0.015625, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.638671875, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.00390625, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.001953125, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0, "standard_len8_executor_target_mass": 0.010306943615432829, "standard_len8_init_accuracy": 0.0078125, "standard_len8_op_accuracy": 0.9580078125, "standard_len8_arg_accuracy": 0.482421875, "standard_len8_program_exact": 0.0, "standard_len8_state_accuracy": 0.00390625, "standard_len8_state_all_exact": 0.0, "standard_len8_state_prefix_fraction": 0.001953125, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0234375, "standard_len12_executor_target_mass": 0.010308144148439169, "standard_len12_init_accuracy": 0.0, "standard_len12_op_accuracy": 0.8053385416666666, "standard_len12_arg_accuracy": 0.3463541666666667, "standard_len12_program_exact": 0.0, "standard_len12_state_accuracy": 0.008463541666666666, "standard_len12_state_all_exact": 0.0, "standard_len12_state_prefix_fraction": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0234375, "standard_len24_executor_target_mass": 0.010308699798770249, "standard_len24_init_accuracy": 0.0078125, "standard_len24_op_accuracy": 0.5872395833333334, "standard_len24_arg_accuracy": 0.1748046875, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.009765625, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.010277913592290133, "paraphrase_len4_init_accuracy": 0.0078125, "paraphrase_len4_op_accuracy": 0.990234375, "paraphrase_len4_arg_accuracy": 0.60546875, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.009765625, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.00390625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0234375, "paraphrase_len8_executor_target_mass": 0.010309994162525982, "paraphrase_len8_init_accuracy": 0.015625, "paraphrase_len8_op_accuracy": 0.763671875, "paraphrase_len8_arg_accuracy": 0.3779296875, "paraphrase_len8_program_exact": 0.0, "paraphrase_len8_state_accuracy": 0.01171875, "paraphrase_len8_state_all_exact": 0.0, "paraphrase_len8_state_prefix_fraction": 0.0009765625, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0, "paraphrase_len12_executor_target_mass": 0.010309726931154728, "paraphrase_len12_init_accuracy": 0.0, "paraphrase_len12_op_accuracy": 0.6458333333333334, "paraphrase_len12_arg_accuracy": 0.2610677083333333, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.004557291666666667, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0078125, "paraphrase_len24_executor_target_mass": 0.010309382632840425, "paraphrase_len24_init_accuracy": 0.015625, "paraphrase_len24_op_accuracy": 0.4677734375, "paraphrase_len24_arg_accuracy": 0.14518229166666666, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.010091145833333334, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.0016276041666666667, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.00390625, "paired_len4_executor_target_mass": 0.010270800034049898, "paired_len4_init_accuracy": 0.0, "paired_len4_op_accuracy": 0.994140625, "paired_len4_arg_accuracy": 0.607421875, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.005859375, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.828125, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.828125, "paired_len4_compiler_pair_state_consistency": 0.828125, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.015625, "paired_len8_executor_target_mass": 0.010308790049748495, "paired_len8_init_accuracy": 0.0, "paired_len8_op_accuracy": 0.87548828125, "paired_len8_arg_accuracy": 0.4560546875, "paired_len8_program_exact": 0.0, "paired_len8_state_accuracy": 0.01220703125, "paired_len8_state_all_exact": 0.0, "paired_len8_state_prefix_fraction": 0.00439453125, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.0234375, "paired_len8_executor_pair_both_correct": 0.0, "paired_len8_compiler_pair_program_consistency": 0.015625, "paired_len8_compiler_pair_state_consistency": 0.015625, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.0078125, "paired_len12_executor_target_mass": 0.01031108308234252, "paired_len12_init_accuracy": 0.015625, "paired_len12_op_accuracy": 0.7265625, "paired_len12_arg_accuracy": 0.3063151041666667, "paired_len12_program_exact": 0.0, "paired_len12_state_accuracy": 0.008138020833333334, "paired_len12_state_all_exact": 0.0, "paired_len12_state_prefix_fraction": 0.0026041666666666665, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.0234375, "paired_len12_executor_pair_both_correct": 0.0, "paired_len12_compiler_pair_program_consistency": 0.0, "paired_len12_compiler_pair_state_consistency": 0.0, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.0078125, "paired_len24_executor_target_mass": 0.010308730037650093, "paired_len24_init_accuracy": 0.015625, "paired_len24_op_accuracy": 0.5262044270833334, "paired_len24_arg_accuracy": 0.16048177083333334, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.009114583333333334, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.0006510416666666666, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0078125, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "register_trace", "stage": "trace_medium", "local_step": 150, "step": 300, "loss": 19.426855087280273, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 3.9105799198150635, "init_loss": 3.8182146549224854, "op_loss": 0.005390385165810585, "arg_loss": 0.05950632318854332, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0234375, "standard_len4_executor_target_mass": 0.011497442348627374, "standard_len4_init_accuracy": 0.0234375, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 1.0, "standard_len4_program_exact": 0.0234375, "standard_len4_state_accuracy": 0.0234375, "standard_len4_state_all_exact": 0.0234375, "standard_len4_state_prefix_fraction": 0.0234375, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0078125, "standard_len8_executor_target_mass": 0.010093690070789307, "standard_len8_init_accuracy": 0.0078125, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 1.0, "standard_len8_program_exact": 0.0078125, "standard_len8_state_accuracy": 0.0078125, "standard_len8_state_all_exact": 0.0078125, "standard_len8_state_prefix_fraction": 0.0078125, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0078125, "standard_len12_executor_target_mass": 0.010417714831419289, "standard_len12_init_accuracy": 0.0078125, "standard_len12_op_accuracy": 0.9947916666666666, "standard_len12_arg_accuracy": 0.9954427083333334, "standard_len12_program_exact": 0.0078125, "standard_len12_state_accuracy": 0.0078125, "standard_len12_state_all_exact": 0.0078125, "standard_len12_state_prefix_fraction": 0.0078125, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0078125, "standard_len24_executor_target_mass": 0.010309025412425399, "standard_len24_init_accuracy": 0.0234375, "standard_len24_op_accuracy": 0.7867838541666666, "standard_len24_arg_accuracy": 0.6725260416666666, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.019856770833333332, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.012044270833333334, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.015625, "paraphrase_len4_executor_target_mass": 0.010869259713217616, "paraphrase_len4_init_accuracy": 0.015625, "paraphrase_len4_op_accuracy": 1.0, "paraphrase_len4_arg_accuracy": 1.0, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len4_state_accuracy": 0.015625, "paraphrase_len4_state_all_exact": 0.015625, "paraphrase_len4_state_prefix_fraction": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0078125, "paraphrase_len8_executor_target_mass": 0.010446342872455716, "paraphrase_len8_init_accuracy": 0.0078125, "paraphrase_len8_op_accuracy": 0.93359375, "paraphrase_len8_arg_accuracy": 0.919921875, "paraphrase_len8_program_exact": 0.0078125, "paraphrase_len8_state_accuracy": 0.0078125, "paraphrase_len8_state_all_exact": 0.0078125, "paraphrase_len8_state_prefix_fraction": 0.0078125, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0, "paraphrase_len12_executor_target_mass": 0.010309398756362498, "paraphrase_len12_init_accuracy": 0.0078125, "paraphrase_len12_op_accuracy": 0.8834635416666666, "paraphrase_len12_arg_accuracy": 0.7962239583333334, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.004557291666666667, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.0032552083333333335, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.015625, "paraphrase_len24_executor_target_mass": 0.01030927081592381, "paraphrase_len24_init_accuracy": 0.0, "paraphrase_len24_op_accuracy": 0.6136067708333334, "paraphrase_len24_arg_accuracy": 0.4300130208333333, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.0048828125, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.0, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.015625, "paired_len4_executor_target_mass": 0.010422767416457646, "paired_len4_init_accuracy": 0.015625, "paired_len4_op_accuracy": 1.0, "paired_len4_arg_accuracy": 1.0, "paired_len4_program_exact": 0.015625, "paired_len4_state_accuracy": 0.015625, "paired_len4_state_all_exact": 0.015625, "paired_len4_state_prefix_fraction": 0.015625, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.96875, "paired_len4_executor_pair_both_correct": 0.015625, "paired_len4_compiler_pair_program_consistency": 0.96875, "paired_len4_compiler_pair_state_consistency": 0.96875, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.0078125, "paired_len8_executor_target_mass": 0.01035861877608113, "paired_len8_init_accuracy": 0.01171875, "paired_len8_op_accuracy": 0.97802734375, "paired_len8_arg_accuracy": 0.9609375, "paired_len8_program_exact": 0.0078125, "paired_len8_state_accuracy": 0.01171875, "paired_len8_state_all_exact": 0.0078125, "paired_len8_state_prefix_fraction": 0.0107421875, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.6640625, "paired_len8_executor_pair_both_correct": 0.0078125, "paired_len8_compiler_pair_program_consistency": 0.6640625, "paired_len8_compiler_pair_state_consistency": 0.6640625, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.02734375, "paired_len12_executor_target_mass": 0.010467672866070643, "paired_len12_init_accuracy": 0.015625, "paired_len12_op_accuracy": 0.9397786458333334, "paired_len12_arg_accuracy": 0.9052734375, "paired_len12_program_exact": 0.015625, "paired_len12_state_accuracy": 0.018229166666666668, "paired_len12_state_all_exact": 0.015625, "paired_len12_state_prefix_fraction": 0.015625, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.2734375, "paired_len12_executor_pair_both_correct": 0.015625, "paired_len12_compiler_pair_program_consistency": 0.2578125, "paired_len12_compiler_pair_state_consistency": 0.2578125, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.00390625, "paired_len24_executor_target_mass": 0.010309123084880412, "paired_len24_init_accuracy": 0.0390625, "paired_len24_op_accuracy": 0.6998697916666666, "paired_len24_arg_accuracy": 0.5457356770833334, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.0224609375, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.015299479166666666, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "register_trace", "stage": "trace_train", "local_step": 1, "step": 301, "loss": 25.35785675048828, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.776772499084473, "init_loss": 5.0456695556640625, "op_loss": 0.021286576986312866, "arg_loss": 0.09428012371063232, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0234375, "standard_len4_executor_target_mass": 0.011575024924241006, "standard_len4_init_accuracy": 0.0234375, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 1.0, "standard_len4_program_exact": 0.0234375, "standard_len4_state_accuracy": 0.0234375, "standard_len4_state_all_exact": 0.0234375, "standard_len4_state_prefix_fraction": 0.0234375, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0078125, "standard_len8_executor_target_mass": 0.010061355074867606, "standard_len8_init_accuracy": 0.0078125, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 1.0, "standard_len8_program_exact": 0.0078125, "standard_len8_state_accuracy": 0.0078125, "standard_len8_state_all_exact": 0.0078125, "standard_len8_state_prefix_fraction": 0.0078125, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0078125, "standard_len12_executor_target_mass": 0.010453091585077345, "standard_len12_init_accuracy": 0.0078125, "standard_len12_op_accuracy": 0.9954427083333334, "standard_len12_arg_accuracy": 0.9954427083333334, "standard_len12_program_exact": 0.0078125, "standard_len12_state_accuracy": 0.0078125, "standard_len12_state_all_exact": 0.0078125, "standard_len12_state_prefix_fraction": 0.0078125, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0078125, "standard_len24_executor_target_mass": 0.010309020755812526, "standard_len24_init_accuracy": 0.0234375, "standard_len24_op_accuracy": 0.7900390625, "standard_len24_arg_accuracy": 0.6764322916666666, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.020833333333333332, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.012044270833333334, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0234375, "paraphrase_len4_executor_target_mass": 0.01095761836040765, "paraphrase_len4_init_accuracy": 0.0234375, "paraphrase_len4_op_accuracy": 1.0, "paraphrase_len4_arg_accuracy": 1.0, "paraphrase_len4_program_exact": 0.0234375, "paraphrase_len4_state_accuracy": 0.0234375, "paraphrase_len4_state_all_exact": 0.0234375, "paraphrase_len4_state_prefix_fraction": 0.0234375, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0, "paraphrase_len8_executor_target_mass": 0.010510866763070226, "paraphrase_len8_init_accuracy": 0.0078125, "paraphrase_len8_op_accuracy": 0.92578125, "paraphrase_len8_arg_accuracy": 0.91015625, "paraphrase_len8_program_exact": 0.0, "paraphrase_len8_state_accuracy": 0.005859375, "paraphrase_len8_state_all_exact": 0.0, "paraphrase_len8_state_prefix_fraction": 0.005859375, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.01031592779327184, "paraphrase_len12_init_accuracy": 0.0078125, "paraphrase_len12_op_accuracy": 0.8736979166666666, "paraphrase_len12_arg_accuracy": 0.78515625, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.005208333333333333, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.0032552083333333335, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0234375, "paraphrase_len24_executor_target_mass": 0.010309282632078975, "paraphrase_len24_init_accuracy": 0.0, "paraphrase_len24_op_accuracy": 0.6129557291666666, "paraphrase_len24_arg_accuracy": 0.4270833333333333, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.004557291666666667, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.0, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.015625, "paired_len4_executor_target_mass": 0.010419070211355574, "paired_len4_init_accuracy": 0.015625, "paired_len4_op_accuracy": 1.0, "paired_len4_arg_accuracy": 1.0, "paired_len4_program_exact": 0.015625, "paired_len4_state_accuracy": 0.015625, "paired_len4_state_all_exact": 0.015625, "paired_len4_state_prefix_fraction": 0.015625, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.9296875, "paired_len4_executor_pair_both_correct": 0.015625, "paired_len4_compiler_pair_program_consistency": 0.9296875, "paired_len4_compiler_pair_state_consistency": 0.9296875, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.015625, "paired_len8_executor_target_mass": 0.010449915076605976, "paired_len8_init_accuracy": 0.01953125, "paired_len8_op_accuracy": 0.974609375, "paired_len8_arg_accuracy": 0.95703125, "paired_len8_program_exact": 0.015625, "paired_len8_state_accuracy": 0.01953125, "paired_len8_state_all_exact": 0.015625, "paired_len8_state_prefix_fraction": 0.0185546875, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.6015625, "paired_len8_executor_pair_both_correct": 0.015625, "paired_len8_compiler_pair_program_consistency": 0.6015625, "paired_len8_compiler_pair_state_consistency": 0.6015625, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.02734375, "paired_len12_executor_target_mass": 0.010511290573049337, "paired_len12_init_accuracy": 0.01953125, "paired_len12_op_accuracy": 0.9368489583333334, "paired_len12_arg_accuracy": 0.9000651041666666, "paired_len12_program_exact": 0.015625, "paired_len12_state_accuracy": 0.020833333333333332, "paired_len12_state_all_exact": 0.015625, "paired_len12_state_prefix_fraction": 0.018229166666666668, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.2421875, "paired_len12_executor_pair_both_correct": 0.015625, "paired_len12_compiler_pair_program_consistency": 0.2265625, "paired_len12_compiler_pair_state_consistency": 0.2265625, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.01171875, "paired_len24_executor_target_mass": 0.010309108154615387, "paired_len24_init_accuracy": 0.0390625, "paired_len24_op_accuracy": 0.697265625, "paired_len24_arg_accuracy": 0.5445963541666666, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.023274739583333332, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.015299479166666666, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "register_trace", "stage": "trace_train", "local_step": 150, "step": 450, "loss": 6.157031536102295, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 1.3553324937820435, "init_loss": 1.1810717582702637, "op_loss": 0.006426467094570398, "arg_loss": 0.01774647645652294, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.59375, "standard_len4_executor_target_mass": 0.29689966095611453, "standard_len4_init_accuracy": 0.59375, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 1.0, "standard_len4_program_exact": 0.59375, "standard_len4_state_accuracy": 0.59375, "standard_len4_state_all_exact": 0.59375, "standard_len4_state_prefix_fraction": 0.59375, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.5625, "standard_len8_executor_target_mass": 0.2348135500214994, "standard_len8_init_accuracy": 0.5625, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 1.0, "standard_len8_program_exact": 0.5625, "standard_len8_state_accuracy": 0.5625, "standard_len8_state_all_exact": 0.5625, "standard_len8_state_prefix_fraction": 0.5625, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.640625, "standard_len12_executor_target_mass": 0.26995949447155, "standard_len12_init_accuracy": 0.640625, "standard_len12_op_accuracy": 0.9993489583333334, "standard_len12_arg_accuracy": 1.0, "standard_len12_program_exact": 0.640625, "standard_len12_state_accuracy": 0.640625, "standard_len12_state_all_exact": 0.640625, "standard_len12_state_prefix_fraction": 0.640625, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0234375, "standard_len24_executor_target_mass": 0.010858368186745793, "standard_len24_init_accuracy": 0.5703125, "standard_len24_op_accuracy": 0.8902994791666666, "standard_len24_arg_accuracy": 0.8232421875, "standard_len24_program_exact": 0.0078125, "standard_len24_state_accuracy": 0.3938802083333333, "standard_len24_state_all_exact": 0.0078125, "standard_len24_state_prefix_fraction": 0.390625, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.5390625, "paraphrase_len4_executor_target_mass": 0.26688641123473644, "paraphrase_len4_init_accuracy": 0.546875, "paraphrase_len4_op_accuracy": 0.994140625, "paraphrase_len4_arg_accuracy": 1.0, "paraphrase_len4_program_exact": 0.5390625, "paraphrase_len4_state_accuracy": 0.541015625, "paraphrase_len4_state_all_exact": 0.5390625, "paraphrase_len4_state_prefix_fraction": 0.541015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.6171875, "paraphrase_len8_executor_target_mass": 0.2827171590179205, "paraphrase_len8_init_accuracy": 0.625, "paraphrase_len8_op_accuracy": 0.9951171875, "paraphrase_len8_arg_accuracy": 1.0, "paraphrase_len8_program_exact": 0.6171875, "paraphrase_len8_state_accuracy": 0.619140625, "paraphrase_len8_state_all_exact": 0.6171875, "paraphrase_len8_state_prefix_fraction": 0.619140625, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.5546875, "paraphrase_len12_executor_target_mass": 0.19448490208014846, "paraphrase_len12_init_accuracy": 0.59375, "paraphrase_len12_op_accuracy": 0.9954427083333334, "paraphrase_len12_arg_accuracy": 0.9993489583333334, "paraphrase_len12_program_exact": 0.5546875, "paraphrase_len12_state_accuracy": 0.5755208333333334, "paraphrase_len12_state_all_exact": 0.5546875, "paraphrase_len12_state_prefix_fraction": 0.5755208333333334, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0078125, "paraphrase_len24_executor_target_mass": 0.010256846668198705, "paraphrase_len24_init_accuracy": 0.5625, "paraphrase_len24_op_accuracy": 0.8059895833333334, "paraphrase_len24_arg_accuracy": 0.7294921875, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.3639322916666667, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.359375, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.57421875, "paired_len4_executor_target_mass": 0.31863838067511097, "paired_len4_init_accuracy": 0.58203125, "paired_len4_op_accuracy": 0.99609375, "paired_len4_arg_accuracy": 1.0, "paired_len4_program_exact": 0.57421875, "paired_len4_state_accuracy": 0.578125, "paired_len4_state_all_exact": 0.57421875, "paired_len4_state_prefix_fraction": 0.578125, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.953125, "paired_len4_executor_pair_both_correct": 0.5625, "paired_len4_compiler_pair_program_consistency": 0.953125, "paired_len4_compiler_pair_state_consistency": 0.953125, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.6640625, "paired_len8_executor_target_mass": 0.2957001042086631, "paired_len8_init_accuracy": 0.68359375, "paired_len8_op_accuracy": 0.99609375, "paired_len8_arg_accuracy": 1.0, "paired_len8_program_exact": 0.6640625, "paired_len8_state_accuracy": 0.67236328125, "paired_len8_state_all_exact": 0.6640625, "paired_len8_state_prefix_fraction": 0.67236328125, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.9296875, "paired_len8_executor_pair_both_correct": 0.640625, "paired_len8_compiler_pair_program_consistency": 0.9296875, "paired_len8_compiler_pair_state_consistency": 0.9296875, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.484375, "paired_len12_executor_target_mass": 0.20943184848874807, "paired_len12_init_accuracy": 0.515625, "paired_len12_op_accuracy": 0.9951171875, "paired_len12_arg_accuracy": 1.0, "paired_len12_program_exact": 0.484375, "paired_len12_state_accuracy": 0.5078125, "paired_len12_state_all_exact": 0.484375, "paired_len12_state_prefix_fraction": 0.5078125, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.9140625, "paired_len12_executor_pair_both_correct": 0.4609375, "paired_len12_compiler_pair_program_consistency": 0.9140625, "paired_len12_compiler_pair_state_consistency": 0.9140625, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.015625, "paired_len24_executor_target_mass": 0.01057539737666957, "paired_len24_init_accuracy": 0.6171875, "paired_len24_op_accuracy": 0.8465169270833334, "paired_len24_arg_accuracy": 0.76318359375, "paired_len24_program_exact": 0.0078125, "paired_len24_state_accuracy": 0.4013671875, "paired_len24_state_all_exact": 0.0078125, "paired_len24_state_prefix_fraction": 0.3987630208333333, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "register_trace", "stage": "trace_long", "local_step": 1, "step": 451, "loss": 10.45849323272705, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 2.4063122272491455, "init_loss": 1.6758575439453125, "op_loss": 0.021361960098147392, "arg_loss": 0.3318473696708679, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.6015625, "standard_len4_executor_target_mass": 0.30084098782390356, "standard_len4_init_accuracy": 0.6015625, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 1.0, "standard_len4_program_exact": 0.6015625, "standard_len4_state_accuracy": 0.6015625, "standard_len4_state_all_exact": 0.6015625, "standard_len4_state_prefix_fraction": 0.6015625, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.5859375, "standard_len8_executor_target_mass": 0.2386309588328004, "standard_len8_init_accuracy": 0.5859375, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 1.0, "standard_len8_program_exact": 0.5859375, "standard_len8_state_accuracy": 0.5859375, "standard_len8_state_all_exact": 0.5859375, "standard_len8_state_prefix_fraction": 0.5859375, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.671875, "standard_len12_executor_target_mass": 0.2675067838281393, "standard_len12_init_accuracy": 0.671875, "standard_len12_op_accuracy": 0.9993489583333334, "standard_len12_arg_accuracy": 1.0, "standard_len12_program_exact": 0.671875, "standard_len12_state_accuracy": 0.671875, "standard_len12_state_all_exact": 0.671875, "standard_len12_state_prefix_fraction": 0.671875, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0859375, "standard_len24_executor_target_mass": 0.017024242784827948, "standard_len24_init_accuracy": 0.5859375, "standard_len24_op_accuracy": 0.9384765625, "standard_len24_arg_accuracy": 0.9033203125, "standard_len24_program_exact": 0.0703125, "standard_len24_state_accuracy": 0.4873046875, "standard_len24_state_all_exact": 0.0703125, "standard_len24_state_prefix_fraction": 0.4837239583333333, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.5625, "paraphrase_len4_executor_target_mass": 0.2687405711039901, "paraphrase_len4_init_accuracy": 0.5703125, "paraphrase_len4_op_accuracy": 0.994140625, "paraphrase_len4_arg_accuracy": 1.0, "paraphrase_len4_program_exact": 0.5625, "paraphrase_len4_state_accuracy": 0.564453125, "paraphrase_len4_state_all_exact": 0.5625, "paraphrase_len4_state_prefix_fraction": 0.564453125, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.6171875, "paraphrase_len8_executor_target_mass": 0.2830108106136322, "paraphrase_len8_init_accuracy": 0.625, "paraphrase_len8_op_accuracy": 0.9931640625, "paraphrase_len8_arg_accuracy": 1.0, "paraphrase_len8_program_exact": 0.6171875, "paraphrase_len8_state_accuracy": 0.619140625, "paraphrase_len8_state_all_exact": 0.6171875, "paraphrase_len8_state_prefix_fraction": 0.619140625, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.5625, "paraphrase_len12_executor_target_mass": 0.19662960548885167, "paraphrase_len12_init_accuracy": 0.59375, "paraphrase_len12_op_accuracy": 0.9947916666666666, "paraphrase_len12_arg_accuracy": 1.0, "paraphrase_len12_program_exact": 0.5625, "paraphrase_len12_state_accuracy": 0.576171875, "paraphrase_len12_state_all_exact": 0.5625, "paraphrase_len12_state_prefix_fraction": 0.576171875, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.015625, "paraphrase_len24_executor_target_mass": 0.010432650044094771, "paraphrase_len24_init_accuracy": 0.5703125, "paraphrase_len24_op_accuracy": 0.8697916666666666, "paraphrase_len24_arg_accuracy": 0.8128255208333334, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.3935546875, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.390625, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.59375, "paired_len4_executor_target_mass": 0.3217685622512363, "paired_len4_init_accuracy": 0.6015625, "paired_len4_op_accuracy": 0.99609375, "paired_len4_arg_accuracy": 1.0, "paired_len4_program_exact": 0.59375, "paired_len4_state_accuracy": 0.59765625, "paired_len4_state_all_exact": 0.59375, "paired_len4_state_prefix_fraction": 0.59765625, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.9375, "paired_len4_executor_pair_both_correct": 0.578125, "paired_len4_compiler_pair_program_consistency": 0.9375, "paired_len4_compiler_pair_state_consistency": 0.9375, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.6953125, "paired_len8_executor_target_mass": 0.2971449987962842, "paired_len8_init_accuracy": 0.71875, "paired_len8_op_accuracy": 0.99560546875, "paired_len8_arg_accuracy": 1.0, "paired_len8_program_exact": 0.6953125, "paired_len8_state_accuracy": 0.70654296875, "paired_len8_state_all_exact": 0.6953125, "paired_len8_state_prefix_fraction": 0.70654296875, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.9140625, "paired_len8_executor_pair_both_correct": 0.6640625, "paired_len8_compiler_pair_program_consistency": 0.9140625, "paired_len8_compiler_pair_state_consistency": 0.9140625, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.48828125, "paired_len12_executor_target_mass": 0.2071643522940576, "paired_len12_init_accuracy": 0.53125, "paired_len12_op_accuracy": 0.9944661458333334, "paired_len12_arg_accuracy": 1.0, "paired_len12_program_exact": 0.48828125, "paired_len12_state_accuracy": 0.5201822916666666, "paired_len12_state_all_exact": 0.48828125, "paired_len12_state_prefix_fraction": 0.5201822916666666, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.9140625, "paired_len12_executor_pair_both_correct": 0.4609375, "paired_len12_compiler_pair_program_consistency": 0.9140625, "paired_len12_compiler_pair_state_consistency": 0.9140625, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.0546875, "paired_len24_executor_target_mass": 0.014266489655710757, "paired_len24_init_accuracy": 0.62890625, "paired_len24_op_accuracy": 0.8966471354166666, "paired_len24_arg_accuracy": 0.8544921875, "paired_len24_program_exact": 0.04296875, "paired_len24_state_accuracy": 0.462890625, "paired_len24_state_all_exact": 0.046875, "paired_len24_state_prefix_fraction": 0.4617513020833333, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.015625, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "register_trace", "stage": "trace_long", "local_step": 150, "step": 600, "loss": 6.971364974975586, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.0432891845703125, "init_loss": 0.3620527982711792, "op_loss": 0.1620725393295288, "arg_loss": 0.32944801449775696, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.8828125, "standard_len4_executor_target_mass": 0.7028905414044857, "standard_len4_init_accuracy": 0.8828125, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 1.0, "standard_len4_program_exact": 0.8828125, "standard_len4_state_accuracy": 0.8828125, "standard_len4_state_all_exact": 0.8828125, "standard_len4_state_prefix_fraction": 0.8828125, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.9453125, "standard_len8_executor_target_mass": 0.6919209510087967, "standard_len8_init_accuracy": 0.9453125, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 1.0, "standard_len8_program_exact": 0.9453125, "standard_len8_state_accuracy": 0.9453125, "standard_len8_state_all_exact": 0.9453125, "standard_len8_state_prefix_fraction": 0.9453125, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.9453125, "standard_len12_executor_target_mass": 0.6752757541835308, "standard_len12_init_accuracy": 0.9453125, "standard_len12_op_accuracy": 1.0, "standard_len12_arg_accuracy": 1.0, "standard_len12_program_exact": 0.9453125, "standard_len12_state_accuracy": 0.9453125, "standard_len12_state_all_exact": 0.9453125, "standard_len12_state_prefix_fraction": 0.9453125, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.21875, "standard_len24_executor_target_mass": 0.01547760347602889, "standard_len24_init_accuracy": 0.9140625, "standard_len24_op_accuracy": 0.9729817708333334, "standard_len24_arg_accuracy": 0.9518229166666666, "standard_len24_program_exact": 0.2109375, "standard_len24_state_accuracy": 0.8391927083333334, "standard_len24_state_all_exact": 0.2109375, "standard_len24_state_prefix_fraction": 0.8388671875, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.90625, "paraphrase_len4_executor_target_mass": 0.7145605906844139, "paraphrase_len4_init_accuracy": 0.90625, "paraphrase_len4_op_accuracy": 1.0, "paraphrase_len4_arg_accuracy": 1.0, "paraphrase_len4_program_exact": 0.90625, "paraphrase_len4_state_accuracy": 0.90625, "paraphrase_len4_state_all_exact": 0.90625, "paraphrase_len4_state_prefix_fraction": 0.90625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.9140625, "paraphrase_len8_executor_target_mass": 0.7244286276400089, "paraphrase_len8_init_accuracy": 0.9140625, "paraphrase_len8_op_accuracy": 1.0, "paraphrase_len8_arg_accuracy": 1.0, "paraphrase_len8_program_exact": 0.9140625, "paraphrase_len8_state_accuracy": 0.9140625, "paraphrase_len8_state_all_exact": 0.9140625, "paraphrase_len8_state_prefix_fraction": 0.9140625, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.890625, "paraphrase_len12_executor_target_mass": 0.6371418684720993, "paraphrase_len12_init_accuracy": 0.890625, "paraphrase_len12_op_accuracy": 1.0, "paraphrase_len12_arg_accuracy": 1.0, "paraphrase_len12_program_exact": 0.890625, "paraphrase_len12_state_accuracy": 0.890625, "paraphrase_len12_state_all_exact": 0.890625, "paraphrase_len12_state_prefix_fraction": 0.890625, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.03125, "paraphrase_len24_executor_target_mass": 0.011212290672119707, "paraphrase_len24_init_accuracy": 0.890625, "paraphrase_len24_op_accuracy": 0.9248046875, "paraphrase_len24_arg_accuracy": 0.8854166666666666, "paraphrase_len24_program_exact": 0.0234375, "paraphrase_len24_state_accuracy": 0.7314453125, "paraphrase_len24_state_all_exact": 0.0234375, "paraphrase_len24_state_prefix_fraction": 0.7294921875, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.87890625, "paired_len4_executor_target_mass": 0.7052889559417963, "paired_len4_init_accuracy": 0.87890625, "paired_len4_op_accuracy": 1.0, "paired_len4_arg_accuracy": 1.0, "paired_len4_program_exact": 0.87890625, "paired_len4_state_accuracy": 0.87890625, "paired_len4_state_all_exact": 0.87890625, "paired_len4_state_prefix_fraction": 0.87890625, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.9921875, "paired_len4_executor_pair_both_correct": 0.875, "paired_len4_compiler_pair_program_consistency": 0.9921875, "paired_len4_compiler_pair_state_consistency": 0.9921875, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.9609375, "paired_len8_executor_target_mass": 0.7325926907360554, "paired_len8_init_accuracy": 0.9609375, "paired_len8_op_accuracy": 1.0, "paired_len8_arg_accuracy": 1.0, "paired_len8_program_exact": 0.9609375, "paired_len8_state_accuracy": 0.9609375, "paired_len8_state_all_exact": 0.9609375, "paired_len8_state_prefix_fraction": 0.9609375, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.9921875, "paired_len8_executor_pair_both_correct": 0.9609375, "paired_len8_compiler_pair_program_consistency": 0.9921875, "paired_len8_compiler_pair_state_consistency": 0.9921875, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.91015625, "paired_len12_executor_target_mass": 0.6914369934238493, "paired_len12_init_accuracy": 0.91015625, "paired_len12_op_accuracy": 1.0, "paired_len12_arg_accuracy": 1.0, "paired_len12_program_exact": 0.91015625, "paired_len12_state_accuracy": 0.91015625, "paired_len12_state_all_exact": 0.91015625, "paired_len12_state_prefix_fraction": 0.91015625, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.984375, "paired_len12_executor_pair_both_correct": 0.90625, "paired_len12_compiler_pair_program_consistency": 0.984375, "paired_len12_compiler_pair_state_consistency": 0.984375, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.125, "paired_len24_executor_target_mass": 0.013859308004612103, "paired_len24_init_accuracy": 0.921875, "paired_len24_op_accuracy": 0.9474283854166666, "paired_len24_arg_accuracy": 0.9241536458333334, "paired_len24_program_exact": 0.12109375, "paired_len24_state_accuracy": 0.7965494791666666, "paired_len24_state_all_exact": 0.12109375, "paired_len24_state_prefix_fraction": 0.794921875, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.046875, "paired_len24_executor_pair_both_correct": 0.015625, "paired_len24_compiler_pair_program_consistency": 0.03125, "paired_len24_compiler_pair_state_consistency": 0.03125 } ], "final_metrics": { "standard_len4": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.8828125, "executor_target_mass": 0.7028905414044857, "init_accuracy": 0.8828125, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.8828125, "state_accuracy": 0.8828125, "state_all_exact": 0.8828125, "state_prefix_fraction": 0.8828125 }, "standard_len8": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.9453125, "executor_target_mass": 0.6919209510087967, "init_accuracy": 0.9453125, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.9453125, "state_accuracy": 0.9453125, "state_all_exact": 0.9453125, "state_prefix_fraction": 0.9453125 }, "standard_len12": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.9453125, "executor_target_mass": 0.6752757541835308, "init_accuracy": 0.9453125, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.9453125, "state_accuracy": 0.9453125, "state_all_exact": 0.9453125, "state_prefix_fraction": 0.9453125 }, "standard_len24": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.21875, "executor_target_mass": 0.01547760347602889, "init_accuracy": 0.9140625, "op_accuracy": 0.9729817708333334, "arg_accuracy": 0.9518229166666666, "program_exact": 0.2109375, "state_accuracy": 0.8391927083333334, "state_all_exact": 0.2109375, "state_prefix_fraction": 0.8388671875 }, "paraphrase_len4": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.90625, "executor_target_mass": 0.7145605906844139, "init_accuracy": 0.90625, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.90625, "state_accuracy": 0.90625, "state_all_exact": 0.90625, "state_prefix_fraction": 0.90625 }, "paraphrase_len8": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.9140625, "executor_target_mass": 0.7244286276400089, "init_accuracy": 0.9140625, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.9140625, "state_accuracy": 0.9140625, "state_all_exact": 0.9140625, "state_prefix_fraction": 0.9140625 }, "paraphrase_len12": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.890625, "executor_target_mass": 0.6371418684720993, "init_accuracy": 0.890625, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.890625, "state_accuracy": 0.890625, "state_all_exact": 0.890625, "state_prefix_fraction": 0.890625 }, "paraphrase_len24": { "variant": "register_trace", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.03125, "executor_target_mass": 0.011212290672119707, "init_accuracy": 0.890625, "op_accuracy": 0.9248046875, "arg_accuracy": 0.8854166666666666, "program_exact": 0.0234375, "state_accuracy": 0.7314453125, "state_all_exact": 0.0234375, "state_prefix_fraction": 0.7294921875 }, "paired_len4": { "variant": "register_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.87890625, "executor_target_mass": 0.7052889559417963, "init_accuracy": 0.87890625, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.87890625, "state_accuracy": 0.87890625, "state_all_exact": 0.87890625, "state_prefix_fraction": 0.87890625, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 0.9921875, "executor_pair_both_correct": 0.875, "compiler_pair_program_consistency": 0.9921875, "compiler_pair_state_consistency": 0.9921875 }, "paired_len8": { "variant": "register_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.9609375, "executor_target_mass": 0.7325926907360554, "init_accuracy": 0.9609375, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.9609375, "state_accuracy": 0.9609375, "state_all_exact": 0.9609375, "state_prefix_fraction": 0.9609375, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 0.9921875, "executor_pair_both_correct": 0.9609375, "compiler_pair_program_consistency": 0.9921875, "compiler_pair_state_consistency": 0.9921875 }, "paired_len12": { "variant": "register_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.91015625, "executor_target_mass": 0.6914369934238493, "init_accuracy": 0.91015625, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.91015625, "state_accuracy": 0.91015625, "state_all_exact": 0.91015625, "state_prefix_fraction": 0.91015625, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 0.984375, "executor_pair_both_correct": 0.90625, "compiler_pair_program_consistency": 0.984375, "compiler_pair_state_consistency": 0.984375 }, "paired_len24": { "variant": "register_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.125, "executor_target_mass": 0.013859308004612103, "init_accuracy": 0.921875, "op_accuracy": 0.9474283854166666, "arg_accuracy": 0.9241536458333334, "program_exact": 0.12109375, "state_accuracy": 0.7965494791666666, "state_all_exact": 0.12109375, "state_prefix_fraction": 0.794921875, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 0.046875, "executor_pair_both_correct": 0.015625, "compiler_pair_program_consistency": 0.03125, "compiler_pair_state_consistency": 0.03125 } } } } }