{ "args": { "model_id": "Qwen/Qwen3-4B", "load_in_4bit": true, "torch_dtype": "bf16", "device_map": "auto", "use_lora": true, "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "lora_target_modules": "all-linear", "gradient_checkpointing": true, "optimizer": "paged_adamw_8bit", "modulus": 97, "max_steps": 24, "train_min_len": 1, "train_max_len": 12, "curriculum_stages": "short:1:4:80,medium:1:8:80,train:1:12:80", "answer_train_min_len": 1, "answer_train_max_len": 24, "eval_lengths": "4,8,12,24", "train_template_mode": "mixed", "paired_train": true, "paired_batches": true, "paired_template_modes": "standard,paraphrase", "pair_size": 2, "answer_train_template_mode": "mixed", "eval_template_modes": "standard,paraphrase", "paired_eval": true, "paired_eval_template_modes": "standard,paraphrase", "train_size": 256, "answer_train_size": 256, "eval_size": 64, "train_batch_size": 4, "eval_batch_size": 8, "bootstrap_steps": 0, "answer_steps": 0, "eval_every": 80, "stage_eval_every": 80, "lr": 0.0002, "answer_lr_scale": 0.1, "weight_decay": 0.0, "grad_clip": 1.0, "head_width": 256, "register_width": 256, "register_layers": 1, "register_heads": 4, "register_dropout": 0.05, "register_style": "bare", "trace_loss_weight": 1.0, "init_trace_loss_weight": 4.0, "op_trace_loss_weight": 1.0, "arg_trace_loss_weight": 4.0, "executor_loss_weight": 1.0, "state_loss_weight": 1.0, "pair_consistency_loss_weight": 0.1, "pair_state_consistency_loss_weight": 0.5, "variants": "structured_trace_state_consistency", "max_length": 768, "seed": 7, "train_seed": -1, "output_dir": "experiments/qwen_register_structured_runtime/runs/pilot_structured_state_consistency_s240", "checkpoint_dir": "large_artifacts/qwen_register_structured_runtime/checkpoints/pilot_structured_state_consistency_s240" }, "metadata": { "python": "3.12.3", "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "torch": "2.8.0+cu128", "cuda_available": true, "transformers_loader": "AutoModelForCausalLM", "peft_installed": true, "use_lora": true, "lora_r": 8, "gpu_name": "NVIDIA RTX 6000 Ada Generation", "gpu_vram_gb": 47.383 }, "hidden_dim": 2560, "register_count": 49, "dataset": { "train_size": 256, "train_examples": 1536, "curriculum_stages": [ { "name": "short", "min_len": 1, "max_len": 4, "steps": 80 }, { "name": "medium", "min_len": 1, "max_len": 8, "steps": 80 }, { "name": "train", "min_len": 1, "max_len": 12, "steps": 80 } ], "answer_train_size": 256, "eval_size": 64, "eval_lengths": [ 4, 8, 12, 24 ], "train_template_mode": "mixed", "paired_train": true, "paired_template_modes": [ "standard", "paraphrase" ], "answer_train_template_mode": "mixed", "eval_template_modes": [ "standard", "paraphrase" ], "paired_eval": true, "paired_eval_template_modes": [ "standard", "paraphrase" ], "register_style": "bare" }, "variants": { "structured_trace_state_consistency": { "variant": "structured_trace_state_consistency", "train_seconds": 921.9288547039032, "checkpoints": [ "large_artifacts/qwen_register_structured_runtime/checkpoints/pilot_structured_state_consistency_s240/structured_trace_state_consistency/adapter", "large_artifacts/qwen_register_structured_runtime/checkpoints/pilot_structured_state_consistency_s240/structured_trace_state_consistency/heads.pt" ], "train_log": [ { "variant": "structured_trace_state_consistency", "stage": "structured_trace_state_consistency_short", "local_step": 1, "step": 1, "loss": 47.07431411743164, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "lr_scale": 1.0, "executor_loss": 4.582798957824707, "state_loss": 4.581491947174072, "state_train_accuracy": 0.0, "pair_program_loss": 0.0052149249240756035, "pair_state_loss": 1.6290605344693176e-05, "init_loss": 4.597006320953369, "op_loss": 1.1381311416625977, "arg_loss": 4.595834255218506, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.046875, "standard_len4_executor_target_mass": 0.010264429613016546, "standard_len4_init_accuracy": 0.015625, "standard_len4_op_accuracy": 0.33984375, "standard_len4_arg_accuracy": 0.04296875, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.01171875, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.03125, "standard_len8_executor_target_mass": 0.010334888123907149, "standard_len8_init_accuracy": 0.015625, "standard_len8_op_accuracy": 0.322265625, "standard_len8_arg_accuracy": 0.029296875, "standard_len8_program_exact": 0.0, "standard_len8_state_accuracy": 0.013671875, "standard_len8_state_all_exact": 0.0, "standard_len8_state_prefix_fraction": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0, "standard_len12_executor_target_mass": 0.010263217729516327, "standard_len12_init_accuracy": 0.0, "standard_len12_op_accuracy": 0.3515625, "standard_len12_arg_accuracy": 0.037760416666666664, "standard_len12_program_exact": 0.0, "standard_len12_state_accuracy": 0.005208333333333333, "standard_len12_state_all_exact": 0.0, "standard_len12_state_prefix_fraction": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.015625, "standard_len24_executor_target_mass": 0.010263299336656928, "standard_len24_init_accuracy": 0.015625, "standard_len24_op_accuracy": 0.3424479166666667, "standard_len24_arg_accuracy": 0.026041666666666668, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.009114583333333334, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.0006510416666666666, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.03125, "paraphrase_len4_executor_target_mass": 0.01026039756834507, "paraphrase_len4_init_accuracy": 0.015625, "paraphrase_len4_op_accuracy": 0.31640625, "paraphrase_len4_arg_accuracy": 0.0390625, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.03125, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0, "paraphrase_len8_executor_target_mass": 0.010334869963116944, "paraphrase_len8_init_accuracy": 0.0, "paraphrase_len8_op_accuracy": 0.310546875, "paraphrase_len8_arg_accuracy": 0.03515625, "paraphrase_len8_program_exact": 0.0, "paraphrase_len8_state_accuracy": 0.01171875, "paraphrase_len8_state_all_exact": 0.0, "paraphrase_len8_state_prefix_fraction": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0, "paraphrase_len12_executor_target_mass": 0.010482321842573583, "paraphrase_len12_init_accuracy": 0.015625, "paraphrase_len12_op_accuracy": 0.3580729166666667, "paraphrase_len12_arg_accuracy": 0.04296875, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.009114583333333334, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0, "paraphrase_len24_executor_target_mass": 0.010264780139550567, "paraphrase_len24_init_accuracy": 0.0, "paraphrase_len24_op_accuracy": 0.3365885416666667, "paraphrase_len24_arg_accuracy": 0.037109375, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.0078125, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.0, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.010261152929160744, "paired_len4_init_accuracy": 0.0, "paired_len4_op_accuracy": 0.28125, "paired_len4_arg_accuracy": 0.03515625, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.009765625, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.001953125, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.015625, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.015625, "paired_len4_compiler_pair_state_consistency": 0.015625, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.03125, "paired_len8_executor_target_mass": 0.010261766670737416, "paired_len8_init_accuracy": 0.03125, "paired_len8_op_accuracy": 0.333984375, "paired_len8_arg_accuracy": 0.0302734375, "paired_len8_program_exact": 0.0, "paired_len8_state_accuracy": 0.009765625, "paired_len8_state_all_exact": 0.0, "paired_len8_state_prefix_fraction": 0.0009765625, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.015625, "paired_len8_executor_pair_both_correct": 0.0, "paired_len8_compiler_pair_program_consistency": 0.0, "paired_len8_compiler_pair_state_consistency": 0.0, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.0078125, "paired_len12_executor_target_mass": 0.010335352970287204, "paired_len12_init_accuracy": 0.0, "paired_len12_op_accuracy": 0.3268229166666667, "paired_len12_arg_accuracy": 0.0390625, "paired_len12_program_exact": 0.0, "paired_len12_state_accuracy": 0.014973958333333334, "paired_len12_state_all_exact": 0.0, "paired_len12_state_prefix_fraction": 0.0013020833333333333, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.046875, "paired_len12_executor_pair_both_correct": 0.0, "paired_len12_compiler_pair_program_consistency": 0.0, "paired_len12_compiler_pair_state_consistency": 0.0, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.0234375, "paired_len24_executor_target_mass": 0.010333538462873548, "paired_len24_init_accuracy": 0.0, "paired_len24_op_accuracy": 0.3274739583333333, "paired_len24_arg_accuracy": 0.040690104166666664, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.009765625, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.0, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "structured_trace_state_consistency", "stage": "structured_trace_state_consistency_short", "local_step": 80, "step": 80, "loss": 37.63404083251953, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "lr_scale": 1.0, "executor_loss": 4.491498947143555, "state_loss": 4.4875264167785645, "state_train_accuracy": 0.0, "pair_program_loss": 0.012317043729126453, "pair_state_loss": 0.00013295430107973516, "init_loss": 4.0138840675354, "op_loss": 0.049567174166440964, "arg_loss": 3.1371536254882812, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.01030040392652154, "standard_len4_init_accuracy": 0.234375, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.38671875, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.03515625, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.03125, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0, "standard_len8_executor_target_mass": 0.010298609733581543, "standard_len8_init_accuracy": 0.203125, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 0.421875, "standard_len8_program_exact": 0.0, "standard_len8_state_accuracy": 0.02734375, "standard_len8_state_all_exact": 0.0, "standard_len8_state_prefix_fraction": 0.01953125, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0, "standard_len12_executor_target_mass": 0.010282543837092817, "standard_len12_init_accuracy": 0.125, "standard_len12_op_accuracy": 0.9453125, "standard_len12_arg_accuracy": 0.3684895833333333, "standard_len12_program_exact": 0.0, "standard_len12_state_accuracy": 0.0234375, "standard_len12_state_all_exact": 0.0, "standard_len12_state_prefix_fraction": 0.015625, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.015625, "standard_len24_executor_target_mass": 0.010281694238074124, "standard_len24_init_accuracy": 0.140625, "standard_len24_op_accuracy": 0.8033854166666666, "standard_len24_arg_accuracy": 0.259765625, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.013671875, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.001953125, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.03125, "paraphrase_len4_executor_target_mass": 0.010285592288710177, "paraphrase_len4_init_accuracy": 0.21875, "paraphrase_len4_op_accuracy": 0.93359375, "paraphrase_len4_arg_accuracy": 0.3671875, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.05078125, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.03125, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.015625, "paraphrase_len8_executor_target_mass": 0.01030318159610033, "paraphrase_len8_init_accuracy": 0.125, "paraphrase_len8_op_accuracy": 0.822265625, "paraphrase_len8_arg_accuracy": 0.279296875, "paraphrase_len8_program_exact": 0.0, "paraphrase_len8_state_accuracy": 0.03125, "paraphrase_len8_state_all_exact": 0.0, "paraphrase_len8_state_prefix_fraction": 0.01953125, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.015625, "paraphrase_len12_executor_target_mass": 0.010479909484274685, "paraphrase_len12_init_accuracy": 0.125, "paraphrase_len12_op_accuracy": 0.7604166666666666, "paraphrase_len12_arg_accuracy": 0.24739583333333334, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.016927083333333332, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.00390625, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.03125, "paraphrase_len24_executor_target_mass": 0.010277464287355542, "paraphrase_len24_init_accuracy": 0.203125, "paraphrase_len24_op_accuracy": 0.5592447916666666, "paraphrase_len24_arg_accuracy": 0.14713541666666666, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.012369791666666666, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.007161458333333333, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.010294955922290683, "paired_len4_init_accuracy": 0.1015625, "paired_len4_op_accuracy": 0.97265625, "paired_len4_arg_accuracy": 0.3359375, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.005859375, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.005859375, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.46875, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.453125, "paired_len4_compiler_pair_state_consistency": 0.453125, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.015625, "paired_len8_executor_target_mass": 0.01028924627462402, "paired_len8_init_accuracy": 0.1953125, "paired_len8_op_accuracy": 0.9326171875, "paired_len8_arg_accuracy": 0.388671875, "paired_len8_program_exact": 0.0, "paired_len8_state_accuracy": 0.0283203125, "paired_len8_state_all_exact": 0.0, "paired_len8_state_prefix_fraction": 0.01953125, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.0625, "paired_len8_executor_pair_both_correct": 0.0, "paired_len8_compiler_pair_program_consistency": 0.0625, "paired_len8_compiler_pair_state_consistency": 0.0625, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.015625, "paired_len12_executor_target_mass": 0.010420453734695911, "paired_len12_init_accuracy": 0.1953125, "paired_len12_op_accuracy": 0.853515625, "paired_len12_arg_accuracy": 0.3131510416666667, "paired_len12_program_exact": 0.0, "paired_len12_state_accuracy": 0.028645833333333332, "paired_len12_state_all_exact": 0.0, "paired_len12_state_prefix_fraction": 0.01953125, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.015625, "paired_len12_executor_pair_both_correct": 0.0, "paired_len12_compiler_pair_program_consistency": 0.0, "paired_len12_compiler_pair_state_consistency": 0.0, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.0234375, "paired_len24_executor_target_mass": 0.01037386542884633, "paired_len24_init_accuracy": 0.1953125, "paired_len24_op_accuracy": 0.6634114583333334, "paired_len24_arg_accuracy": 0.19791666666666666, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.012369791666666666, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.0022786458333333335, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.015625, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "structured_trace_state_consistency", "stage": "structured_trace_state_consistency_medium", "local_step": 1, "step": 81, "loss": 44.23200988769531, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "lr_scale": 1.0, "executor_loss": 4.578611850738525, "state_loss": 4.575760364532471, "state_train_accuracy": 0.0, "pair_program_loss": 0.8843308687210083, "pair_state_loss": 8.893662015907466e-05, "init_loss": 5.105445384979248, "op_loss": 0.6174622774124146, "arg_loss": 3.4874799251556396, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.010300629306584597, "standard_len4_init_accuracy": 0.21875, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.375, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.01953125, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.015625, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.015625, "standard_len8_executor_target_mass": 0.010304971947334707, "standard_len8_init_accuracy": 0.21875, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 0.412109375, "standard_len8_program_exact": 0.0, "standard_len8_state_accuracy": 0.033203125, "standard_len8_state_all_exact": 0.0, "standard_len8_state_prefix_fraction": 0.021484375, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0, "standard_len12_executor_target_mass": 0.010281685623340309, "standard_len12_init_accuracy": 0.125, "standard_len12_op_accuracy": 0.9296875, "standard_len12_arg_accuracy": 0.3658854166666667, "standard_len12_program_exact": 0.0, "standard_len12_state_accuracy": 0.022135416666666668, "standard_len12_state_all_exact": 0.0, "standard_len12_state_prefix_fraction": 0.015625, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.015625, "standard_len24_executor_target_mass": 0.01028335781302303, "standard_len24_init_accuracy": 0.140625, "standard_len24_op_accuracy": 0.796875, "standard_len24_arg_accuracy": 0.2578125, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.013671875, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.001953125, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.03125, "paraphrase_len4_executor_target_mass": 0.010285517200827599, "paraphrase_len4_init_accuracy": 0.21875, "paraphrase_len4_op_accuracy": 0.93359375, "paraphrase_len4_arg_accuracy": 0.36328125, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.0546875, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.02734375, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0, "paraphrase_len8_executor_target_mass": 0.010314087267033756, "paraphrase_len8_init_accuracy": 0.109375, "paraphrase_len8_op_accuracy": 0.806640625, "paraphrase_len8_arg_accuracy": 0.271484375, "paraphrase_len8_program_exact": 0.0, "paraphrase_len8_state_accuracy": 0.015625, "paraphrase_len8_state_all_exact": 0.0, "paraphrase_len8_state_prefix_fraction": 0.01171875, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0, "paraphrase_len12_executor_target_mass": 0.010485190316103399, "paraphrase_len12_init_accuracy": 0.15625, "paraphrase_len12_op_accuracy": 0.7473958333333334, "paraphrase_len12_arg_accuracy": 0.24609375, "paraphrase_len12_program_exact": 0.0, "paraphrase_len12_state_accuracy": 0.018229166666666668, "paraphrase_len12_state_all_exact": 0.0, "paraphrase_len12_state_prefix_fraction": 0.006510416666666667, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0, "paraphrase_len24_executor_target_mass": 0.010279944981448352, "paraphrase_len24_init_accuracy": 0.1875, "paraphrase_len24_op_accuracy": 0.5553385416666666, "paraphrase_len24_arg_accuracy": 0.14518229166666666, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.01171875, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.007161458333333333, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.010296111984644085, "paired_len4_init_accuracy": 0.09375, "paired_len4_op_accuracy": 0.978515625, "paired_len4_arg_accuracy": 0.341796875, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.0078125, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0078125, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.546875, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.53125, "paired_len4_compiler_pair_state_consistency": 0.546875, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.015625, "paired_len8_executor_target_mass": 0.010290001169778407, "paired_len8_init_accuracy": 0.203125, "paired_len8_op_accuracy": 0.9345703125, "paired_len8_arg_accuracy": 0.3876953125, "paired_len8_program_exact": 0.0, "paired_len8_state_accuracy": 0.029296875, "paired_len8_state_all_exact": 0.0, "paired_len8_state_prefix_fraction": 0.021484375, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.078125, "paired_len8_executor_pair_both_correct": 0.0, "paired_len8_compiler_pair_program_consistency": 0.0625, "paired_len8_compiler_pair_state_consistency": 0.0625, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.0, "paired_len12_executor_target_mass": 0.010413670621346682, "paired_len12_init_accuracy": 0.1875, "paired_len12_op_accuracy": 0.841796875, "paired_len12_arg_accuracy": 0.3118489583333333, "paired_len12_program_exact": 0.0, "paired_len12_state_accuracy": 0.029947916666666668, "paired_len12_state_all_exact": 0.0, "paired_len12_state_prefix_fraction": 0.018880208333333332, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.015625, "paired_len12_executor_pair_both_correct": 0.0, "paired_len12_compiler_pair_program_consistency": 0.015625, "paired_len12_compiler_pair_state_consistency": 0.015625, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.0078125, "paired_len24_executor_target_mass": 0.01035986456554383, "paired_len24_init_accuracy": 0.1796875, "paired_len24_op_accuracy": 0.658203125, "paired_len24_arg_accuracy": 0.2060546875, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.01171875, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.0022786458333333335, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.015625, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "structured_trace_state_consistency", "stage": "structured_trace_state_consistency_medium", "local_step": 80, "step": 160, "loss": 10.033712387084961, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "lr_scale": 1.0, "executor_loss": 2.654987096786499, "state_loss": 2.5366129875183105, "state_train_accuracy": 0.8333333134651184, "pair_program_loss": 0.0037166306283324957, "pair_state_loss": 0.00047525085392408073, "init_loss": 0.6143349409103394, "op_loss": 0.007244561333209276, "arg_loss": 0.5942298173904419, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.609375, "standard_len4_executor_target_mass": 0.023149261018261313, "standard_len4_init_accuracy": 0.671875, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.95703125, "standard_len4_program_exact": 0.609375, "standard_len4_state_accuracy": 0.625, "standard_len4_state_all_exact": 0.609375, "standard_len4_state_prefix_fraction": 0.625, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.453125, "standard_len8_executor_target_mass": 0.011317304684780538, "standard_len8_init_accuracy": 0.578125, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 0.95703125, "standard_len8_program_exact": 0.421875, "standard_len8_state_accuracy": 0.498046875, "standard_len8_state_all_exact": 0.421875, "standard_len8_state_prefix_fraction": 0.490234375, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.484375, "standard_len12_executor_target_mass": 0.010403058375231922, "standard_len12_init_accuracy": 0.6875, "standard_len12_op_accuracy": 1.0, "standard_len12_arg_accuracy": 0.9596354166666666, "standard_len12_program_exact": 0.484375, "standard_len12_state_accuracy": 0.5872395833333334, "standard_len12_state_all_exact": 0.484375, "standard_len12_state_prefix_fraction": 0.5872395833333334, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0, "standard_len24_executor_target_mass": 0.010319901979528368, "standard_len24_init_accuracy": 0.625, "standard_len24_op_accuracy": 0.81640625, "standard_len24_arg_accuracy": 0.6569010416666666, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.24544270833333334, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.24348958333333334, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.515625, "paraphrase_len4_executor_target_mass": 0.02675232314504683, "paraphrase_len4_init_accuracy": 0.609375, "paraphrase_len4_op_accuracy": 1.0, "paraphrase_len4_arg_accuracy": 0.96875, "paraphrase_len4_program_exact": 0.515625, "paraphrase_len4_state_accuracy": 0.57421875, "paraphrase_len4_state_all_exact": 0.515625, "paraphrase_len4_state_prefix_fraction": 0.57421875, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.234375, "paraphrase_len8_executor_target_mass": 0.010708913439884782, "paraphrase_len8_init_accuracy": 0.453125, "paraphrase_len8_op_accuracy": 0.96484375, "paraphrase_len8_arg_accuracy": 0.88671875, "paraphrase_len8_program_exact": 0.21875, "paraphrase_len8_state_accuracy": 0.330078125, "paraphrase_len8_state_all_exact": 0.21875, "paraphrase_len8_state_prefix_fraction": 0.318359375, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.1875, "paraphrase_len12_executor_target_mass": 0.01036103954538703, "paraphrase_len12_init_accuracy": 0.5625, "paraphrase_len12_op_accuracy": 0.9609375, "paraphrase_len12_arg_accuracy": 0.8841145833333334, "paraphrase_len12_program_exact": 0.171875, "paraphrase_len12_state_accuracy": 0.3489583333333333, "paraphrase_len12_state_all_exact": 0.171875, "paraphrase_len12_state_prefix_fraction": 0.3463541666666667, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0, "paraphrase_len24_executor_target_mass": 0.010304185329005122, "paraphrase_len24_init_accuracy": 0.546875, "paraphrase_len24_op_accuracy": 0.701171875, "paraphrase_len24_arg_accuracy": 0.529296875, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.21809895833333334, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.21028645833333334, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.5, "paired_len4_executor_target_mass": 0.01750925072701648, "paired_len4_init_accuracy": 0.6015625, "paired_len4_op_accuracy": 1.0, "paired_len4_arg_accuracy": 0.951171875, "paired_len4_program_exact": 0.5, "paired_len4_state_accuracy": 0.537109375, "paired_len4_state_all_exact": 0.5, "paired_len4_state_prefix_fraction": 0.537109375, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.90625, "paired_len4_executor_pair_both_correct": 0.46875, "paired_len4_compiler_pair_program_consistency": 0.90625, "paired_len4_compiler_pair_state_consistency": 0.90625, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.4921875, "paired_len8_executor_target_mass": 0.012341370049398392, "paired_len8_init_accuracy": 0.6328125, "paired_len8_op_accuracy": 0.9912109375, "paired_len8_arg_accuracy": 0.94921875, "paired_len8_program_exact": 0.4921875, "paired_len8_state_accuracy": 0.552734375, "paired_len8_state_all_exact": 0.4921875, "paired_len8_state_prefix_fraction": 0.552734375, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.875, "paired_len8_executor_pair_both_correct": 0.46875, "paired_len8_compiler_pair_program_consistency": 0.875, "paired_len8_compiler_pair_state_consistency": 0.875, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.328125, "paired_len12_executor_target_mass": 0.010525063204113394, "paired_len12_init_accuracy": 0.7265625, "paired_len12_op_accuracy": 0.9694010416666666, "paired_len12_arg_accuracy": 0.8932291666666666, "paired_len12_program_exact": 0.328125, "paired_len12_state_accuracy": 0.4973958333333333, "paired_len12_state_all_exact": 0.328125, "paired_len12_state_prefix_fraction": 0.49609375, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.5625, "paired_len12_executor_pair_both_correct": 0.296875, "paired_len12_compiler_pair_program_consistency": 0.5625, "paired_len12_compiler_pair_state_consistency": 0.5625, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.0078125, "paired_len24_executor_target_mass": 0.010303019313141704, "paired_len24_init_accuracy": 0.5078125, "paired_len24_op_accuracy": 0.7412109375, "paired_len24_arg_accuracy": 0.5875651041666666, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.22819010416666666, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.2216796875, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "structured_trace_state_consistency", "stage": "structured_trace_state_consistency_train", "local_step": 1, "step": 161, "loss": 27.609663009643555, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "lr_scale": 1.0, "executor_loss": 4.575992107391357, "state_loss": 4.440532207489014, "state_train_accuracy": 0.23529411852359772, "pair_program_loss": 0.015211493708193302, "pair_state_loss": 6.0324236983433366e-05, "init_loss": 3.531378746032715, "op_loss": 0.01474193949252367, "arg_loss": 1.11283278465271, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.640625, "standard_len4_executor_target_mass": 0.024013491114601493, "standard_len4_init_accuracy": 0.671875, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.98046875, "standard_len4_program_exact": 0.640625, "standard_len4_state_accuracy": 0.65625, "standard_len4_state_all_exact": 0.640625, "standard_len4_state_prefix_fraction": 0.65625, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.4375, "standard_len8_executor_target_mass": 0.011457669665105641, "standard_len8_init_accuracy": 0.546875, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 0.966796875, "standard_len8_program_exact": 0.421875, "standard_len8_state_accuracy": 0.4765625, "standard_len8_state_all_exact": 0.421875, "standard_len8_state_prefix_fraction": 0.474609375, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.53125, "standard_len12_executor_target_mass": 0.01041950355283916, "standard_len12_init_accuracy": 0.671875, "standard_len12_op_accuracy": 1.0, "standard_len12_arg_accuracy": 0.97265625, "standard_len12_program_exact": 0.53125, "standard_len12_state_accuracy": 0.6119791666666666, "standard_len12_state_all_exact": 0.53125, "standard_len12_state_prefix_fraction": 0.6119791666666666, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0, "standard_len24_executor_target_mass": 0.010321857291273773, "standard_len24_init_accuracy": 0.625, "standard_len24_op_accuracy": 0.81640625, "standard_len24_arg_accuracy": 0.6640625, "standard_len24_program_exact": 0.0, "standard_len24_state_accuracy": 0.2805989583333333, "standard_len24_state_all_exact": 0.0, "standard_len24_state_prefix_fraction": 0.2779947916666667, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.5625, "paraphrase_len4_executor_target_mass": 0.027572402032092214, "paraphrase_len4_init_accuracy": 0.640625, "paraphrase_len4_op_accuracy": 1.0, "paraphrase_len4_arg_accuracy": 0.97265625, "paraphrase_len4_program_exact": 0.5625, "paraphrase_len4_state_accuracy": 0.609375, "paraphrase_len4_state_all_exact": 0.5625, "paraphrase_len4_state_prefix_fraction": 0.609375, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.234375, "paraphrase_len8_executor_target_mass": 0.010779290460050106, "paraphrase_len8_init_accuracy": 0.453125, "paraphrase_len8_op_accuracy": 0.962890625, "paraphrase_len8_arg_accuracy": 0.90234375, "paraphrase_len8_program_exact": 0.234375, "paraphrase_len8_state_accuracy": 0.330078125, "paraphrase_len8_state_all_exact": 0.234375, "paraphrase_len8_state_prefix_fraction": 0.330078125, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.265625, "paraphrase_len12_executor_target_mass": 0.01037182321306318, "paraphrase_len12_init_accuracy": 0.5625, "paraphrase_len12_op_accuracy": 0.9609375, "paraphrase_len12_arg_accuracy": 0.8971354166666666, "paraphrase_len12_program_exact": 0.25, "paraphrase_len12_state_accuracy": 0.3841145833333333, "paraphrase_len12_state_all_exact": 0.25, "paraphrase_len12_state_prefix_fraction": 0.3828125, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0, "paraphrase_len24_executor_target_mass": 0.010304395109415054, "paraphrase_len24_init_accuracy": 0.53125, "paraphrase_len24_op_accuracy": 0.7024739583333334, "paraphrase_len24_arg_accuracy": 0.537109375, "paraphrase_len24_program_exact": 0.0, "paraphrase_len24_state_accuracy": 0.21549479166666666, "paraphrase_len24_state_all_exact": 0.0, "paraphrase_len24_state_prefix_fraction": 0.21028645833333334, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.4921875, "paired_len4_executor_target_mass": 0.018111868179403245, "paired_len4_init_accuracy": 0.578125, "paired_len4_op_accuracy": 1.0, "paired_len4_arg_accuracy": 0.958984375, "paired_len4_program_exact": 0.4921875, "paired_len4_state_accuracy": 0.525390625, "paired_len4_state_all_exact": 0.4921875, "paired_len4_state_prefix_fraction": 0.525390625, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.9375, "paired_len4_executor_pair_both_correct": 0.46875, "paired_len4_compiler_pair_program_consistency": 0.9375, "paired_len4_compiler_pair_state_consistency": 0.9375, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.53125, "paired_len8_executor_target_mass": 0.012587831530254334, "paired_len8_init_accuracy": 0.6328125, "paired_len8_op_accuracy": 0.9931640625, "paired_len8_arg_accuracy": 0.958984375, "paired_len8_program_exact": 0.53125, "paired_len8_state_accuracy": 0.580078125, "paired_len8_state_all_exact": 0.53125, "paired_len8_state_prefix_fraction": 0.580078125, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 0.859375, "paired_len8_executor_pair_both_correct": 0.515625, "paired_len8_compiler_pair_program_consistency": 0.859375, "paired_len8_compiler_pair_state_consistency": 0.859375, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.4140625, "paired_len12_executor_target_mass": 0.010555185028351843, "paired_len12_init_accuracy": 0.7265625, "paired_len12_op_accuracy": 0.97265625, "paired_len12_arg_accuracy": 0.9140625, "paired_len12_program_exact": 0.40625, "paired_len12_state_accuracy": 0.5703125, "paired_len12_state_all_exact": 0.40625, "paired_len12_state_prefix_fraction": 0.5690104166666666, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.578125, "paired_len12_executor_pair_both_correct": 0.359375, "paired_len12_compiler_pair_program_consistency": 0.578125, "paired_len12_compiler_pair_state_consistency": 0.578125, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.0078125, "paired_len24_executor_target_mass": 0.010302758193574846, "paired_len24_init_accuracy": 0.484375, "paired_len24_op_accuracy": 0.740234375, "paired_len24_arg_accuracy": 0.5930989583333334, "paired_len24_program_exact": 0.0, "paired_len24_state_accuracy": 0.21256510416666666, "paired_len24_state_all_exact": 0.0, "paired_len24_state_prefix_fraction": 0.20475260416666666, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.0, "paired_len24_executor_pair_both_correct": 0.0, "paired_len24_compiler_pair_program_consistency": 0.0, "paired_len24_compiler_pair_state_consistency": 0.0 }, { "variant": "structured_trace_state_consistency", "stage": "structured_trace_state_consistency_train", "local_step": 80, "step": 240, "loss": 7.763700485229492, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "lr_scale": 1.0, "executor_loss": 1.5169703960418701, "state_loss": 1.3111953735351562, "state_train_accuracy": 1.0, "pair_program_loss": 0.0011123993899673223, "pair_state_loss": 0.00047938033821992576, "init_loss": 1.1815276145935059, "op_loss": 0.0011618572752922773, "arg_loss": 0.051977794617414474, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.859375, "standard_len4_executor_target_mass": 0.2408065591007471, "standard_len4_init_accuracy": 0.859375, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 1.0, "standard_len4_program_exact": 0.859375, "standard_len4_state_accuracy": 0.859375, "standard_len4_state_all_exact": 0.859375, "standard_len4_state_prefix_fraction": 0.859375, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.796875, "standard_len8_executor_target_mass": 0.1302952878177166, "standard_len8_init_accuracy": 0.796875, "standard_len8_op_accuracy": 1.0, "standard_len8_arg_accuracy": 1.0, "standard_len8_program_exact": 0.796875, "standard_len8_state_accuracy": 0.796875, "standard_len8_state_all_exact": 0.796875, "standard_len8_state_prefix_fraction": 0.796875, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.859375, "standard_len12_executor_target_mass": 0.10725165531039238, "standard_len12_init_accuracy": 0.859375, "standard_len12_op_accuracy": 1.0, "standard_len12_arg_accuracy": 1.0, "standard_len12_program_exact": 0.859375, "standard_len12_state_accuracy": 0.859375, "standard_len12_state_all_exact": 0.859375, "standard_len12_state_prefix_fraction": 0.859375, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.078125, "standard_len24_executor_target_mass": 0.011610763031058013, "standard_len24_init_accuracy": 0.84375, "standard_len24_op_accuracy": 0.9127604166666666, "standard_len24_arg_accuracy": 0.85546875, "standard_len24_program_exact": 0.0625, "standard_len24_state_accuracy": 0.6432291666666666, "standard_len24_state_all_exact": 0.0625, "standard_len24_state_prefix_fraction": 0.6412760416666666, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.828125, "paraphrase_len4_executor_target_mass": 0.21950156427919865, "paraphrase_len4_init_accuracy": 0.828125, "paraphrase_len4_op_accuracy": 1.0, "paraphrase_len4_arg_accuracy": 1.0, "paraphrase_len4_program_exact": 0.828125, "paraphrase_len4_state_accuracy": 0.828125, "paraphrase_len4_state_all_exact": 0.828125, "paraphrase_len4_state_prefix_fraction": 0.828125, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.75, "paraphrase_len8_executor_target_mass": 0.12286304496228695, "paraphrase_len8_init_accuracy": 0.75, "paraphrase_len8_op_accuracy": 1.0, "paraphrase_len8_arg_accuracy": 1.0, "paraphrase_len8_program_exact": 0.75, "paraphrase_len8_state_accuracy": 0.75, "paraphrase_len8_state_all_exact": 0.75, "paraphrase_len8_state_prefix_fraction": 0.75, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.78125, "paraphrase_len12_executor_target_mass": 0.10984228271991014, "paraphrase_len12_init_accuracy": 0.78125, "paraphrase_len12_op_accuracy": 1.0, "paraphrase_len12_arg_accuracy": 1.0, "paraphrase_len12_program_exact": 0.78125, "paraphrase_len12_state_accuracy": 0.78125, "paraphrase_len12_state_all_exact": 0.78125, "paraphrase_len12_state_prefix_fraction": 0.78125, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.09375, "paraphrase_len24_executor_target_mass": 0.01400190859567374, "paraphrase_len24_init_accuracy": 0.703125, "paraphrase_len24_op_accuracy": 0.892578125, "paraphrase_len24_arg_accuracy": 0.8294270833333334, "paraphrase_len24_program_exact": 0.078125, "paraphrase_len24_state_accuracy": 0.5221354166666666, "paraphrase_len24_state_all_exact": 0.078125, "paraphrase_len24_state_prefix_fraction": 0.5188802083333334, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.765625, "paired_len4_executor_target_mass": 0.15648487000726163, "paired_len4_init_accuracy": 0.765625, "paired_len4_op_accuracy": 1.0, "paired_len4_arg_accuracy": 1.0, "paired_len4_program_exact": 0.765625, "paired_len4_state_accuracy": 0.765625, "paired_len4_state_all_exact": 0.765625, "paired_len4_state_prefix_fraction": 0.765625, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.96875, "paired_len4_executor_pair_both_correct": 0.765625, "paired_len4_compiler_pair_program_consistency": 0.96875, "paired_len4_compiler_pair_state_consistency": 0.96875, "paired_len8_direct_accuracy": NaN, "paired_len8_direct_target_mass": NaN, "paired_len8_executor_accuracy": 0.875, "paired_len8_executor_target_mass": 0.18304786505177617, "paired_len8_init_accuracy": 0.875, "paired_len8_op_accuracy": 1.0, "paired_len8_arg_accuracy": 1.0, "paired_len8_program_exact": 0.875, "paired_len8_state_accuracy": 0.875, "paired_len8_state_all_exact": 0.875, "paired_len8_state_prefix_fraction": 0.875, "paired_len8_pair_true_answer_consistency": 1.0, "paired_len8_executor_pair_answer_consistency": 1.0, "paired_len8_executor_pair_both_correct": 0.875, "paired_len8_compiler_pair_program_consistency": 1.0, "paired_len8_compiler_pair_state_consistency": 1.0, "paired_len12_direct_accuracy": NaN, "paired_len12_direct_target_mass": NaN, "paired_len12_executor_accuracy": 0.8203125, "paired_len12_executor_target_mass": 0.14072793535888195, "paired_len12_init_accuracy": 0.84375, "paired_len12_op_accuracy": 0.9993489583333334, "paired_len12_arg_accuracy": 0.9986979166666666, "paired_len12_program_exact": 0.8203125, "paired_len12_state_accuracy": 0.83984375, "paired_len12_state_all_exact": 0.8203125, "paired_len12_state_prefix_fraction": 0.83984375, "paired_len12_pair_true_answer_consistency": 1.0, "paired_len12_executor_pair_answer_consistency": 0.953125, "paired_len12_executor_pair_both_correct": 0.796875, "paired_len12_compiler_pair_program_consistency": 0.953125, "paired_len12_compiler_pair_state_consistency": 0.953125, "paired_len24_direct_accuracy": NaN, "paired_len24_direct_target_mass": NaN, "paired_len24_executor_accuracy": 0.09375, "paired_len24_executor_target_mass": 0.012367482180707157, "paired_len24_init_accuracy": 0.890625, "paired_len24_op_accuracy": 0.9010416666666666, "paired_len24_arg_accuracy": 0.8365885416666666, "paired_len24_program_exact": 0.078125, "paired_len24_state_accuracy": 0.6630859375, "paired_len24_state_all_exact": 0.078125, "paired_len24_state_prefix_fraction": 0.6572265625, "paired_len24_pair_true_answer_consistency": 1.0, "paired_len24_executor_pair_answer_consistency": 0.03125, "paired_len24_executor_pair_both_correct": 0.03125, "paired_len24_compiler_pair_program_consistency": 0.03125, "paired_len24_compiler_pair_state_consistency": 0.03125 } ], "final_metrics": { "standard_len4": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.859375, "executor_target_mass": 0.2408065591007471, "init_accuracy": 0.859375, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.859375, "state_accuracy": 0.859375, "state_all_exact": 0.859375, "state_prefix_fraction": 0.859375 }, "standard_len8": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.796875, "executor_target_mass": 0.1302952878177166, "init_accuracy": 0.796875, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.796875, "state_accuracy": 0.796875, "state_all_exact": 0.796875, "state_prefix_fraction": 0.796875 }, "standard_len12": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.859375, "executor_target_mass": 0.10725165531039238, "init_accuracy": 0.859375, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.859375, "state_accuracy": 0.859375, "state_all_exact": 0.859375, "state_prefix_fraction": 0.859375 }, "standard_len24": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.078125, "executor_target_mass": 0.011610763031058013, "init_accuracy": 0.84375, "op_accuracy": 0.9127604166666666, "arg_accuracy": 0.85546875, "program_exact": 0.0625, "state_accuracy": 0.6432291666666666, "state_all_exact": 0.0625, "state_prefix_fraction": 0.6412760416666666 }, "paraphrase_len4": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.828125, "executor_target_mass": 0.21950156427919865, "init_accuracy": 0.828125, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.828125, "state_accuracy": 0.828125, "state_all_exact": 0.828125, "state_prefix_fraction": 0.828125 }, "paraphrase_len8": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.75, "executor_target_mass": 0.12286304496228695, "init_accuracy": 0.75, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.75, "state_accuracy": 0.75, "state_all_exact": 0.75, "state_prefix_fraction": 0.75 }, "paraphrase_len12": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.78125, "executor_target_mass": 0.10984228271991014, "init_accuracy": 0.78125, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.78125, "state_accuracy": 0.78125, "state_all_exact": 0.78125, "state_prefix_fraction": 0.78125 }, "paraphrase_len24": { "variant": "structured_trace_state_consistency", "n": 64.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.09375, "executor_target_mass": 0.01400190859567374, "init_accuracy": 0.703125, "op_accuracy": 0.892578125, "arg_accuracy": 0.8294270833333334, "program_exact": 0.078125, "state_accuracy": 0.5221354166666666, "state_all_exact": 0.078125, "state_prefix_fraction": 0.5188802083333334 }, "paired_len4": { "variant": "structured_trace_state_consistency", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.765625, "executor_target_mass": 0.15648487000726163, "init_accuracy": 0.765625, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.765625, "state_accuracy": 0.765625, "state_all_exact": 0.765625, "state_prefix_fraction": 0.765625, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 0.96875, "executor_pair_both_correct": 0.765625, "compiler_pair_program_consistency": 0.96875, "compiler_pair_state_consistency": 0.96875 }, "paired_len8": { "variant": "structured_trace_state_consistency", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.875, "executor_target_mass": 0.18304786505177617, "init_accuracy": 0.875, "op_accuracy": 1.0, "arg_accuracy": 1.0, "program_exact": 0.875, "state_accuracy": 0.875, "state_all_exact": 0.875, "state_prefix_fraction": 0.875, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 1.0, "executor_pair_both_correct": 0.875, "compiler_pair_program_consistency": 1.0, "compiler_pair_state_consistency": 1.0 }, "paired_len12": { "variant": "structured_trace_state_consistency", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.8203125, "executor_target_mass": 0.14072793535888195, "init_accuracy": 0.84375, "op_accuracy": 0.9993489583333334, "arg_accuracy": 0.9986979166666666, "program_exact": 0.8203125, "state_accuracy": 0.83984375, "state_all_exact": 0.8203125, "state_prefix_fraction": 0.83984375, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 0.953125, "executor_pair_both_correct": 0.796875, "compiler_pair_program_consistency": 0.953125, "compiler_pair_state_consistency": 0.953125 }, "paired_len24": { "variant": "structured_trace_state_consistency", "n": 128.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.09375, "executor_target_mass": 0.012367482180707157, "init_accuracy": 0.890625, "op_accuracy": 0.9010416666666666, "arg_accuracy": 0.8365885416666666, "program_exact": 0.078125, "state_accuracy": 0.6630859375, "state_all_exact": 0.078125, "state_prefix_fraction": 0.6572265625, "pair_true_answer_consistency": 1.0, "executor_pair_answer_consistency": 0.03125, "executor_pair_both_correct": 0.03125, "compiler_pair_program_consistency": 0.03125, "compiler_pair_state_consistency": 0.03125 } } } } }