{ "args": { "model_id": "Qwen/Qwen3-4B", "load_in_4bit": true, "torch_dtype": "bf16", "device_map": "auto", "use_lora": true, "lora_r": 4, "lora_alpha": 8, "lora_dropout": 0.05, "lora_target_modules": "all-linear", "gradient_checkpointing": true, "optimizer": "paged_adamw_8bit", "modulus": 97, "max_steps": 4, "train_min_len": 1, "train_max_len": 12, "curriculum_stages": "short:1:2:2,long:2:4:2", "answer_train_min_len": 1, "answer_train_max_len": 24, "eval_lengths": "2,4", "train_template_mode": "mixed", "paired_train": false, "paired_batches": true, "paired_template_modes": "standard,paraphrase", "pair_size": 2, "answer_train_template_mode": "mixed", "eval_template_modes": "standard,paraphrase", "paired_eval": true, "paired_eval_template_modes": "standard,paraphrase", "train_size": 16, "answer_train_size": 16, "eval_size": 4, "train_batch_size": 2, "eval_batch_size": 2, "bootstrap_steps": 0, "answer_steps": 0, "eval_every": 2, "stage_eval_every": 2, "lr": 0.0002, "answer_lr_scale": 0.1, "weight_decay": 0.0, "grad_clip": 1.0, "head_width": 128, "rank_temperature": 1.0, "arg_reader_mode": "after_op", "arg_window": 8, "arg_distance_temperature": 2.0, "trace_loss_weight": 1.0, "init_trace_loss_weight": 4.0, "op_trace_loss_weight": 1.0, "arg_trace_loss_weight": 4.0, "selection_loss_weight": 1.0, "role_pos_weight": 20.0, "init_selection_loss_weight": 1.0, "op_selection_loss_weight": 1.0, "arg_selection_loss_weight": 4.0, "executor_loss_weight": 1.0, "state_loss_weight": 0.25, "state_loss_schedule": "constant", "save_eval_checkpoints": true, "selection_metric": "paired_len4_repair_executor_accuracy", "variants": "copy_trace_state_repair", "repair_eval": true, "repair_verifier_mode": "state", "repair_topk": 3, "repair_max_edits": 2, "repair_max_pair_arg_slots": 24, "repair_include_init": true, "repair_include_ops": true, "repair_include_args": true, "repair_same_step_op_arg": true, "max_length": 192, "seed": 7, "train_seed": -1, "output_dir": "experiments/qwen_verifier_guided_slot_repair/runs/smoke_qwen3_4b_repair", "checkpoint_dir": "", "state_loss_weight_by_stage": { "short": 0.25, "long": 0.25 } }, "metadata": { "python": "3.12.3", "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "torch": "2.8.0+cu128", "cuda_available": true, "transformers_loader": "AutoModelForCausalLM", "peft_installed": true, "use_lora": true, "lora_r": 4, "gpu_name": "NVIDIA RTX 6000 Ada Generation", "gpu_vram_gb": 47.383 }, "hidden_dim": 2560, "dataset": { "train_size": 16, "train_examples": 32, "curriculum_stages": [ { "name": "short", "min_len": 1, "max_len": 2, "steps": 2 }, { "name": "long", "min_len": 2, "max_len": 4, "steps": 2 } ], "state_loss_weight_by_stage": { "short": 0.25, "long": 0.25 }, "answer_train_size": 16, "eval_size": 4, "train_lengths": [ 1, 12 ], "answer_train_lengths": [ 1, 24 ], "eval_lengths": [ 2, 4 ], "train_template_mode": "mixed", "paired_train": false, "paired_batches": true, "paired_template_modes": [ "standard", "paraphrase" ], "answer_train_template_mode": "mixed", "eval_template_modes": [ "standard", "paraphrase" ], "paired_eval": true, "paired_eval_template_modes": [ "standard", "paraphrase" ] }, "variants": { "copy_trace_state_repair": { "variant": "copy_trace_state_repair", "train_seconds": 13.284536123275757, "checkpoints": [ { "variant": "copy_trace_state_repair", "tag": "eval_trace_state_repair_short", "step": 1, "checkpoint_dir": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00001", "model_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00001/adapter", "heads_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00001/heads.pt", "selection_metric": "paired_len4_repair_executor_accuracy", "selection_value": 0.5 }, { "variant": "copy_trace_state_repair", "tag": "eval_trace_state_repair_short", "step": 2, "checkpoint_dir": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00002", "model_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00002/adapter", "heads_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00002/heads.pt", "selection_metric": "paired_len4_repair_executor_accuracy", "selection_value": 0.125 }, { "variant": "copy_trace_state_repair", "tag": "eval_trace_state_repair_long", "step": 3, "checkpoint_dir": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_long_step00003", "model_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_long_step00003/adapter", "heads_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_long_step00003/heads.pt", "selection_metric": "paired_len4_repair_executor_accuracy", "selection_value": 0.0 }, { "variant": "copy_trace_state_repair", "tag": "eval_trace_state_repair_long", "step": 4, "checkpoint_dir": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_long_step00004", "model_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_long_step00004/adapter", "heads_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_long_step00004/heads.pt", "selection_metric": "paired_len4_repair_executor_accuracy", "selection_value": 0.0 }, { "variant": "copy_trace_state_repair", "tag": "final", "step": 4, "checkpoint_dir": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/final_step00004", "model_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/final_step00004/adapter", "heads_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/final_step00004/heads.pt", "selection_metric": "paired_len4_repair_executor_accuracy", "selection_value": 0.0 } ], "selected_checkpoint": { "variant": "copy_trace_state_repair", "tag": "eval_trace_state_repair_short", "step": 1, "checkpoint_dir": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00001", "model_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00001/adapter", "heads_checkpoint": "large_artifacts/qwen_verifier_guided_slot_repair/checkpoints/smoke_qwen3_4b_repair/copy_trace_state_repair/eval_trace_state_repair_short_step00001/heads.pt", "selection_metric": "paired_len4_repair_executor_accuracy", "selection_value": 0.5 }, "train_log": [ { "variant": "copy_trace_state_repair", "stage": "trace_state_repair_short", "local_step": 1, "step": 1, "loss": 34.591251373291016, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "state_loss_weight": 0.25, "lr_scale": 1.0, "executor_loss": 2.8669564723968506, "state_loss": 2.4506173133850098, "state_train_accuracy": 0.0, "init_loss": 1.0965834856033325, "op_loss": 0.0, "arg_loss": 0.9611554145812988, "init_attn_loss": 3.8201065063476562, "op_attn_loss": 3.277343273162842, "arg_attn_loss": 3.446834087371826, "role_loss": 1.9958983659744263, "standard_len2_direct_accuracy": NaN, "standard_len2_direct_target_mass": NaN, "standard_len2_executor_accuracy": 0.0, "standard_len2_executor_target_mass": 0.08580778166651726, "standard_len2_init_accuracy": 0.25, "standard_len2_init_pos_accuracy": 0.0, "standard_len2_op_accuracy": 1.0, "standard_len2_arg_accuracy": 0.5, "standard_len2_op_pos_accuracy": 0.0, "standard_len2_arg_pos_accuracy": 0.0, "standard_len2_program_exact": 0.0, "standard_len2_state_accuracy": 0.0, "standard_len2_state_all_exact": 0.0, "standard_len2_state_prefix_fraction": 0.0, "standard_len2_state_mean_correct_prefix": 0.0, "standard_len2_repair_executor_accuracy": 0.75, "standard_len2_repair_program_exact": 0.0, "standard_len2_repair_state_accuracy": 0.75, "standard_len2_repair_state_all_exact": 0.75, "standard_len2_repair_state_prefix_fraction": 0.75, "standard_len2_repair_state_mean_correct_prefix": 1.5, "standard_len2_repair_changed_fraction": 0.75, "standard_len2_repair_found_fraction": 0.75, "standard_len2_repair_avg_candidates": 23.0, "standard_len2_repair_avg_verified_candidates": 1.0, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.010629043448716402, "standard_len4_init_accuracy": 0.0, "standard_len4_init_pos_accuracy": 0.0, "standard_len4_op_accuracy": 0.75, "standard_len4_arg_accuracy": 0.4375, "standard_len4_op_pos_accuracy": 0.0, "standard_len4_arg_pos_accuracy": 0.0, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.0, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.0, "standard_len4_state_mean_correct_prefix": 0.0, "standard_len4_repair_executor_accuracy": 0.0, "standard_len4_repair_program_exact": 0.0, "standard_len4_repair_state_accuracy": 0.0, "standard_len4_repair_state_all_exact": 0.0, "standard_len4_repair_state_prefix_fraction": 0.0, "standard_len4_repair_state_mean_correct_prefix": 0.0, "standard_len4_repair_changed_fraction": 0.0, "standard_len4_repair_found_fraction": 0.0, "standard_len4_repair_avg_candidates": 59.0, "standard_len4_repair_avg_verified_candidates": 0.0, "paraphrase_len2_direct_accuracy": NaN, "paraphrase_len2_direct_target_mass": NaN, "paraphrase_len2_executor_accuracy": 0.25, "paraphrase_len2_executor_target_mass": 0.08631835132837296, "paraphrase_len2_init_accuracy": 0.5, "paraphrase_len2_init_pos_accuracy": 0.0, "paraphrase_len2_op_accuracy": 1.0, "paraphrase_len2_arg_accuracy": 0.5, "paraphrase_len2_op_pos_accuracy": 0.0, "paraphrase_len2_arg_pos_accuracy": 0.125, "paraphrase_len2_program_exact": 0.0, "paraphrase_len2_state_accuracy": 0.25, "paraphrase_len2_state_all_exact": 0.25, "paraphrase_len2_state_prefix_fraction": 0.25, "paraphrase_len2_state_mean_correct_prefix": 0.5, "paraphrase_len2_repair_executor_accuracy": 1.0, "paraphrase_len2_repair_program_exact": 0.25, "paraphrase_len2_repair_state_accuracy": 1.0, "paraphrase_len2_repair_state_all_exact": 1.0, "paraphrase_len2_repair_state_prefix_fraction": 1.0, "paraphrase_len2_repair_state_mean_correct_prefix": 2.0, "paraphrase_len2_repair_changed_fraction": 0.75, "paraphrase_len2_repair_found_fraction": 1.0, "paraphrase_len2_repair_avg_candidates": 23.0, "paraphrase_len2_repair_avg_verified_candidates": 2.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.012707134708762169, "paraphrase_len4_init_accuracy": 0.75, "paraphrase_len4_init_pos_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.6875, "paraphrase_len4_arg_accuracy": 0.625, "paraphrase_len4_op_pos_accuracy": 0.0, "paraphrase_len4_arg_pos_accuracy": 0.0, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.0625, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.0625, "paraphrase_len4_state_mean_correct_prefix": 0.25, "paraphrase_len4_repair_executor_accuracy": 0.0, "paraphrase_len4_repair_program_exact": 0.0, "paraphrase_len4_repair_state_accuracy": 0.0625, "paraphrase_len4_repair_state_all_exact": 0.0, "paraphrase_len4_repair_state_prefix_fraction": 0.0625, "paraphrase_len4_repair_state_mean_correct_prefix": 0.25, "paraphrase_len4_repair_changed_fraction": 0.0, "paraphrase_len4_repair_found_fraction": 0.0, "paraphrase_len4_repair_avg_candidates": 59.0, "paraphrase_len4_repair_avg_verified_candidates": 0.0, "paired_len2_direct_accuracy": NaN, "paired_len2_direct_target_mass": NaN, "paired_len2_executor_accuracy": 0.0, "paired_len2_executor_target_mass": 0.05962490662932396, "paired_len2_init_accuracy": 0.5, "paired_len2_init_pos_accuracy": 0.0, "paired_len2_op_accuracy": 0.9375, "paired_len2_arg_accuracy": 0.25, "paired_len2_op_pos_accuracy": 0.0, "paired_len2_arg_pos_accuracy": 0.0, "paired_len2_program_exact": 0.0, "paired_len2_state_accuracy": 0.0, "paired_len2_state_all_exact": 0.0, "paired_len2_state_prefix_fraction": 0.0, "paired_len2_state_mean_correct_prefix": 0.0, "paired_len2_repair_executor_accuracy": 0.875, "paired_len2_repair_program_exact": 0.375, "paired_len2_repair_state_accuracy": 0.875, "paired_len2_repair_state_all_exact": 0.875, "paired_len2_repair_state_prefix_fraction": 0.875, "paired_len2_repair_state_mean_correct_prefix": 1.75, "paired_len2_repair_changed_fraction": 0.875, "paired_len2_repair_found_fraction": 0.875, "paired_len2_repair_avg_candidates": 23.0, "paired_len2_repair_avg_verified_candidates": 0.875, "paired_len2_pair_true_answer_consistency": 1.0, "paired_len2_pair_true_length_consistency": 1.0, "paired_len2_executor_pair_answer_consistency": 0.75, "paired_len2_executor_pair_both_correct": 0.0, "paired_len2_compiler_pair_program_consistency": 0.75, "paired_len2_compiler_pair_state_consistency": 0.75, "paired_len2_repair_pair_answer_consistency": 0.75, "paired_len2_repair_pair_both_correct": 0.75, "paired_len2_repair_pair_program_consistency": 0.75, "paired_len2_repair_pair_state_consistency": 0.75, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.037523571867495775, "paired_len4_init_accuracy": 0.375, "paired_len4_init_pos_accuracy": 0.0, "paired_len4_op_accuracy": 0.8125, "paired_len4_arg_accuracy": 0.5, "paired_len4_op_pos_accuracy": 0.0, "paired_len4_arg_pos_accuracy": 0.0, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.0625, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0625, "paired_len4_state_mean_correct_prefix": 0.25, "paired_len4_repair_executor_accuracy": 0.5, "paired_len4_repair_program_exact": 0.25, "paired_len4_repair_state_accuracy": 0.5, "paired_len4_repair_state_all_exact": 0.5, "paired_len4_repair_state_prefix_fraction": 0.5, "paired_len4_repair_state_mean_correct_prefix": 2.0, "paired_len4_repair_changed_fraction": 0.5, "paired_len4_repair_found_fraction": 0.5, "paired_len4_repair_avg_candidates": 59.0, "paired_len4_repair_avg_verified_candidates": 0.5, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_pair_true_length_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.25, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.25, "paired_len4_compiler_pair_state_consistency": 0.25, "paired_len4_repair_pair_answer_consistency": 0.5, "paired_len4_repair_pair_both_correct": 0.5, "paired_len4_repair_pair_program_consistency": 0.5, "paired_len4_repair_pair_state_consistency": 0.5 }, { "variant": "copy_trace_state_repair", "stage": "trace_state_repair_short", "local_step": 2, "step": 2, "loss": 36.41120910644531, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "state_loss_weight": 0.25, "lr_scale": 1.0, "executor_loss": 3.2947421073913574, "state_loss": 2.971410036087036, "state_train_accuracy": 0.0, "init_loss": 0.7973906993865967, "op_loss": 0.44011881947517395, "arg_loss": 1.224782109260559, "init_attn_loss": 3.7258458137512207, "op_attn_loss": 3.509270668029785, "arg_attn_loss": 3.721574068069458, "role_loss": 1.7233928442001343, "standard_len2_direct_accuracy": NaN, "standard_len2_direct_target_mass": NaN, "standard_len2_executor_accuracy": 0.0, "standard_len2_executor_target_mass": 0.0878305546939373, "standard_len2_init_accuracy": 0.5, "standard_len2_init_pos_accuracy": 0.0, "standard_len2_op_accuracy": 1.0, "standard_len2_arg_accuracy": 0.5, "standard_len2_op_pos_accuracy": 0.0, "standard_len2_arg_pos_accuracy": 0.0, "standard_len2_program_exact": 0.0, "standard_len2_state_accuracy": 0.0, "standard_len2_state_all_exact": 0.0, "standard_len2_state_prefix_fraction": 0.0, "standard_len2_state_mean_correct_prefix": 0.0, "standard_len2_repair_executor_accuracy": 0.75, "standard_len2_repair_program_exact": 0.5, "standard_len2_repair_state_accuracy": 0.75, "standard_len2_repair_state_all_exact": 0.75, "standard_len2_repair_state_prefix_fraction": 0.75, "standard_len2_repair_state_mean_correct_prefix": 1.5, "standard_len2_repair_changed_fraction": 0.75, "standard_len2_repair_found_fraction": 0.75, "standard_len2_repair_avg_candidates": 23.0, "standard_len2_repair_avg_verified_candidates": 1.25, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.008664404973387718, "standard_len4_init_accuracy": 0.25, "standard_len4_init_pos_accuracy": 0.0, "standard_len4_op_accuracy": 0.75, "standard_len4_arg_accuracy": 0.5, "standard_len4_op_pos_accuracy": 0.0, "standard_len4_arg_pos_accuracy": 0.0, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.0625, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.0625, "standard_len4_state_mean_correct_prefix": 0.25, "standard_len4_repair_executor_accuracy": 0.0, "standard_len4_repair_program_exact": 0.0, "standard_len4_repair_state_accuracy": 0.0625, "standard_len4_repair_state_all_exact": 0.0, "standard_len4_repair_state_prefix_fraction": 0.0625, "standard_len4_repair_state_mean_correct_prefix": 0.25, "standard_len4_repair_changed_fraction": 0.0, "standard_len4_repair_found_fraction": 0.0, "standard_len4_repair_avg_candidates": 59.0, "standard_len4_repair_avg_verified_candidates": 0.0, "paraphrase_len2_direct_accuracy": NaN, "paraphrase_len2_direct_target_mass": NaN, "paraphrase_len2_executor_accuracy": 0.5, "paraphrase_len2_executor_target_mass": 0.08819271996617317, "paraphrase_len2_init_accuracy": 0.5, "paraphrase_len2_init_pos_accuracy": 0.0, "paraphrase_len2_op_accuracy": 1.0, "paraphrase_len2_arg_accuracy": 0.625, "paraphrase_len2_op_pos_accuracy": 0.25, "paraphrase_len2_arg_pos_accuracy": 0.125, "paraphrase_len2_program_exact": 0.25, "paraphrase_len2_state_accuracy": 0.5, "paraphrase_len2_state_all_exact": 0.5, "paraphrase_len2_state_prefix_fraction": 0.5, "paraphrase_len2_state_mean_correct_prefix": 1.0, "paraphrase_len2_repair_executor_accuracy": 1.0, "paraphrase_len2_repair_program_exact": 0.5, "paraphrase_len2_repair_state_accuracy": 1.0, "paraphrase_len2_repair_state_all_exact": 1.0, "paraphrase_len2_repair_state_prefix_fraction": 1.0, "paraphrase_len2_repair_state_mean_correct_prefix": 2.0, "paraphrase_len2_repair_changed_fraction": 0.5, "paraphrase_len2_repair_found_fraction": 1.0, "paraphrase_len2_repair_avg_candidates": 23.0, "paraphrase_len2_repair_avg_verified_candidates": 1.75, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.010785003192722797, "paraphrase_len4_init_accuracy": 1.0, "paraphrase_len4_init_pos_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.625, "paraphrase_len4_arg_accuracy": 0.375, "paraphrase_len4_op_pos_accuracy": 0.0, "paraphrase_len4_arg_pos_accuracy": 0.0625, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.0, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.0, "paraphrase_len4_state_mean_correct_prefix": 0.0, "paraphrase_len4_repair_executor_accuracy": 0.0, "paraphrase_len4_repair_program_exact": 0.0, "paraphrase_len4_repair_state_accuracy": 0.0, "paraphrase_len4_repair_state_all_exact": 0.0, "paraphrase_len4_repair_state_prefix_fraction": 0.0, "paraphrase_len4_repair_state_mean_correct_prefix": 0.0, "paraphrase_len4_repair_changed_fraction": 0.0, "paraphrase_len4_repair_found_fraction": 0.0, "paraphrase_len4_repair_avg_candidates": 59.0, "paraphrase_len4_repair_avg_verified_candidates": 0.0, "paired_len2_direct_accuracy": NaN, "paired_len2_direct_target_mass": NaN, "paired_len2_executor_accuracy": 0.0, "paired_len2_executor_target_mass": 0.05796311795711517, "paired_len2_init_accuracy": 0.75, "paired_len2_init_pos_accuracy": 0.0, "paired_len2_op_accuracy": 1.0, "paired_len2_arg_accuracy": 0.375, "paired_len2_op_pos_accuracy": 0.125, "paired_len2_arg_pos_accuracy": 0.25, "paired_len2_program_exact": 0.0, "paired_len2_state_accuracy": 0.125, "paired_len2_state_all_exact": 0.0, "paired_len2_state_prefix_fraction": 0.125, "paired_len2_state_mean_correct_prefix": 0.25, "paired_len2_repair_executor_accuracy": 0.875, "paired_len2_repair_program_exact": 0.625, "paired_len2_repair_state_accuracy": 0.875, "paired_len2_repair_state_all_exact": 0.875, "paired_len2_repair_state_prefix_fraction": 0.875, "paired_len2_repair_state_mean_correct_prefix": 1.75, "paired_len2_repair_changed_fraction": 0.875, "paired_len2_repair_found_fraction": 0.875, "paired_len2_repair_avg_candidates": 23.0, "paired_len2_repair_avg_verified_candidates": 1.25, "paired_len2_pair_true_answer_consistency": 1.0, "paired_len2_pair_true_length_consistency": 1.0, "paired_len2_executor_pair_answer_consistency": 0.25, "paired_len2_executor_pair_both_correct": 0.0, "paired_len2_compiler_pair_program_consistency": 0.25, "paired_len2_compiler_pair_state_consistency": 0.25, "paired_len2_repair_pair_answer_consistency": 0.75, "paired_len2_repair_pair_both_correct": 0.75, "paired_len2_repair_pair_program_consistency": 0.5, "paired_len2_repair_pair_state_consistency": 0.75, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.03032680111937225, "paired_len4_init_accuracy": 0.5, "paired_len4_init_pos_accuracy": 0.0, "paired_len4_op_accuracy": 0.75, "paired_len4_arg_accuracy": 0.46875, "paired_len4_op_pos_accuracy": 0.0625, "paired_len4_arg_pos_accuracy": 0.0625, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.03125, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.03125, "paired_len4_state_mean_correct_prefix": 0.125, "paired_len4_repair_executor_accuracy": 0.125, "paired_len4_repair_program_exact": 0.0, "paired_len4_repair_state_accuracy": 0.125, "paired_len4_repair_state_all_exact": 0.125, "paired_len4_repair_state_prefix_fraction": 0.125, "paired_len4_repair_state_mean_correct_prefix": 0.5, "paired_len4_repair_changed_fraction": 0.125, "paired_len4_repair_found_fraction": 0.125, "paired_len4_repair_avg_candidates": 59.0, "paired_len4_repair_avg_verified_candidates": 0.125, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_pair_true_length_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.0, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.0, "paired_len4_compiler_pair_state_consistency": 0.0, "paired_len4_repair_pair_answer_consistency": 0.0, "paired_len4_repair_pair_both_correct": 0.0, "paired_len4_repair_pair_program_consistency": 0.0, "paired_len4_repair_pair_state_consistency": 0.0 }, { "variant": "copy_trace_state_repair", "stage": "trace_state_repair_long", "local_step": 1, "step": 3, "loss": 34.99763107299805, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "state_loss_weight": 0.25, "lr_scale": 1.0, "executor_loss": 2.797220230102539, "state_loss": 2.3146872520446777, "state_train_accuracy": 0.6000000238418579, "init_loss": 1.2220213413238525, "op_loss": 0.3618469536304474, "arg_loss": 0.8457635641098022, "init_attn_loss": 3.95582914352417, "op_attn_loss": 3.3689022064208984, "arg_attn_loss": 3.3971798419952393, "role_loss": 2.0753026008605957, "standard_len2_direct_accuracy": NaN, "standard_len2_direct_target_mass": NaN, "standard_len2_executor_accuracy": 0.0, "standard_len2_executor_target_mass": 0.08821655251085758, "standard_len2_init_accuracy": 0.5, "standard_len2_init_pos_accuracy": 0.0, "standard_len2_op_accuracy": 1.0, "standard_len2_arg_accuracy": 0.5, "standard_len2_op_pos_accuracy": 0.0, "standard_len2_arg_pos_accuracy": 0.5, "standard_len2_program_exact": 0.0, "standard_len2_state_accuracy": 0.0, "standard_len2_state_all_exact": 0.0, "standard_len2_state_prefix_fraction": 0.0, "standard_len2_state_mean_correct_prefix": 0.0, "standard_len2_repair_executor_accuracy": 0.75, "standard_len2_repair_program_exact": 0.5, "standard_len2_repair_state_accuracy": 0.75, "standard_len2_repair_state_all_exact": 0.75, "standard_len2_repair_state_prefix_fraction": 0.75, "standard_len2_repair_state_mean_correct_prefix": 1.5, "standard_len2_repair_changed_fraction": 0.75, "standard_len2_repair_found_fraction": 0.75, "standard_len2_repair_avg_candidates": 23.0, "standard_len2_repair_avg_verified_candidates": 1.25, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.007841526996344328, "standard_len4_init_accuracy": 0.75, "standard_len4_init_pos_accuracy": 0.0, "standard_len4_op_accuracy": 0.625, "standard_len4_arg_accuracy": 0.4375, "standard_len4_op_pos_accuracy": 0.0, "standard_len4_arg_pos_accuracy": 0.1875, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.0, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.0, "standard_len4_state_mean_correct_prefix": 0.0, "standard_len4_repair_executor_accuracy": 0.0, "standard_len4_repair_program_exact": 0.0, "standard_len4_repair_state_accuracy": 0.0, "standard_len4_repair_state_all_exact": 0.0, "standard_len4_repair_state_prefix_fraction": 0.0, "standard_len4_repair_state_mean_correct_prefix": 0.0, "standard_len4_repair_changed_fraction": 0.0, "standard_len4_repair_found_fraction": 0.0, "standard_len4_repair_avg_candidates": 59.0, "standard_len4_repair_avg_verified_candidates": 0.0, "paraphrase_len2_direct_accuracy": NaN, "paraphrase_len2_direct_target_mass": NaN, "paraphrase_len2_executor_accuracy": 0.25, "paraphrase_len2_executor_target_mass": 0.08595943450927734, "paraphrase_len2_init_accuracy": 0.75, "paraphrase_len2_init_pos_accuracy": 0.0, "paraphrase_len2_op_accuracy": 1.0, "paraphrase_len2_arg_accuracy": 0.625, "paraphrase_len2_op_pos_accuracy": 0.25, "paraphrase_len2_arg_pos_accuracy": 0.5, "paraphrase_len2_program_exact": 0.25, "paraphrase_len2_state_accuracy": 0.25, "paraphrase_len2_state_all_exact": 0.25, "paraphrase_len2_state_prefix_fraction": 0.25, "paraphrase_len2_state_mean_correct_prefix": 0.5, "paraphrase_len2_repair_executor_accuracy": 1.0, "paraphrase_len2_repair_program_exact": 0.75, "paraphrase_len2_repair_state_accuracy": 1.0, "paraphrase_len2_repair_state_all_exact": 1.0, "paraphrase_len2_repair_state_prefix_fraction": 1.0, "paraphrase_len2_repair_state_mean_correct_prefix": 2.0, "paraphrase_len2_repair_changed_fraction": 0.75, "paraphrase_len2_repair_found_fraction": 1.0, "paraphrase_len2_repair_avg_candidates": 23.0, "paraphrase_len2_repair_avg_verified_candidates": 1.75, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.010633292607963085, "paraphrase_len4_init_accuracy": 1.0, "paraphrase_len4_init_pos_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.625, "paraphrase_len4_arg_accuracy": 0.5, "paraphrase_len4_op_pos_accuracy": 0.0, "paraphrase_len4_arg_pos_accuracy": 0.0625, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.1875, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.1875, "paraphrase_len4_state_mean_correct_prefix": 0.75, "paraphrase_len4_repair_executor_accuracy": 0.0, "paraphrase_len4_repair_program_exact": 0.0, "paraphrase_len4_repair_state_accuracy": 0.1875, "paraphrase_len4_repair_state_all_exact": 0.0, "paraphrase_len4_repair_state_prefix_fraction": 0.1875, "paraphrase_len4_repair_state_mean_correct_prefix": 0.75, "paraphrase_len4_repair_changed_fraction": 0.0, "paraphrase_len4_repair_found_fraction": 0.0, "paraphrase_len4_repair_avg_candidates": 59.0, "paraphrase_len4_repair_avg_verified_candidates": 0.0, "paired_len2_direct_accuracy": NaN, "paired_len2_direct_target_mass": NaN, "paired_len2_executor_accuracy": 0.25, "paired_len2_executor_target_mass": 0.05538773164153099, "paired_len2_init_accuracy": 0.75, "paired_len2_init_pos_accuracy": 0.0, "paired_len2_op_accuracy": 1.0, "paired_len2_arg_accuracy": 0.5625, "paired_len2_op_pos_accuracy": 0.1875, "paired_len2_arg_pos_accuracy": 0.375, "paired_len2_program_exact": 0.25, "paired_len2_state_accuracy": 0.4375, "paired_len2_state_all_exact": 0.25, "paired_len2_state_prefix_fraction": 0.4375, "paired_len2_state_mean_correct_prefix": 0.875, "paired_len2_repair_executor_accuracy": 0.75, "paired_len2_repair_program_exact": 0.75, "paired_len2_repair_state_accuracy": 0.75, "paired_len2_repair_state_all_exact": 0.75, "paired_len2_repair_state_prefix_fraction": 0.75, "paired_len2_repair_state_mean_correct_prefix": 1.5, "paired_len2_repair_changed_fraction": 0.5, "paired_len2_repair_found_fraction": 0.75, "paired_len2_repair_avg_candidates": 23.0, "paired_len2_repair_avg_verified_candidates": 0.875, "paired_len2_pair_true_answer_consistency": 1.0, "paired_len2_pair_true_length_consistency": 1.0, "paired_len2_executor_pair_answer_consistency": 0.75, "paired_len2_executor_pair_both_correct": 0.25, "paired_len2_compiler_pair_program_consistency": 0.75, "paired_len2_compiler_pair_state_consistency": 0.75, "paired_len2_repair_pair_answer_consistency": 1.0, "paired_len2_repair_pair_both_correct": 0.75, "paired_len2_repair_pair_program_consistency": 1.0, "paired_len2_repair_pair_state_consistency": 1.0, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.025159823708236217, "paired_len4_init_accuracy": 0.75, "paired_len4_init_pos_accuracy": 0.0, "paired_len4_op_accuracy": 0.71875, "paired_len4_arg_accuracy": 0.40625, "paired_len4_op_pos_accuracy": 0.03125, "paired_len4_arg_pos_accuracy": 0.1875, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.0625, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0625, "paired_len4_state_mean_correct_prefix": 0.25, "paired_len4_repair_executor_accuracy": 0.0, "paired_len4_repair_program_exact": 0.0, "paired_len4_repair_state_accuracy": 0.0625, "paired_len4_repair_state_all_exact": 0.0, "paired_len4_repair_state_prefix_fraction": 0.0625, "paired_len4_repair_state_mean_correct_prefix": 0.25, "paired_len4_repair_changed_fraction": 0.0, "paired_len4_repair_found_fraction": 0.0, "paired_len4_repair_avg_candidates": 59.0, "paired_len4_repair_avg_verified_candidates": 0.0, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_pair_true_length_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.75, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.75, "paired_len4_compiler_pair_state_consistency": 0.75, "paired_len4_repair_pair_answer_consistency": 0.75, "paired_len4_repair_pair_both_correct": 0.0, "paired_len4_repair_pair_program_consistency": 0.75, "paired_len4_repair_pair_state_consistency": 0.75 }, { "variant": "copy_trace_state_repair", "stage": "trace_state_repair_long", "local_step": 2, "step": 4, "loss": 38.19786071777344, "trace_loss_active": true, "executor_loss_active": true, "state_loss_active": true, "state_loss_weight": 0.25, "lr_scale": 1.0, "executor_loss": 3.874542236328125, "state_loss": 3.4366188049316406, "state_train_accuracy": 0.1428571492433548, "init_loss": 1.302640676498413, "op_loss": 0.5894113183021545, "arg_loss": 0.9705113172531128, "init_attn_loss": 3.9890329837799072, "op_attn_loss": 3.645843505859375, "arg_attn_loss": 3.400242328643799, "role_loss": 2.546299695968628, "standard_len2_direct_accuracy": NaN, "standard_len2_direct_target_mass": NaN, "standard_len2_executor_accuracy": 0.0, "standard_len2_executor_target_mass": 0.08922983705997467, "standard_len2_init_accuracy": 0.5, "standard_len2_init_pos_accuracy": 0.0, "standard_len2_op_accuracy": 1.0, "standard_len2_arg_accuracy": 0.625, "standard_len2_op_pos_accuracy": 0.0, "standard_len2_arg_pos_accuracy": 0.5, "standard_len2_program_exact": 0.0, "standard_len2_state_accuracy": 0.0, "standard_len2_state_all_exact": 0.0, "standard_len2_state_prefix_fraction": 0.0, "standard_len2_state_mean_correct_prefix": 0.0, "standard_len2_repair_executor_accuracy": 0.75, "standard_len2_repair_program_exact": 0.75, "standard_len2_repair_state_accuracy": 0.75, "standard_len2_repair_state_all_exact": 0.75, "standard_len2_repair_state_prefix_fraction": 0.75, "standard_len2_repair_state_mean_correct_prefix": 1.5, "standard_len2_repair_changed_fraction": 0.75, "standard_len2_repair_found_fraction": 0.75, "standard_len2_repair_avg_candidates": 23.0, "standard_len2_repair_avg_verified_candidates": 1.5, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.007719352142885327, "standard_len4_init_accuracy": 0.75, "standard_len4_init_pos_accuracy": 0.0, "standard_len4_op_accuracy": 0.625, "standard_len4_arg_accuracy": 0.4375, "standard_len4_op_pos_accuracy": 0.0, "standard_len4_arg_pos_accuracy": 0.25, "standard_len4_program_exact": 0.0, "standard_len4_state_accuracy": 0.0, "standard_len4_state_all_exact": 0.0, "standard_len4_state_prefix_fraction": 0.0, "standard_len4_state_mean_correct_prefix": 0.0, "standard_len4_repair_executor_accuracy": 0.0, "standard_len4_repair_program_exact": 0.0, "standard_len4_repair_state_accuracy": 0.0, "standard_len4_repair_state_all_exact": 0.0, "standard_len4_repair_state_prefix_fraction": 0.0, "standard_len4_repair_state_mean_correct_prefix": 0.0, "standard_len4_repair_changed_fraction": 0.0, "standard_len4_repair_found_fraction": 0.0, "standard_len4_repair_avg_candidates": 59.0, "standard_len4_repair_avg_verified_candidates": 0.0, "paraphrase_len2_direct_accuracy": NaN, "paraphrase_len2_direct_target_mass": NaN, "paraphrase_len2_executor_accuracy": 0.25, "paraphrase_len2_executor_target_mass": 0.08601406589150429, "paraphrase_len2_init_accuracy": 0.75, "paraphrase_len2_init_pos_accuracy": 0.0, "paraphrase_len2_op_accuracy": 1.0, "paraphrase_len2_arg_accuracy": 0.625, "paraphrase_len2_op_pos_accuracy": 0.25, "paraphrase_len2_arg_pos_accuracy": 0.5, "paraphrase_len2_program_exact": 0.25, "paraphrase_len2_state_accuracy": 0.25, "paraphrase_len2_state_all_exact": 0.25, "paraphrase_len2_state_prefix_fraction": 0.25, "paraphrase_len2_state_mean_correct_prefix": 0.5, "paraphrase_len2_repair_executor_accuracy": 1.0, "paraphrase_len2_repair_program_exact": 0.75, "paraphrase_len2_repair_state_accuracy": 1.0, "paraphrase_len2_repair_state_all_exact": 1.0, "paraphrase_len2_repair_state_prefix_fraction": 1.0, "paraphrase_len2_repair_state_mean_correct_prefix": 2.0, "paraphrase_len2_repair_changed_fraction": 0.75, "paraphrase_len2_repair_found_fraction": 1.0, "paraphrase_len2_repair_avg_candidates": 23.0, "paraphrase_len2_repair_avg_verified_candidates": 1.75, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0, "paraphrase_len4_executor_target_mass": 0.01068304292857647, "paraphrase_len4_init_accuracy": 1.0, "paraphrase_len4_init_pos_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.625, "paraphrase_len4_arg_accuracy": 0.4375, "paraphrase_len4_op_pos_accuracy": 0.0, "paraphrase_len4_arg_pos_accuracy": 0.125, "paraphrase_len4_program_exact": 0.0, "paraphrase_len4_state_accuracy": 0.25, "paraphrase_len4_state_all_exact": 0.0, "paraphrase_len4_state_prefix_fraction": 0.25, "paraphrase_len4_state_mean_correct_prefix": 1.0, "paraphrase_len4_repair_executor_accuracy": 0.25, "paraphrase_len4_repair_program_exact": 0.0, "paraphrase_len4_repair_state_accuracy": 0.375, "paraphrase_len4_repair_state_all_exact": 0.25, "paraphrase_len4_repair_state_prefix_fraction": 0.375, "paraphrase_len4_repair_state_mean_correct_prefix": 1.5, "paraphrase_len4_repair_changed_fraction": 0.25, "paraphrase_len4_repair_found_fraction": 0.25, "paraphrase_len4_repair_avg_candidates": 59.0, "paraphrase_len4_repair_avg_verified_candidates": 0.25, "paired_len2_direct_accuracy": NaN, "paired_len2_direct_target_mass": NaN, "paired_len2_executor_accuracy": 0.375, "paired_len2_executor_target_mass": 0.05530170165002346, "paired_len2_init_accuracy": 0.75, "paired_len2_init_pos_accuracy": 0.0, "paired_len2_op_accuracy": 1.0, "paired_len2_arg_accuracy": 0.6875, "paired_len2_op_pos_accuracy": 0.1875, "paired_len2_arg_pos_accuracy": 0.5, "paired_len2_program_exact": 0.375, "paired_len2_state_accuracy": 0.5625, "paired_len2_state_all_exact": 0.375, "paired_len2_state_prefix_fraction": 0.5625, "paired_len2_state_mean_correct_prefix": 1.125, "paired_len2_repair_executor_accuracy": 0.75, "paired_len2_repair_program_exact": 0.75, "paired_len2_repair_state_accuracy": 0.75, "paired_len2_repair_state_all_exact": 0.75, "paired_len2_repair_state_prefix_fraction": 0.75, "paired_len2_repair_state_mean_correct_prefix": 1.5, "paired_len2_repair_changed_fraction": 0.375, "paired_len2_repair_found_fraction": 0.75, "paired_len2_repair_avg_candidates": 23.0, "paired_len2_repair_avg_verified_candidates": 1.0, "paired_len2_pair_true_answer_consistency": 1.0, "paired_len2_pair_true_length_consistency": 1.0, "paired_len2_executor_pair_answer_consistency": 0.75, "paired_len2_executor_pair_both_correct": 0.25, "paired_len2_compiler_pair_program_consistency": 0.75, "paired_len2_compiler_pair_state_consistency": 0.75, "paired_len2_repair_pair_answer_consistency": 1.0, "paired_len2_repair_pair_both_correct": 0.75, "paired_len2_repair_pair_program_consistency": 1.0, "paired_len2_repair_pair_state_consistency": 1.0, "paired_len4_direct_accuracy": NaN, "paired_len4_direct_target_mass": NaN, "paired_len4_executor_accuracy": 0.0, "paired_len4_executor_target_mass": 0.02413753792643547, "paired_len4_init_accuracy": 0.75, "paired_len4_init_pos_accuracy": 0.0, "paired_len4_op_accuracy": 0.71875, "paired_len4_arg_accuracy": 0.40625, "paired_len4_op_pos_accuracy": 0.03125, "paired_len4_arg_pos_accuracy": 0.28125, "paired_len4_program_exact": 0.0, "paired_len4_state_accuracy": 0.0625, "paired_len4_state_all_exact": 0.0, "paired_len4_state_prefix_fraction": 0.0625, "paired_len4_state_mean_correct_prefix": 0.25, "paired_len4_repair_executor_accuracy": 0.0, "paired_len4_repair_program_exact": 0.0, "paired_len4_repair_state_accuracy": 0.0625, "paired_len4_repair_state_all_exact": 0.0, "paired_len4_repair_state_prefix_fraction": 0.0625, "paired_len4_repair_state_mean_correct_prefix": 0.25, "paired_len4_repair_changed_fraction": 0.0, "paired_len4_repair_found_fraction": 0.0, "paired_len4_repair_avg_candidates": 59.0, "paired_len4_repair_avg_verified_candidates": 0.0, "paired_len4_pair_true_answer_consistency": 1.0, "paired_len4_pair_true_length_consistency": 1.0, "paired_len4_executor_pair_answer_consistency": 0.75, "paired_len4_executor_pair_both_correct": 0.0, "paired_len4_compiler_pair_program_consistency": 0.75, "paired_len4_compiler_pair_state_consistency": 0.75, "paired_len4_repair_pair_answer_consistency": 0.75, "paired_len4_repair_pair_both_correct": 0.0, "paired_len4_repair_pair_program_consistency": 0.75, "paired_len4_repair_pair_state_consistency": 0.75 } ], "final_metrics": { "standard_len2": { "variant": "copy_trace_state_repair", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.08922983705997467, "init_accuracy": 0.5, "init_pos_accuracy": 0.0, "op_accuracy": 1.0, "arg_accuracy": 0.625, "op_pos_accuracy": 0.0, "arg_pos_accuracy": 0.5, "program_exact": 0.0, "state_accuracy": 0.0, "state_all_exact": 0.0, "state_prefix_fraction": 0.0, "state_mean_correct_prefix": 0.0, "repair_executor_accuracy": 0.75, "repair_program_exact": 0.75, "repair_state_accuracy": 0.75, "repair_state_all_exact": 0.75, "repair_state_prefix_fraction": 0.75, "repair_state_mean_correct_prefix": 1.5, "repair_changed_fraction": 0.75, "repair_found_fraction": 0.75, "repair_avg_candidates": 23.0, "repair_avg_verified_candidates": 1.5 }, "standard_len4": { "variant": "copy_trace_state_repair", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.007719352142885327, "init_accuracy": 0.75, "init_pos_accuracy": 0.0, "op_accuracy": 0.625, "arg_accuracy": 0.4375, "op_pos_accuracy": 0.0, "arg_pos_accuracy": 0.25, "program_exact": 0.0, "state_accuracy": 0.0, "state_all_exact": 0.0, "state_prefix_fraction": 0.0, "state_mean_correct_prefix": 0.0, "repair_executor_accuracy": 0.0, "repair_program_exact": 0.0, "repair_state_accuracy": 0.0, "repair_state_all_exact": 0.0, "repair_state_prefix_fraction": 0.0, "repair_state_mean_correct_prefix": 0.0, "repair_changed_fraction": 0.0, "repair_found_fraction": 0.0, "repair_avg_candidates": 59.0, "repair_avg_verified_candidates": 0.0 }, "paraphrase_len2": { "variant": "copy_trace_state_repair", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.25, "executor_target_mass": 0.08601406589150429, "init_accuracy": 0.75, "init_pos_accuracy": 0.0, "op_accuracy": 1.0, "arg_accuracy": 0.625, "op_pos_accuracy": 0.25, "arg_pos_accuracy": 0.5, "program_exact": 0.25, "state_accuracy": 0.25, "state_all_exact": 0.25, "state_prefix_fraction": 0.25, "state_mean_correct_prefix": 0.5, "repair_executor_accuracy": 1.0, "repair_program_exact": 0.75, "repair_state_accuracy": 1.0, "repair_state_all_exact": 1.0, "repair_state_prefix_fraction": 1.0, "repair_state_mean_correct_prefix": 2.0, "repair_changed_fraction": 0.75, "repair_found_fraction": 1.0, "repair_avg_candidates": 23.0, "repair_avg_verified_candidates": 1.75 }, "paraphrase_len4": { "variant": "copy_trace_state_repair", "n": 4.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.01068304292857647, "init_accuracy": 1.0, "init_pos_accuracy": 0.0, "op_accuracy": 0.625, "arg_accuracy": 0.4375, "op_pos_accuracy": 0.0, "arg_pos_accuracy": 0.125, "program_exact": 0.0, "state_accuracy": 0.25, "state_all_exact": 0.0, "state_prefix_fraction": 0.25, "state_mean_correct_prefix": 1.0, "repair_executor_accuracy": 0.25, "repair_program_exact": 0.0, "repair_state_accuracy": 0.375, "repair_state_all_exact": 0.25, "repair_state_prefix_fraction": 0.375, "repair_state_mean_correct_prefix": 1.5, "repair_changed_fraction": 0.25, "repair_found_fraction": 0.25, "repair_avg_candidates": 59.0, "repair_avg_verified_candidates": 0.25 }, "paired_len2": { "variant": "copy_trace_state_repair", "n": 8.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.375, "executor_target_mass": 0.05530170165002346, "init_accuracy": 0.75, "init_pos_accuracy": 0.0, "op_accuracy": 1.0, "arg_accuracy": 0.6875, "op_pos_accuracy": 0.1875, "arg_pos_accuracy": 0.5, "program_exact": 0.375, "state_accuracy": 0.5625, "state_all_exact": 0.375, "state_prefix_fraction": 0.5625, "state_mean_correct_prefix": 1.125, "repair_executor_accuracy": 0.75, "repair_program_exact": 0.75, "repair_state_accuracy": 0.75, "repair_state_all_exact": 0.75, "repair_state_prefix_fraction": 0.75, "repair_state_mean_correct_prefix": 1.5, "repair_changed_fraction": 0.375, "repair_found_fraction": 0.75, "repair_avg_candidates": 23.0, "repair_avg_verified_candidates": 1.0, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.75, "executor_pair_both_correct": 0.25, "compiler_pair_program_consistency": 0.75, "compiler_pair_state_consistency": 0.75, "repair_pair_answer_consistency": 1.0, "repair_pair_both_correct": 0.75, "repair_pair_program_consistency": 1.0, "repair_pair_state_consistency": 1.0 }, "paired_len4": { "variant": "copy_trace_state_repair", "n": 8.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.02413753792643547, "init_accuracy": 0.75, "init_pos_accuracy": 0.0, "op_accuracy": 0.71875, "arg_accuracy": 0.40625, "op_pos_accuracy": 0.03125, "arg_pos_accuracy": 0.28125, "program_exact": 0.0, "state_accuracy": 0.0625, "state_all_exact": 0.0, "state_prefix_fraction": 0.0625, "state_mean_correct_prefix": 0.25, "repair_executor_accuracy": 0.0, "repair_program_exact": 0.0, "repair_state_accuracy": 0.0625, "repair_state_all_exact": 0.0, "repair_state_prefix_fraction": 0.0625, "repair_state_mean_correct_prefix": 0.25, "repair_changed_fraction": 0.0, "repair_found_fraction": 0.0, "repair_avg_candidates": 59.0, "repair_avg_verified_candidates": 0.0, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.75, "executor_pair_both_correct": 0.0, "compiler_pair_program_consistency": 0.75, "compiler_pair_state_consistency": 0.75, "repair_pair_answer_consistency": 0.75, "repair_pair_both_correct": 0.0, "repair_pair_program_consistency": 0.75, "repair_pair_state_consistency": 0.75 } } } } }