{ "args": { "model_id": "Qwen/Qwen3.5-4B", "text_model_fallback": "Qwen/Qwen3-4B", "model_class": "auto", "load_in_4bit": true, "torch_dtype": "bf16", "device_map": "auto", "modulus": 97, "max_steps": 24, "train_min_len": 4, "train_max_len": 4, "answer_train_min_len": 4, "answer_train_max_len": 4, "eval_lengths": "4,8,12,24", "train_template_mode": "standard", "answer_train_template_mode": "standard", "eval_template_modes": "standard,paraphrase", "train_size": 2048, "answer_train_size": 2048, "eval_size": 256, "feature_batch_size": 8, "train_batch_size": 128, "eval_batch_size": 256, "bootstrap_steps": 1000, "answer_steps": 1000, "eval_every": 500, "stage_eval_every": 500, "lr": 0.001, "answer_lr_scale": 0.1, "weight_decay": 0.01, "grad_clip": 1.0, "head_width": 512, "query_mode": "independent", "trace_loss_weight": 1.0, "init_trace_loss_weight": 4.0, "op_trace_loss_weight": 1.0, "arg_trace_loss_weight": 1.0, "attention_loss_weight": 3.0, "executor_loss_weight": 1.0, "variants": "direct,compiler_trace,compiler_answer_only,compiler_trace_then_answer", "max_length": 512, "feature_seq_len": 320, "seed": 7, "output_dir": "experiments/qwen_span_free_compiler/runs/main_qwen35_attention_len4_retention", "checkpoint_dir": "", "verbose": false }, "metadata": { "python": "3.12.3", "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39", "torch": "2.8.0+cu128", "cuda_available": true, "transformers_loader": "AutoModelForMultimodalLM", "peft_installed": true, "gpu_name": "NVIDIA RTX 6000 Ada Generation", "gpu_vram_gb": 47.383 }, "hidden_dim": 2560, "dataset": { "bootstrap_train_size": 2048, "answer_train_size": 2048, "eval_size": 256, "bootstrap_train_lengths": [ 4, 4 ], "answer_train_lengths": [ 4, 4 ], "eval_lengths": [ 4, 8, 12, 24 ], "train_template_mode": "standard", "answer_train_template_mode": "standard", "eval_template_modes": [ "standard", "paraphrase" ], "feature_seq_len": 320 }, "variants": { "direct": { "variant": "direct", "train_seconds": 146.69391131401062, "checkpoints": [ "large_artifacts/qwen_span_free_compiler/checkpoints/main_qwen35_attention_len4_retention/direct/checkpoint_final.pt" ], "train_log": [ { "stage": "answer_only", "local_step": 1, "step": 1, "loss": 4.573838233947754, "trace_loss_active": false, "lr_scale": 1.0, "direct_loss": 4.573838233947754, "standard_len4_direct_accuracy": 0.0, "standard_len4_direct_target_mass": 0.008949733339250088, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len8_direct_accuracy": 0.01171875, "standard_len8_direct_target_mass": 0.010155560448765755, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len12_direct_accuracy": 0.00390625, "standard_len12_direct_target_mass": 0.008634192869067192, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len24_direct_accuracy": 0.01171875, "standard_len24_direct_target_mass": 0.009988012723624706, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "paraphrase_len4_direct_accuracy": 0.015625, "paraphrase_len4_direct_target_mass": 0.01069931872189045, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len8_direct_accuracy": 0.0078125, "paraphrase_len8_direct_target_mass": 0.010645576752722263, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len12_direct_accuracy": 0.015625, "paraphrase_len12_direct_target_mass": 0.010487204417586327, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len24_direct_accuracy": 0.01171875, "paraphrase_len24_direct_target_mass": 0.010785894468426704, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN }, { "stage": "answer_only", "local_step": 500, "step": 500, "loss": 4.5156073570251465, "trace_loss_active": false, "lr_scale": 1.0, "direct_loss": 4.5156073570251465, "standard_len4_direct_accuracy": 0.01953125, "standard_len4_direct_target_mass": 0.010596760548651218, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len8_direct_accuracy": 0.01171875, "standard_len8_direct_target_mass": 0.010171017609536648, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len12_direct_accuracy": 0.0, "standard_len12_direct_target_mass": 0.010157035663723946, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len24_direct_accuracy": 0.00390625, "standard_len24_direct_target_mass": 0.010422471910715103, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "paraphrase_len4_direct_accuracy": 0.015625, "paraphrase_len4_direct_target_mass": 0.010416360571980476, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len8_direct_accuracy": 0.0078125, "paraphrase_len8_direct_target_mass": 0.010431280359625816, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len12_direct_accuracy": 0.0, "paraphrase_len12_direct_target_mass": 0.010418955236673355, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len24_direct_accuracy": 0.00390625, "paraphrase_len24_direct_target_mass": 0.010294370353221893, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN }, { "stage": "answer_only", "local_step": 1000, "step": 1000, "loss": 4.5098981857299805, "trace_loss_active": false, "lr_scale": 1.0, "direct_loss": 4.5098981857299805, "standard_len4_direct_accuracy": 0.00390625, "standard_len4_direct_target_mass": 0.010599544271826744, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len8_direct_accuracy": 0.0078125, "standard_len8_direct_target_mass": 0.010337657295167446, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len12_direct_accuracy": 0.0, "standard_len12_direct_target_mass": 0.010198182426393032, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len24_direct_accuracy": 0.0078125, "standard_len24_direct_target_mass": 0.010384295135736465, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "paraphrase_len4_direct_accuracy": 0.02734375, "paraphrase_len4_direct_target_mass": 0.010115444660186768, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len8_direct_accuracy": 0.015625, "paraphrase_len8_direct_target_mass": 0.010564621537923813, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len12_direct_accuracy": 0.01171875, "paraphrase_len12_direct_target_mass": 0.010639360174536705, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len24_direct_accuracy": 0.00390625, "paraphrase_len24_direct_target_mass": 0.010217440314590931, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN }, { "stage": "answer_only", "local_step": 1500, "step": 1500, "loss": 4.504985809326172, "trace_loss_active": false, "lr_scale": 1.0, "direct_loss": 4.504985809326172, "standard_len4_direct_accuracy": 0.01171875, "standard_len4_direct_target_mass": 0.010771668516099453, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len8_direct_accuracy": 0.01171875, "standard_len8_direct_target_mass": 0.01045084185898304, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len12_direct_accuracy": 0.00390625, "standard_len12_direct_target_mass": 0.010296698659658432, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len24_direct_accuracy": 0.01953125, "standard_len24_direct_target_mass": 0.010738033801317215, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "paraphrase_len4_direct_accuracy": 0.01171875, "paraphrase_len4_direct_target_mass": 0.010257979854941368, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len8_direct_accuracy": 0.0078125, "paraphrase_len8_direct_target_mass": 0.01088618766516447, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len12_direct_accuracy": 0.00390625, "paraphrase_len12_direct_target_mass": 0.010655775666236877, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len24_direct_accuracy": 0.01171875, "paraphrase_len24_direct_target_mass": 0.01033344678580761, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN }, { "stage": "answer_only", "local_step": 2000, "step": 2000, "loss": 4.449176788330078, "trace_loss_active": false, "lr_scale": 1.0, "direct_loss": 4.449176788330078, "standard_len4_direct_accuracy": 0.015625, "standard_len4_direct_target_mass": 0.010611070320010185, "standard_len4_executor_accuracy": NaN, "standard_len4_executor_target_mass": NaN, "standard_len4_init_accuracy": NaN, "standard_len4_op_accuracy": NaN, "standard_len4_arg_accuracy": NaN, "standard_len4_program_exact": NaN, "standard_len8_direct_accuracy": 0.01171875, "standard_len8_direct_target_mass": 0.010338552296161652, "standard_len8_executor_accuracy": NaN, "standard_len8_executor_target_mass": NaN, "standard_len8_init_accuracy": NaN, "standard_len8_op_accuracy": NaN, "standard_len8_arg_accuracy": NaN, "standard_len8_program_exact": NaN, "standard_len12_direct_accuracy": 0.0, "standard_len12_direct_target_mass": 0.010243278928101063, "standard_len12_executor_accuracy": NaN, "standard_len12_executor_target_mass": NaN, "standard_len12_init_accuracy": NaN, "standard_len12_op_accuracy": NaN, "standard_len12_arg_accuracy": NaN, "standard_len12_program_exact": NaN, "standard_len24_direct_accuracy": 0.01171875, "standard_len24_direct_target_mass": 0.011326039209961891, "standard_len24_executor_accuracy": NaN, "standard_len24_executor_target_mass": NaN, "standard_len24_init_accuracy": NaN, "standard_len24_op_accuracy": NaN, "standard_len24_arg_accuracy": NaN, "standard_len24_program_exact": NaN, "paraphrase_len4_direct_accuracy": 0.01953125, "paraphrase_len4_direct_target_mass": 0.010637488216161728, "paraphrase_len4_executor_accuracy": NaN, "paraphrase_len4_executor_target_mass": NaN, "paraphrase_len4_init_accuracy": NaN, "paraphrase_len4_op_accuracy": NaN, "paraphrase_len4_arg_accuracy": NaN, "paraphrase_len4_program_exact": NaN, "paraphrase_len8_direct_accuracy": 0.0078125, "paraphrase_len8_direct_target_mass": 0.011127347126603127, "paraphrase_len8_executor_accuracy": NaN, "paraphrase_len8_executor_target_mass": NaN, "paraphrase_len8_init_accuracy": NaN, "paraphrase_len8_op_accuracy": NaN, "paraphrase_len8_arg_accuracy": NaN, "paraphrase_len8_program_exact": NaN, "paraphrase_len12_direct_accuracy": 0.00390625, "paraphrase_len12_direct_target_mass": 0.010678712278604507, "paraphrase_len12_executor_accuracy": NaN, "paraphrase_len12_executor_target_mass": NaN, "paraphrase_len12_init_accuracy": NaN, "paraphrase_len12_op_accuracy": NaN, "paraphrase_len12_arg_accuracy": NaN, "paraphrase_len12_program_exact": NaN, "paraphrase_len24_direct_accuracy": 0.01171875, "paraphrase_len24_direct_target_mass": 0.010375700891017914, "paraphrase_len24_executor_accuracy": NaN, "paraphrase_len24_executor_target_mass": NaN, "paraphrase_len24_init_accuracy": NaN, "paraphrase_len24_op_accuracy": NaN, "paraphrase_len24_arg_accuracy": NaN, "paraphrase_len24_program_exact": NaN } ], "final_metrics": { "standard_len4": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.015625, "direct_target_mass": 0.010611070320010185, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN }, "standard_len8": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.01171875, "direct_target_mass": 0.010338552296161652, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN }, "standard_len12": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.0, "direct_target_mass": 0.010243278928101063, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN }, "standard_len24": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.01171875, "direct_target_mass": 0.011326039209961891, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN }, "paraphrase_len4": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.01953125, "direct_target_mass": 0.010637488216161728, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN }, "paraphrase_len8": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.0078125, "direct_target_mass": 0.011127347126603127, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN }, "paraphrase_len12": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.00390625, "direct_target_mass": 0.010678712278604507, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN }, "paraphrase_len24": { "variant": "direct", "n": 256.0, "direct_accuracy": 0.01171875, "direct_target_mass": 0.010375700891017914, "executor_accuracy": NaN, "executor_target_mass": NaN, "init_accuracy": NaN, "op_accuracy": NaN, "arg_accuracy": NaN, "program_exact": NaN } } }, "compiler_trace": { "variant": "compiler_trace", "train_seconds": 197.72957015037537, "checkpoints": [ "large_artifacts/qwen_span_free_compiler/checkpoints/main_qwen35_attention_len4_retention/compiler_trace/checkpoint_final.pt" ], "train_log": [ { "stage": "trace_bootstrap", "local_step": 1, "step": 1, "loss": 66.31420135498047, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 4.577545166015625, "init_loss": 4.602025985717773, "op_loss": 1.1087757349014282, "arg_loss": 4.668872833251953, "init_attn_loss": 4.346830368041992, "op_attn_loss": 4.123574256896973, "arg_attn_loss": 4.046563148498535, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.01171875, "standard_len4_executor_target_mass": 0.010315706953406334, "standard_len4_init_accuracy": 0.0078125, "standard_len4_op_accuracy": 0.35546875, "standard_len4_arg_accuracy": 0.041015625, "standard_len4_program_exact": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0078125, "standard_len8_executor_target_mass": 0.010321954265236855, "standard_len8_init_accuracy": 0.0078125, "standard_len8_op_accuracy": 0.3388671875, "standard_len8_arg_accuracy": 0.04052734375, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0078125, "standard_len12_executor_target_mass": 0.010278942063450813, "standard_len12_init_accuracy": 0.0078125, "standard_len12_op_accuracy": 0.333984375, "standard_len12_arg_accuracy": 0.0498046875, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.01953125, "standard_len24_executor_target_mass": 0.010206853970885277, "standard_len24_init_accuracy": 0.0078125, "standard_len24_op_accuracy": 0.33154296875, "standard_len24_arg_accuracy": 0.0458984375, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.01171875, "paraphrase_len4_executor_target_mass": 0.010343879461288452, "paraphrase_len4_init_accuracy": 0.0, "paraphrase_len4_op_accuracy": 0.3173828125, "paraphrase_len4_arg_accuracy": 0.0478515625, "paraphrase_len4_program_exact": 0.0, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0078125, "paraphrase_len8_executor_target_mass": 0.010363485664129257, "paraphrase_len8_init_accuracy": 0.01171875, "paraphrase_len8_op_accuracy": 0.33447265625, "paraphrase_len8_arg_accuracy": 0.0439453125, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.015625, "paraphrase_len12_executor_target_mass": 0.010244667530059814, "paraphrase_len12_init_accuracy": 0.00390625, "paraphrase_len12_op_accuracy": 0.3479817708333333, "paraphrase_len12_arg_accuracy": 0.04296875, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.00390625, "paraphrase_len24_executor_target_mass": 0.010406127199530602, "paraphrase_len24_init_accuracy": 0.00390625, "paraphrase_len24_op_accuracy": 0.3388671875, "paraphrase_len24_arg_accuracy": 0.050455729166666664, "paraphrase_len24_program_exact": 0.0 }, { "stage": "trace_bootstrap", "local_step": 500, "step": 500, "loss": 0.0008803260279819369, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 6.640430365223438e-05, "init_loss": 3.869634383590892e-06, "op_loss": 1.4514545910060406e-05, "arg_loss": 1.1264280601608334e-06, "init_attn_loss": 2.9243489052532823e-07, "op_attn_loss": 3.937574365409091e-05, "arg_attn_loss": 0.00022126590192783624, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.9921875, "standard_len4_executor_target_mass": 0.9918644428253174, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.998046875, "standard_len4_program_exact": 0.9921875, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.01171875, "standard_len8_executor_target_mass": 0.010961522348225117, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.62646484375, "standard_len8_arg_accuracy": 0.3935546875, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.01953125, "standard_len12_executor_target_mass": 0.013713983818888664, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5439453125, "standard_len12_arg_accuracy": 0.2760416666666667, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.02734375, "standard_len24_executor_target_mass": 0.010379348881542683, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.42919921875, "standard_len24_arg_accuracy": 0.14990234375, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.02734375, "paraphrase_len4_executor_target_mass": 0.021627463400363922, "paraphrase_len4_init_accuracy": 0.9375, "paraphrase_len4_op_accuracy": 0.626953125, "paraphrase_len4_arg_accuracy": 0.5712890625, "paraphrase_len4_program_exact": 0.01171875, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.00390625, "paraphrase_len8_executor_target_mass": 0.01047806441783905, "paraphrase_len8_init_accuracy": 0.9453125, "paraphrase_len8_op_accuracy": 0.48828125, "paraphrase_len8_arg_accuracy": 0.255859375, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.009816160425543785, "paraphrase_len12_init_accuracy": 0.921875, "paraphrase_len12_op_accuracy": 0.431640625, "paraphrase_len12_arg_accuracy": 0.171875, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0, "paraphrase_len24_executor_target_mass": 0.010027882643043995, "paraphrase_len24_init_accuracy": 0.9296875, "paraphrase_len24_op_accuracy": 0.388671875, "paraphrase_len24_arg_accuracy": 0.09423828125, "paraphrase_len24_program_exact": 0.0 }, { "stage": "trace_bootstrap", "local_step": 1000, "step": 1000, "loss": 0.00031066624796949327, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 1.4990122508606873e-05, "init_loss": 2.067533387162257e-06, "op_loss": 2.7930448140978115e-06, "arg_loss": 4.4284260525273567e-07, "init_attn_loss": 2.88709316009772e-07, "op_attn_loss": 1.576722388563212e-05, "arg_attn_loss": 7.866742816986516e-05, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.9921875, "standard_len4_executor_target_mass": 0.9933204650878906, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.998046875, "standard_len4_program_exact": 0.9921875, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.015625, "standard_len8_executor_target_mass": 0.010806126520037651, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.623046875, "standard_len8_arg_accuracy": 0.39453125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.03125, "standard_len12_executor_target_mass": 0.013389334082603455, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5390625, "standard_len12_arg_accuracy": 0.2750651041666667, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.00390625, "standard_len24_executor_target_mass": 0.010301909409463406, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.4264322916666667, "standard_len24_arg_accuracy": 0.14876302083333334, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.03125, "paraphrase_len4_executor_target_mass": 0.02824772335588932, "paraphrase_len4_init_accuracy": 0.9375, "paraphrase_len4_op_accuracy": 0.634765625, "paraphrase_len4_arg_accuracy": 0.5810546875, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.00390625, "paraphrase_len8_executor_target_mass": 0.00815360713750124, "paraphrase_len8_init_accuracy": 0.9453125, "paraphrase_len8_op_accuracy": 0.4970703125, "paraphrase_len8_arg_accuracy": 0.2587890625, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.015625, "paraphrase_len12_executor_target_mass": 0.009708615019917488, "paraphrase_len12_init_accuracy": 0.9296875, "paraphrase_len12_op_accuracy": 0.4368489583333333, "paraphrase_len12_arg_accuracy": 0.17024739583333334, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.015625, "paraphrase_len24_executor_target_mass": 0.01002294011414051, "paraphrase_len24_init_accuracy": 0.9296875, "paraphrase_len24_op_accuracy": 0.3876953125, "paraphrase_len24_arg_accuracy": 0.09293619791666667, "paraphrase_len24_program_exact": 0.0 }, { "stage": "trace_continuation", "local_step": 1, "step": 1001, "loss": 0.0009734175400808454, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 1.0265270248055458e-05, "init_loss": 1.9501867427607067e-06, "op_loss": 1.5636555872333702e-06, "arg_loss": 5.19441528012976e-07, "init_attn_loss": 5.010433596908115e-07, "op_attn_loss": 3.868149724439718e-05, "arg_attn_loss": 0.0002785735996440053, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.9921875, "standard_len4_executor_target_mass": 0.9933227300643921, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.998046875, "standard_len4_program_exact": 0.9921875, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.015625, "standard_len8_executor_target_mass": 0.01080083753913641, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.623046875, "standard_len8_arg_accuracy": 0.39453125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.03125, "standard_len12_executor_target_mass": 0.013387772254645824, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5390625, "standard_len12_arg_accuracy": 0.2750651041666667, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.00390625, "standard_len24_executor_target_mass": 0.010301978327333927, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.4264322916666667, "standard_len24_arg_accuracy": 0.14876302083333334, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.03125, "paraphrase_len4_executor_target_mass": 0.02824312075972557, "paraphrase_len4_init_accuracy": 0.9375, "paraphrase_len4_op_accuracy": 0.6328125, "paraphrase_len4_arg_accuracy": 0.5810546875, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.00390625, "paraphrase_len8_executor_target_mass": 0.008155565708875656, "paraphrase_len8_init_accuracy": 0.9453125, "paraphrase_len8_op_accuracy": 0.4970703125, "paraphrase_len8_arg_accuracy": 0.2587890625, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.015625, "paraphrase_len12_executor_target_mass": 0.009708944708108902, "paraphrase_len12_init_accuracy": 0.9296875, "paraphrase_len12_op_accuracy": 0.4371744791666667, "paraphrase_len12_arg_accuracy": 0.17024739583333334, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.015625, "paraphrase_len24_executor_target_mass": 0.01002243161201477, "paraphrase_len24_init_accuracy": 0.9296875, "paraphrase_len24_op_accuracy": 0.3875325520833333, "paraphrase_len24_arg_accuracy": 0.09309895833333333, "paraphrase_len24_program_exact": 0.0 }, { "stage": "trace_continuation", "local_step": 500, "step": 1500, "loss": 0.0022699632681906223, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 0.0016062172362580895, "init_loss": 1.9669366793095833e-06, "op_loss": 0.00030417967354878783, "arg_loss": 9.688196587376297e-05, "init_attn_loss": 3.054734634133638e-07, "op_attn_loss": 3.0097016860963777e-05, "arg_attn_loss": 5.4536409152206033e-05, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.9921875, "standard_len4_executor_target_mass": 0.9899213314056396, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.998046875, "standard_len4_program_exact": 0.9921875, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.00390625, "standard_len8_executor_target_mass": 0.0060020992532372475, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.62255859375, "standard_len8_arg_accuracy": 0.4033203125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.01171875, "standard_len12_executor_target_mass": 0.010000649839639664, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5390625, "standard_len12_arg_accuracy": 0.2809244791666667, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.015625, "standard_len24_executor_target_mass": 0.010397789068520069, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.43505859375, "standard_len24_arg_accuracy": 0.16259765625, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0390625, "paraphrase_len4_executor_target_mass": 0.03664183244109154, "paraphrase_len4_init_accuracy": 0.953125, "paraphrase_len4_op_accuracy": 0.619140625, "paraphrase_len4_arg_accuracy": 0.5380859375, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0078125, "paraphrase_len8_executor_target_mass": 0.006448133382946253, "paraphrase_len8_init_accuracy": 0.95703125, "paraphrase_len8_op_accuracy": 0.48828125, "paraphrase_len8_arg_accuracy": 0.24658203125, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0, "paraphrase_len12_executor_target_mass": 0.009368103928864002, "paraphrase_len12_init_accuracy": 0.92578125, "paraphrase_len12_op_accuracy": 0.4222005208333333, "paraphrase_len12_arg_accuracy": 0.18196614583333334, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.01953125, "paraphrase_len24_executor_target_mass": 0.010300425812602043, "paraphrase_len24_init_accuracy": 0.94921875, "paraphrase_len24_op_accuracy": 0.3805338541666667, "paraphrase_len24_arg_accuracy": 0.10172526041666667, "paraphrase_len24_program_exact": 0.0 }, { "stage": "trace_continuation", "local_step": 1000, "step": 2000, "loss": 0.0009479307918809354, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 0.00011056731455028057, "init_loss": 1.3411042232291948e-07, "op_loss": 2.753250919340644e-05, "arg_loss": 7.636835874791359e-08, "init_attn_loss": 3.3434409374422103e-07, "op_attn_loss": 0.0002595155092421919, "arg_attn_loss": 9.889536158880219e-06, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.99609375, "standard_len4_executor_target_mass": 0.9939929842948914, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.9990234375, "standard_len4_program_exact": 0.99609375, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.00390625, "standard_len8_executor_target_mass": 0.00694353599101305, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.63330078125, "standard_len8_arg_accuracy": 0.40283203125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.01171875, "standard_len12_executor_target_mass": 0.010480374097824097, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5498046875, "standard_len12_arg_accuracy": 0.2796223958333333, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0078125, "standard_len24_executor_target_mass": 0.010218922048807144, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.4285481770833333, "standard_len24_arg_accuracy": 0.16324869791666666, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.015625, "paraphrase_len4_executor_target_mass": 0.01809144765138626, "paraphrase_len4_init_accuracy": 0.94921875, "paraphrase_len4_op_accuracy": 0.580078125, "paraphrase_len4_arg_accuracy": 0.5400390625, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.00390625, "paraphrase_len8_executor_target_mass": 0.008965857326984406, "paraphrase_len8_init_accuracy": 0.95703125, "paraphrase_len8_op_accuracy": 0.46337890625, "paraphrase_len8_arg_accuracy": 0.2451171875, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.011412767693400383, "paraphrase_len12_init_accuracy": 0.9296875, "paraphrase_len12_op_accuracy": 0.41796875, "paraphrase_len12_arg_accuracy": 0.1787109375, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0, "paraphrase_len24_executor_target_mass": 0.010152388364076614, "paraphrase_len24_init_accuracy": 0.95703125, "paraphrase_len24_op_accuracy": 0.3748372395833333, "paraphrase_len24_arg_accuracy": 0.10188802083333333, "paraphrase_len24_program_exact": 0.0 } ], "final_metrics": { "standard_len4": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.99609375, "executor_target_mass": 0.9939929842948914, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 0.9990234375, "program_exact": 0.99609375 }, "standard_len8": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.00390625, "executor_target_mass": 0.00694353599101305, "init_accuracy": 1.0, "op_accuracy": 0.63330078125, "arg_accuracy": 0.40283203125, "program_exact": 0.0 }, "standard_len12": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.01171875, "executor_target_mass": 0.010480374097824097, "init_accuracy": 1.0, "op_accuracy": 0.5498046875, "arg_accuracy": 0.2796223958333333, "program_exact": 0.0 }, "standard_len24": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.010218922048807144, "init_accuracy": 1.0, "op_accuracy": 0.4285481770833333, "arg_accuracy": 0.16324869791666666, "program_exact": 0.0 }, "paraphrase_len4": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.015625, "executor_target_mass": 0.01809144765138626, "init_accuracy": 0.94921875, "op_accuracy": 0.580078125, "arg_accuracy": 0.5400390625, "program_exact": 0.015625 }, "paraphrase_len8": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.00390625, "executor_target_mass": 0.008965857326984406, "init_accuracy": 0.95703125, "op_accuracy": 0.46337890625, "arg_accuracy": 0.2451171875, "program_exact": 0.0 }, "paraphrase_len12": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.011412767693400383, "init_accuracy": 0.9296875, "op_accuracy": 0.41796875, "arg_accuracy": 0.1787109375, "program_exact": 0.0 }, "paraphrase_len24": { "variant": "compiler_trace", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.010152388364076614, "init_accuracy": 0.95703125, "op_accuracy": 0.3748372395833333, "arg_accuracy": 0.10188802083333333, "program_exact": 0.0 } } }, "compiler_answer_only": { "variant": "compiler_answer_only", "train_seconds": 189.8375744819641, "checkpoints": [ "large_artifacts/qwen_span_free_compiler/checkpoints/main_qwen35_attention_len4_retention/compiler_answer_only/checkpoint_final.pt" ], "train_log": [ { "stage": "answer_only", "local_step": 1, "step": 1, "loss": 4.576056480407715, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.576056480407715, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.01171875, "standard_len4_executor_target_mass": 0.01030926126986742, "standard_len4_init_accuracy": 0.0, "standard_len4_op_accuracy": 0.333984375, "standard_len4_arg_accuracy": 0.0185546875, "standard_len4_program_exact": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.015625, "standard_len8_executor_target_mass": 0.010309278964996338, "standard_len8_init_accuracy": 0.00390625, "standard_len8_op_accuracy": 0.33251953125, "standard_len8_arg_accuracy": 0.0126953125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.01171875, "standard_len12_executor_target_mass": 0.010309278964996338, "standard_len12_init_accuracy": 0.01171875, "standard_len12_op_accuracy": 0.3310546875, "standard_len12_arg_accuracy": 0.015625, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.01953125, "standard_len24_executor_target_mass": 0.010309278964996338, "standard_len24_init_accuracy": 0.01171875, "standard_len24_op_accuracy": 0.3367513020833333, "standard_len24_arg_accuracy": 0.015625, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.00390625, "paraphrase_len4_executor_target_mass": 0.010309281758964062, "paraphrase_len4_init_accuracy": 0.00390625, "paraphrase_len4_op_accuracy": 0.3642578125, "paraphrase_len4_arg_accuracy": 0.01171875, "paraphrase_len4_program_exact": 0.0, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.015625, "paraphrase_len8_executor_target_mass": 0.010309278964996338, "paraphrase_len8_init_accuracy": 0.015625, "paraphrase_len8_op_accuracy": 0.32470703125, "paraphrase_len8_arg_accuracy": 0.013671875, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0234375, "paraphrase_len12_executor_target_mass": 0.010309278964996338, "paraphrase_len12_init_accuracy": 0.015625, "paraphrase_len12_op_accuracy": 0.3216145833333333, "paraphrase_len12_arg_accuracy": 0.015625, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.01953125, "paraphrase_len24_executor_target_mass": 0.010309278964996338, "paraphrase_len24_init_accuracy": 0.0078125, "paraphrase_len24_op_accuracy": 0.32763671875, "paraphrase_len24_arg_accuracy": 0.017252604166666668, "paraphrase_len24_program_exact": 0.0 }, { "stage": "answer_only", "local_step": 500, "step": 500, "loss": 4.573419570922852, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.573419570922852, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.01953125, "standard_len4_executor_target_mass": 0.010315753519535065, "standard_len4_init_accuracy": 0.00390625, "standard_len4_op_accuracy": 0.3720703125, "standard_len4_arg_accuracy": 0.0166015625, "standard_len4_program_exact": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0234375, "standard_len8_executor_target_mass": 0.010310424491763115, "standard_len8_init_accuracy": 0.01171875, "standard_len8_op_accuracy": 0.3408203125, "standard_len8_arg_accuracy": 0.02783203125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.00390625, "standard_len12_executor_target_mass": 0.01030243095010519, "standard_len12_init_accuracy": 0.01171875, "standard_len12_op_accuracy": 0.3382161458333333, "standard_len12_arg_accuracy": 0.026041666666666668, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0078125, "standard_len24_executor_target_mass": 0.010280674323439598, "standard_len24_init_accuracy": 0.01171875, "standard_len24_op_accuracy": 0.33642578125, "standard_len24_arg_accuracy": 0.0224609375, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0234375, "paraphrase_len4_executor_target_mass": 0.01030553225427866, "paraphrase_len4_init_accuracy": 0.015625, "paraphrase_len4_op_accuracy": 0.3173828125, "paraphrase_len4_arg_accuracy": 0.0263671875, "paraphrase_len4_program_exact": 0.0, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0234375, "paraphrase_len8_executor_target_mass": 0.010328009724617004, "paraphrase_len8_init_accuracy": 0.01171875, "paraphrase_len8_op_accuracy": 0.33447265625, "paraphrase_len8_arg_accuracy": 0.02880859375, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.010232641361653805, "paraphrase_len12_init_accuracy": 0.0078125, "paraphrase_len12_op_accuracy": 0.3479817708333333, "paraphrase_len12_arg_accuracy": 0.020182291666666668, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.015625, "paraphrase_len24_executor_target_mass": 0.01052815467119217, "paraphrase_len24_init_accuracy": 0.0078125, "paraphrase_len24_op_accuracy": 0.3388671875, "paraphrase_len24_arg_accuracy": 0.026529947916666668, "paraphrase_len24_program_exact": 0.0 }, { "stage": "answer_only", "local_step": 1000, "step": 1000, "loss": 4.547166347503662, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.547166347503662, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.015625, "standard_len4_executor_target_mass": 0.010365470312535763, "standard_len4_init_accuracy": 0.02734375, "standard_len4_op_accuracy": 0.35546875, "standard_len4_arg_accuracy": 0.0166015625, "standard_len4_program_exact": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0, "standard_len8_executor_target_mass": 0.010219271294772625, "standard_len8_init_accuracy": 0.0078125, "standard_len8_op_accuracy": 0.3388671875, "standard_len8_arg_accuracy": 0.03125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.015625, "standard_len12_executor_target_mass": 0.010310128331184387, "standard_len12_init_accuracy": 0.00390625, "standard_len12_op_accuracy": 0.333984375, "standard_len12_arg_accuracy": 0.0341796875, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.00390625, "standard_len24_executor_target_mass": 0.010307364165782928, "standard_len24_init_accuracy": 0.0078125, "standard_len24_op_accuracy": 0.33154296875, "standard_len24_arg_accuracy": 0.0302734375, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.015625, "paraphrase_len4_executor_target_mass": 0.010259416885674, "paraphrase_len4_init_accuracy": 0.015625, "paraphrase_len4_op_accuracy": 0.3173828125, "paraphrase_len4_arg_accuracy": 0.02734375, "paraphrase_len4_program_exact": 0.0, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0078125, "paraphrase_len8_executor_target_mass": 0.010334429331123829, "paraphrase_len8_init_accuracy": 0.0078125, "paraphrase_len8_op_accuracy": 0.33447265625, "paraphrase_len8_arg_accuracy": 0.03369140625, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.00390625, "paraphrase_len12_executor_target_mass": 0.010285687632858753, "paraphrase_len12_init_accuracy": 0.01171875, "paraphrase_len12_op_accuracy": 0.3479817708333333, "paraphrase_len12_arg_accuracy": 0.034505208333333336, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.01953125, "paraphrase_len24_executor_target_mass": 0.010345970280468464, "paraphrase_len24_init_accuracy": 0.015625, "paraphrase_len24_op_accuracy": 0.3388671875, "paraphrase_len24_arg_accuracy": 0.03955078125, "paraphrase_len24_program_exact": 0.0 }, { "stage": "answer_only", "local_step": 1500, "step": 1500, "loss": 4.583963394165039, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.583963394165039, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0078125, "standard_len4_executor_target_mass": 0.010528070852160454, "standard_len4_init_accuracy": 0.02734375, "standard_len4_op_accuracy": 0.35546875, "standard_len4_arg_accuracy": 0.0166015625, "standard_len4_program_exact": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0078125, "standard_len8_executor_target_mass": 0.010141816921532154, "standard_len8_init_accuracy": 0.0078125, "standard_len8_op_accuracy": 0.3388671875, "standard_len8_arg_accuracy": 0.03125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.015625, "standard_len12_executor_target_mass": 0.010318243876099586, "standard_len12_init_accuracy": 0.00390625, "standard_len12_op_accuracy": 0.333984375, "standard_len12_arg_accuracy": 0.0341796875, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0078125, "standard_len24_executor_target_mass": 0.010311760008335114, "standard_len24_init_accuracy": 0.0078125, "standard_len24_op_accuracy": 0.33154296875, "standard_len24_arg_accuracy": 0.0302734375, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0078125, "paraphrase_len4_executor_target_mass": 0.01035543903708458, "paraphrase_len4_init_accuracy": 0.0078125, "paraphrase_len4_op_accuracy": 0.3173828125, "paraphrase_len4_arg_accuracy": 0.02734375, "paraphrase_len4_program_exact": 0.0, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.01171875, "paraphrase_len8_executor_target_mass": 0.010307414457201958, "paraphrase_len8_init_accuracy": 0.0078125, "paraphrase_len8_op_accuracy": 0.33447265625, "paraphrase_len8_arg_accuracy": 0.033203125, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.01171875, "paraphrase_len12_executor_target_mass": 0.01029904093593359, "paraphrase_len12_init_accuracy": 0.01953125, "paraphrase_len12_op_accuracy": 0.3479817708333333, "paraphrase_len12_arg_accuracy": 0.033528645833333336, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.00390625, "paraphrase_len24_executor_target_mass": 0.010318422690033913, "paraphrase_len24_init_accuracy": 0.01953125, "paraphrase_len24_op_accuracy": 0.3388671875, "paraphrase_len24_arg_accuracy": 0.038736979166666664, "paraphrase_len24_program_exact": 0.0 }, { "stage": "answer_only", "local_step": 2000, "step": 2000, "loss": 4.565924644470215, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 4.565924644470215, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.010376090183854103, "standard_len4_init_accuracy": 0.00390625, "standard_len4_op_accuracy": 0.35546875, "standard_len4_arg_accuracy": 0.0166015625, "standard_len4_program_exact": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0078125, "standard_len8_executor_target_mass": 0.010234260931611061, "standard_len8_init_accuracy": 0.01171875, "standard_len8_op_accuracy": 0.3388671875, "standard_len8_arg_accuracy": 0.03125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0078125, "standard_len12_executor_target_mass": 0.010312571190297604, "standard_len12_init_accuracy": 0.01171875, "standard_len12_op_accuracy": 0.333984375, "standard_len12_arg_accuracy": 0.0341796875, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0, "standard_len24_executor_target_mass": 0.010306041687726974, "standard_len24_init_accuracy": 0.01171875, "standard_len24_op_accuracy": 0.33154296875, "standard_len24_arg_accuracy": 0.0302734375, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.015625, "paraphrase_len4_executor_target_mass": 0.01023918017745018, "paraphrase_len4_init_accuracy": 0.015625, "paraphrase_len4_op_accuracy": 0.3173828125, "paraphrase_len4_arg_accuracy": 0.02734375, "paraphrase_len4_program_exact": 0.0, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.0078125, "paraphrase_len8_executor_target_mass": 0.010322628542780876, "paraphrase_len8_init_accuracy": 0.01171875, "paraphrase_len8_op_accuracy": 0.33447265625, "paraphrase_len8_arg_accuracy": 0.033203125, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.010273873805999756, "paraphrase_len12_init_accuracy": 0.0078125, "paraphrase_len12_op_accuracy": 0.3479817708333333, "paraphrase_len12_arg_accuracy": 0.033528645833333336, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0078125, "paraphrase_len24_executor_target_mass": 0.010350679978728294, "paraphrase_len24_init_accuracy": 0.0078125, "paraphrase_len24_op_accuracy": 0.3388671875, "paraphrase_len24_arg_accuracy": 0.038248697916666664, "paraphrase_len24_program_exact": 0.0 } ], "final_metrics": { "standard_len4": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.010376090183854103, "init_accuracy": 0.00390625, "op_accuracy": 0.35546875, "arg_accuracy": 0.0166015625, "program_exact": 0.0 }, "standard_len8": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.010234260931611061, "init_accuracy": 0.01171875, "op_accuracy": 0.3388671875, "arg_accuracy": 0.03125, "program_exact": 0.0 }, "standard_len12": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.010312571190297604, "init_accuracy": 0.01171875, "op_accuracy": 0.333984375, "arg_accuracy": 0.0341796875, "program_exact": 0.0 }, "standard_len24": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0, "executor_target_mass": 0.010306041687726974, "init_accuracy": 0.01171875, "op_accuracy": 0.33154296875, "arg_accuracy": 0.0302734375, "program_exact": 0.0 }, "paraphrase_len4": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.015625, "executor_target_mass": 0.01023918017745018, "init_accuracy": 0.015625, "op_accuracy": 0.3173828125, "arg_accuracy": 0.02734375, "program_exact": 0.0 }, "paraphrase_len8": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.010322628542780876, "init_accuracy": 0.01171875, "op_accuracy": 0.33447265625, "arg_accuracy": 0.033203125, "program_exact": 0.0 }, "paraphrase_len12": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.010273873805999756, "init_accuracy": 0.0078125, "op_accuracy": 0.3479817708333333, "arg_accuracy": 0.033528645833333336, "program_exact": 0.0 }, "paraphrase_len24": { "variant": "compiler_answer_only", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.010350679978728294, "init_accuracy": 0.0078125, "op_accuracy": 0.3388671875, "arg_accuracy": 0.038248697916666664, "program_exact": 0.0 } } }, "compiler_trace_then_answer": { "variant": "compiler_trace_then_answer", "train_seconds": 192.8309621810913, "checkpoints": [ "large_artifacts/qwen_span_free_compiler/checkpoints/main_qwen35_attention_len4_retention/compiler_trace_then_answer/checkpoint_final.pt" ], "train_log": [ { "stage": "trace_bootstrap", "local_step": 1, "step": 1, "loss": 66.37966918945312, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 4.577260971069336, "init_loss": 4.609462738037109, "op_loss": 1.095146656036377, "arg_loss": 4.622659206390381, "init_attn_loss": 3.8998355865478516, "op_attn_loss": 4.366880416870117, "arg_attn_loss": 4.282199859619141, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.0, "standard_len4_executor_target_mass": 0.01031267736107111, "standard_len4_init_accuracy": 0.0078125, "standard_len4_op_accuracy": 0.333984375, "standard_len4_arg_accuracy": 0.048828125, "standard_len4_program_exact": 0.0, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.015625, "standard_len8_executor_target_mass": 0.01031205803155899, "standard_len8_init_accuracy": 0.0, "standard_len8_op_accuracy": 0.33251953125, "standard_len8_arg_accuracy": 0.05322265625, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.015625, "standard_len12_executor_target_mass": 0.010306648910045624, "standard_len12_init_accuracy": 0.00390625, "standard_len12_op_accuracy": 0.3310546875, "standard_len12_arg_accuracy": 0.0458984375, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.01171875, "standard_len24_executor_target_mass": 0.010295618325471878, "standard_len24_init_accuracy": 0.0078125, "standard_len24_op_accuracy": 0.3367513020833333, "standard_len24_arg_accuracy": 0.048014322916666664, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.015625, "paraphrase_len4_executor_target_mass": 0.010323893278837204, "paraphrase_len4_init_accuracy": 0.03125, "paraphrase_len4_op_accuracy": 0.3642578125, "paraphrase_len4_arg_accuracy": 0.044921875, "paraphrase_len4_program_exact": 0.0, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.00390625, "paraphrase_len8_executor_target_mass": 0.010317619889974594, "paraphrase_len8_init_accuracy": 0.00390625, "paraphrase_len8_op_accuracy": 0.32470703125, "paraphrase_len8_arg_accuracy": 0.05029296875, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.01171875, "paraphrase_len12_executor_target_mass": 0.01029968447983265, "paraphrase_len12_init_accuracy": 0.0078125, "paraphrase_len12_op_accuracy": 0.3216145833333333, "paraphrase_len12_arg_accuracy": 0.049479166666666664, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.015625, "paraphrase_len24_executor_target_mass": 0.01032134611159563, "paraphrase_len24_init_accuracy": 0.0078125, "paraphrase_len24_op_accuracy": 0.32763671875, "paraphrase_len24_arg_accuracy": 0.048014322916666664, "paraphrase_len24_program_exact": 0.0 }, { "stage": "trace_bootstrap", "local_step": 500, "step": 500, "loss": 0.0013213630300015211, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 0.00025511026615276933, "init_loss": 5.18185697728768e-06, "op_loss": 5.732939098379575e-05, "arg_loss": 5.166736627870705e-06, "init_attn_loss": 6.97557084095024e-07, "op_attn_loss": 0.00015619784244336188, "arg_attn_loss": 0.00017078101518563926, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.9921875, "standard_len4_executor_target_mass": 0.9929526448249817, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.998046875, "standard_len4_program_exact": 0.9921875, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.00390625, "standard_len8_executor_target_mass": 0.009669668972492218, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.595703125, "standard_len8_arg_accuracy": 0.39013671875, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.00390625, "standard_len12_executor_target_mass": 0.011587940156459808, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5478515625, "standard_len12_arg_accuracy": 0.2718098958333333, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0078125, "standard_len24_executor_target_mass": 0.01030875463038683, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.431640625, "standard_len24_arg_accuracy": 0.1396484375, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.03515625, "paraphrase_len4_executor_target_mass": 0.026344850659370422, "paraphrase_len4_init_accuracy": 0.921875, "paraphrase_len4_op_accuracy": 0.6005859375, "paraphrase_len4_arg_accuracy": 0.5693359375, "paraphrase_len4_program_exact": 0.0234375, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.01953125, "paraphrase_len8_executor_target_mass": 0.014081336557865143, "paraphrase_len8_init_accuracy": 0.93359375, "paraphrase_len8_op_accuracy": 0.47216796875, "paraphrase_len8_arg_accuracy": 0.23828125, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.010251011699438095, "paraphrase_len12_init_accuracy": 0.91796875, "paraphrase_len12_op_accuracy": 0.4342447916666667, "paraphrase_len12_arg_accuracy": 0.16015625, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.00390625, "paraphrase_len24_executor_target_mass": 0.010301128029823303, "paraphrase_len24_init_accuracy": 0.921875, "paraphrase_len24_op_accuracy": 0.3898111979166667, "paraphrase_len24_arg_accuracy": 0.08626302083333333, "paraphrase_len24_program_exact": 0.0 }, { "stage": "trace_bootstrap", "local_step": 1000, "step": 1000, "loss": 0.0003294815542176366, "trace_loss_active": true, "lr_scale": 1.0, "executor_loss": 2.128154483216349e-05, "init_loss": 2.033074224527809e-06, "op_loss": 4.428352440299932e-06, "arg_loss": 3.839371629510424e-07, "init_attn_loss": 4.731108447231236e-07, "op_attn_loss": 1.0833451597136445e-05, "arg_attn_loss": 8.711191185284406e-05, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.9921875, "standard_len4_executor_target_mass": 0.9922643303871155, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.998046875, "standard_len4_program_exact": 0.9921875, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.00390625, "standard_len8_executor_target_mass": 0.009029692970216274, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.59326171875, "standard_len8_arg_accuracy": 0.388671875, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.00390625, "standard_len12_executor_target_mass": 0.011047746054828167, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.54296875, "standard_len12_arg_accuracy": 0.2708333333333333, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.01171875, "standard_len24_executor_target_mass": 0.010204633697867393, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.42724609375, "standard_len24_arg_accuracy": 0.13916015625, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.0234375, "paraphrase_len4_executor_target_mass": 0.024674706161022186, "paraphrase_len4_init_accuracy": 0.9140625, "paraphrase_len4_op_accuracy": 0.607421875, "paraphrase_len4_arg_accuracy": 0.5693359375, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.01953125, "paraphrase_len8_executor_target_mass": 0.0137432049959898, "paraphrase_len8_init_accuracy": 0.91796875, "paraphrase_len8_op_accuracy": 0.47509765625, "paraphrase_len8_arg_accuracy": 0.23681640625, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.01102103479206562, "paraphrase_len12_init_accuracy": 0.90234375, "paraphrase_len12_op_accuracy": 0.4423828125, "paraphrase_len12_arg_accuracy": 0.15852864583333334, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.00390625, "paraphrase_len24_executor_target_mass": 0.010262850672006607, "paraphrase_len24_init_accuracy": 0.91796875, "paraphrase_len24_op_accuracy": 0.3865559895833333, "paraphrase_len24_arg_accuracy": 0.083984375, "paraphrase_len24_program_exact": 0.0 }, { "stage": "answer_retention", "local_step": 1, "step": 1001, "loss": 0.06314650177955627, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 0.06314650177955627, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.99609375, "standard_len4_executor_target_mass": 0.9943959712982178, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.9990234375, "standard_len4_program_exact": 0.99609375, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.0, "standard_len8_executor_target_mass": 0.007943274453282356, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.59521484375, "standard_len8_arg_accuracy": 0.39013671875, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0, "standard_len12_executor_target_mass": 0.010810667648911476, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5452473958333334, "standard_len12_arg_accuracy": 0.2727864583333333, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.02734375, "standard_len24_executor_target_mass": 0.010184265673160553, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.4280598958333333, "standard_len24_arg_accuracy": 0.13883463541666666, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.02734375, "paraphrase_len4_executor_target_mass": 0.02556075155735016, "paraphrase_len4_init_accuracy": 0.90625, "paraphrase_len4_op_accuracy": 0.6083984375, "paraphrase_len4_arg_accuracy": 0.5693359375, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.015625, "paraphrase_len8_executor_target_mass": 0.013406741432845592, "paraphrase_len8_init_accuracy": 0.9140625, "paraphrase_len8_op_accuracy": 0.47802734375, "paraphrase_len8_arg_accuracy": 0.2353515625, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.0078125, "paraphrase_len12_executor_target_mass": 0.010506167076528072, "paraphrase_len12_init_accuracy": 0.90234375, "paraphrase_len12_op_accuracy": 0.4420572916666667, "paraphrase_len12_arg_accuracy": 0.15852864583333334, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0078125, "paraphrase_len24_executor_target_mass": 0.010139396414160728, "paraphrase_len24_init_accuracy": 0.91796875, "paraphrase_len24_op_accuracy": 0.3860677083333333, "paraphrase_len24_arg_accuracy": 0.08365885416666667, "paraphrase_len24_program_exact": 0.0 }, { "stage": "answer_retention", "local_step": 500, "step": 1500, "loss": 6.857244443381205e-05, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 6.857244443381205e-05, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.9921875, "standard_len4_executor_target_mass": 0.9920581579208374, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.998046875, "standard_len4_program_exact": 0.9921875, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.015625, "standard_len8_executor_target_mass": 0.009462045505642891, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.59130859375, "standard_len8_arg_accuracy": 0.39111328125, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.0, "standard_len12_executor_target_mass": 0.007872702553868294, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5367838541666666, "standard_len12_arg_accuracy": 0.2737630208333333, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.0078125, "standard_len24_executor_target_mass": 0.009491177275776863, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.4391276041666667, "standard_len24_arg_accuracy": 0.14404296875, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.01953125, "paraphrase_len4_executor_target_mass": 0.024961359798908234, "paraphrase_len4_init_accuracy": 0.953125, "paraphrase_len4_op_accuracy": 0.646484375, "paraphrase_len4_arg_accuracy": 0.5595703125, "paraphrase_len4_program_exact": 0.015625, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.01953125, "paraphrase_len8_executor_target_mass": 0.009718411602079868, "paraphrase_len8_init_accuracy": 0.93359375, "paraphrase_len8_op_accuracy": 0.48828125, "paraphrase_len8_arg_accuracy": 0.24658203125, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.00390625, "paraphrase_len12_executor_target_mass": 0.009374216198921204, "paraphrase_len12_init_accuracy": 0.921875, "paraphrase_len12_op_accuracy": 0.443359375, "paraphrase_len12_arg_accuracy": 0.16471354166666666, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.01171875, "paraphrase_len24_executor_target_mass": 0.009697790257632732, "paraphrase_len24_init_accuracy": 0.90234375, "paraphrase_len24_op_accuracy": 0.3818359375, "paraphrase_len24_arg_accuracy": 0.08463541666666667, "paraphrase_len24_program_exact": 0.0 }, { "stage": "answer_retention", "local_step": 1000, "step": 2000, "loss": 1.1090018233517185e-05, "trace_loss_active": false, "lr_scale": 1.0, "executor_loss": 1.1090018233517185e-05, "standard_len4_direct_accuracy": NaN, "standard_len4_direct_target_mass": NaN, "standard_len4_executor_accuracy": 0.99609375, "standard_len4_executor_target_mass": 0.9960671663284302, "standard_len4_init_accuracy": 1.0, "standard_len4_op_accuracy": 1.0, "standard_len4_arg_accuracy": 0.9990234375, "standard_len4_program_exact": 0.99609375, "standard_len8_direct_accuracy": NaN, "standard_len8_direct_target_mass": NaN, "standard_len8_executor_accuracy": 0.015625, "standard_len8_executor_target_mass": 0.01060498971492052, "standard_len8_init_accuracy": 1.0, "standard_len8_op_accuracy": 0.59228515625, "standard_len8_arg_accuracy": 0.3994140625, "standard_len8_program_exact": 0.0, "standard_len12_direct_accuracy": NaN, "standard_len12_direct_target_mass": NaN, "standard_len12_executor_accuracy": 0.015625, "standard_len12_executor_target_mass": 0.010229174047708511, "standard_len12_init_accuracy": 1.0, "standard_len12_op_accuracy": 0.5260416666666666, "standard_len12_arg_accuracy": 0.283203125, "standard_len12_program_exact": 0.0, "standard_len24_direct_accuracy": NaN, "standard_len24_direct_target_mass": NaN, "standard_len24_executor_accuracy": 0.00390625, "standard_len24_executor_target_mass": 0.010200701653957367, "standard_len24_init_accuracy": 1.0, "standard_len24_op_accuracy": 0.4326171875, "standard_len24_arg_accuracy": 0.15983072916666666, "standard_len24_program_exact": 0.0, "paraphrase_len4_direct_accuracy": NaN, "paraphrase_len4_direct_target_mass": NaN, "paraphrase_len4_executor_accuracy": 0.01953125, "paraphrase_len4_executor_target_mass": 0.018929533660411835, "paraphrase_len4_init_accuracy": 0.9453125, "paraphrase_len4_op_accuracy": 0.62890625, "paraphrase_len4_arg_accuracy": 0.556640625, "paraphrase_len4_program_exact": 0.01171875, "paraphrase_len8_direct_accuracy": NaN, "paraphrase_len8_direct_target_mass": NaN, "paraphrase_len8_executor_accuracy": 0.01171875, "paraphrase_len8_executor_target_mass": 0.009994937106966972, "paraphrase_len8_init_accuracy": 0.94140625, "paraphrase_len8_op_accuracy": 0.4765625, "paraphrase_len8_arg_accuracy": 0.27685546875, "paraphrase_len8_program_exact": 0.0, "paraphrase_len12_direct_accuracy": NaN, "paraphrase_len12_direct_target_mass": NaN, "paraphrase_len12_executor_accuracy": 0.00390625, "paraphrase_len12_executor_target_mass": 0.010198865085840225, "paraphrase_len12_init_accuracy": 0.953125, "paraphrase_len12_op_accuracy": 0.4388020833333333, "paraphrase_len12_arg_accuracy": 0.18815104166666666, "paraphrase_len12_program_exact": 0.0, "paraphrase_len24_direct_accuracy": NaN, "paraphrase_len24_direct_target_mass": NaN, "paraphrase_len24_executor_accuracy": 0.0078125, "paraphrase_len24_executor_target_mass": 0.01033688336610794, "paraphrase_len24_init_accuracy": 0.94140625, "paraphrase_len24_op_accuracy": 0.3761393229166667, "paraphrase_len24_arg_accuracy": 0.11311848958333333, "paraphrase_len24_program_exact": 0.0 } ], "final_metrics": { "standard_len4": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.99609375, "executor_target_mass": 0.9960671663284302, "init_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 0.9990234375, "program_exact": 0.99609375 }, "standard_len8": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.015625, "executor_target_mass": 0.01060498971492052, "init_accuracy": 1.0, "op_accuracy": 0.59228515625, "arg_accuracy": 0.3994140625, "program_exact": 0.0 }, "standard_len12": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.015625, "executor_target_mass": 0.010229174047708511, "init_accuracy": 1.0, "op_accuracy": 0.5260416666666666, "arg_accuracy": 0.283203125, "program_exact": 0.0 }, "standard_len24": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.00390625, "executor_target_mass": 0.010200701653957367, "init_accuracy": 1.0, "op_accuracy": 0.4326171875, "arg_accuracy": 0.15983072916666666, "program_exact": 0.0 }, "paraphrase_len4": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.01953125, "executor_target_mass": 0.018929533660411835, "init_accuracy": 0.9453125, "op_accuracy": 0.62890625, "arg_accuracy": 0.556640625, "program_exact": 0.01171875 }, "paraphrase_len8": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.01171875, "executor_target_mass": 0.009994937106966972, "init_accuracy": 0.94140625, "op_accuracy": 0.4765625, "arg_accuracy": 0.27685546875, "program_exact": 0.0 }, "paraphrase_len12": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.00390625, "executor_target_mass": 0.010198865085840225, "init_accuracy": 0.953125, "op_accuracy": 0.4388020833333333, "arg_accuracy": 0.18815104166666666, "program_exact": 0.0 }, "paraphrase_len24": { "variant": "compiler_trace_then_answer", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.0078125, "executor_target_mass": 0.01033688336610794, "init_accuracy": 0.94140625, "op_accuracy": 0.3761393229166667, "arg_accuracy": 0.11311848958333333, "program_exact": 0.0 } } } } }