{
  "run": "smoke_semantic_prefix_value",
  "args": {
    "run_name": "smoke_semantic_prefix_value",
    "model_id": "Qwen/Qwen3-4B",
    "load_in_4bit": true,
    "torch_dtype": "bf16",
    "device_map": "auto",
    "gradient_checkpointing": true,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "lora_target_modules": "all-linear",
    "optimizer": "adamw",
    "lr": 0.0005,
    "weight_decay": 0.0,
    "grad_clip": 1.0,
    "compiler_train_size": 32,
    "verifier_train_size": 32,
    "verifier_collect_limit": 32,
    "verifier_val_collect_limit": 16,
    "val_size": 16,
    "fresh_size": 16,
    "hard_size": 16,
    "max_arith_steps": 4,
    "max_prompt_len": 160,
    "d_model": 96,
    "layers": 1,
    "heads": 4,
    "dropout": 0.1,
    "answer_head_width": 512,
    "batch_size": 16,
    "eval_batch_size": 4,
    "feature_batch_size": 4,
    "compiler_steps": 5,
    "eval_every": 5,
    "quick_eval_size": 16,
    "op_loss_weight": 1.0,
    "arg_loss_weight": 1.0,
    "answer_loss_weight": 0.2,
    "search_topk": 3,
    "max_two_arg_pairs": 8,
    "prefix_collect_beam": 4,
    "prefix_op_topk": 3,
    "prefix_arg_topk": 3,
    "max_negative_prefixes": 24,
    "verifier_d_model": 96,
    "verifier_epochs": 2,
    "verifier_batch_size": 128,
    "verifier_lr": 0.0008,
    "verifier_weight_decay": 0.01,
    "verifier_quick_eval": 1024,
    "max_pos_weight": 8.0,
    "label_modes": [
      "exact",
      "semantic"
    ],
    "beam_width": 6,
    "beam_op_topk": 3,
    "beam_arg_topk": 3,
    "verifier_weights": [
      0.5,
      1.0
    ],
    "seed": 101
  },
  "metadata": {
    "python": "3.12.3",
    "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39",
    "torch": "2.8.0+cu128",
    "cuda_available": true,
    "transformers_loader": "AutoModelForCausalLM",
    "peft_installed": true,
    "model_id": "Qwen/Qwen3-4B",
    "load_in_4bit": true,
    "lora_r": 8,
    "gpu_name": "NVIDIA RTX 6000 Ada Generation",
    "gpu_vram_gb": 47.383
  },
  "prefix_sample_stats": {
    "train": {
      "source_examples": 32,
      "prefix_samples": 1288,
      "exact_positive_samples": 179,
      "semantic_positive_samples": 616,
      "semantic_extra_positive_samples": 437,
      "exact_negative_samples": 1109,
      "semantic_negative_samples": 672,
      "exact_positive_rate": 0.1389751552795031,
      "semantic_positive_rate": 0.4782608695652174,
      "semantic_extra_positive_rate": 0.3392857142857143
    },
    "val": {
      "source_examples": 16,
      "prefix_samples": 522,
      "exact_positive_samples": 87,
      "semantic_positive_samples": 138,
      "semantic_extra_positive_samples": 51,
      "exact_negative_samples": 435,
      "semantic_negative_samples": 384,
      "exact_positive_rate": 0.16666666666666666,
      "semantic_positive_rate": 0.26436781609195403,
      "semantic_extra_positive_rate": 0.09770114942528736
    }
  },
  "label_modes": [
    "exact",
    "semantic"
  ],
  "checkpoint": "large_artifacts/qwen_semantic_prefix_value_model/checkpoints/smoke_semantic_prefix_value",
  "results": [
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "val_mixed",
      "decoder": "greedy",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.0,
      "mean_expansions": 1.0,
      "mean_score": -55.03436927869916
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "val_mixed",
      "decoder": "local_answer",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.0,
      "mean_expansions": 92.25,
      "mean_score": -55.03436927869916
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "val_mixed",
      "decoder": "beam_logprob",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 12.0,
      "oracle_accuracy": 0.0,
      "mean_expansions": 83.0,
      "mean_score": -10.294268533587456
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "val_mixed",
      "decoder": "beam_exact_w0.5",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.0,
      "mean_expansions": 78.0,
      "mean_score": -13.177047996781766
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "val_mixed",
      "decoder": "beam_exact_w1",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.0,
      "mean_expansions": 78.0,
      "mean_score": -15.664478404447436
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "val_mixed",
      "decoder": "beam_semantic_w0.5",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 14.0,
      "oracle_accuracy": 0.0,
      "mean_expansions": 98.0,
      "mean_score": -12.059495456051081
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "val_mixed",
      "decoder": "beam_semantic_w1",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 17.375,
      "oracle_accuracy": 0.0,
      "mean_expansions": 129.3125,
      "mean_score": -13.824722378514707
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_standard",
      "decoder": "greedy",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.0625,
      "mean_expansions": 1.0,
      "mean_score": -55.32255048677325
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_standard",
      "decoder": "local_answer",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.0625,
      "mean_expansions": 92.25,
      "mean_score": -55.32255048677325
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_standard",
      "decoder": "beam_logprob",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 12.0,
      "oracle_accuracy": 0.125,
      "mean_expansions": 83.0,
      "mean_score": -10.284539058804512
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_standard",
      "decoder": "beam_exact_w0.5",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.0625,
      "mean_expansions": 78.0,
      "mean_score": -13.169715369585901
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_standard",
      "decoder": "beam_exact_w1",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.0625,
      "mean_expansions": 78.0,
      "mean_score": -15.645599697716534
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_standard",
      "decoder": "beam_semantic_w0.5",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 14.4375,
      "oracle_accuracy": 0.125,
      "mean_expansions": 99.9375,
      "mean_score": -12.050580549053848
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_standard",
      "decoder": "beam_semantic_w1",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 17.8125,
      "oracle_accuracy": 0.0625,
      "mean_expansions": 131.25,
      "mean_score": -13.816622039303184
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paraphrase",
      "decoder": "greedy",
      "n": 16,
      "accuracy": 0.1875,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.1875,
      "mean_expansions": 1.0,
      "mean_score": -54.53128854930401
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paraphrase",
      "decoder": "local_answer",
      "n": 16,
      "accuracy": 0.1875,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.1875,
      "mean_expansions": 91.5625,
      "mean_score": -54.53128854930401
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paraphrase",
      "decoder": "beam_logprob",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 12.0,
      "oracle_accuracy": 0.3125,
      "mean_expansions": 83.0,
      "mean_score": -10.290128901600838
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paraphrase",
      "decoder": "beam_exact_w0.5",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0625,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.3125,
      "mean_expansions": 78.0,
      "mean_score": -13.164054941385984
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paraphrase",
      "decoder": "beam_exact_w1",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0625,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.3125,
      "mean_expansions": 78.0,
      "mean_score": -15.640768997371197
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paraphrase",
      "decoder": "beam_semantic_w0.5",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 14.0,
      "oracle_accuracy": 0.3125,
      "mean_expansions": 98.0,
      "mean_score": -12.056816628202796
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paraphrase",
      "decoder": "beam_semantic_w1",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 16.875,
      "oracle_accuracy": 0.3125,
      "mean_expansions": 126.1875,
      "mean_score": -13.823504354804754
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paired",
      "decoder": "greedy",
      "n": 16,
      "accuracy": 0.25,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 1.0,
      "mean_score": -54.56863175705075
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paired",
      "decoder": "local_answer",
      "n": 16,
      "accuracy": 0.25,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 90.875,
      "mean_score": -54.56863175705075
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paired",
      "decoder": "beam_logprob",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 12.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 83.0,
      "mean_score": -10.34575367718935
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paired",
      "decoder": "beam_exact_w0.5",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 78.0,
      "mean_score": -13.23045794479549
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paired",
      "decoder": "beam_exact_w1",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 78.0,
      "mean_score": -15.714474398642778
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paired",
      "decoder": "beam_semantic_w0.5",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 14.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 98.0,
      "mean_score": -12.112335826270282
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "fresh_paired",
      "decoder": "beam_semantic_w1",
      "n": 16,
      "accuracy": 0.0,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 16.4375,
      "oracle_accuracy": 0.25,
      "mean_expansions": 124.25,
      "mean_score": -13.878917975351214
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "hard_composition",
      "decoder": "greedy",
      "n": 16,
      "accuracy": 0.125,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.125,
      "mean_expansions": 1.0,
      "mean_score": -55.08675675466657
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "hard_composition",
      "decoder": "local_answer",
      "n": 16,
      "accuracy": 0.125,
      "program_exact": 0.0,
      "valid_rate": 1.0,
      "mean_completed": 1.0,
      "oracle_accuracy": 0.125,
      "mean_expansions": 92.25,
      "mean_score": -55.08675675466657
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "hard_composition",
      "decoder": "beam_logprob",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0625,
      "valid_rate": 1.0,
      "mean_completed": 12.0,
      "oracle_accuracy": 0.375,
      "mean_expansions": 83.0,
      "mean_score": -10.303512908518314
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "hard_composition",
      "decoder": "beam_exact_w0.5",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0625,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 78.0,
      "mean_score": -13.186987661290914
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "hard_composition",
      "decoder": "beam_exact_w1",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0625,
      "valid_rate": 1.0,
      "mean_completed": 10.0,
      "oracle_accuracy": 0.25,
      "mean_expansions": 78.0,
      "mean_score": -15.669939854182303
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "hard_composition",
      "decoder": "beam_semantic_w0.5",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0625,
      "valid_rate": 1.0,
      "mean_completed": 14.4375,
      "oracle_accuracy": 0.375,
      "mean_expansions": 99.9375,
      "mean_score": -12.070581158157438
    },
    {
      "run": "smoke_semantic_prefix_value",
      "phase": "final",
      "split": "hard_composition",
      "decoder": "beam_semantic_w1",
      "n": 16,
      "accuracy": 0.0625,
      "program_exact": 0.0625,
      "valid_rate": 1.0,
      "mean_completed": 18.5,
      "oracle_accuracy": 0.3125,
      "mean_expansions": 134.75,
      "mean_score": -13.837649407796562
    }
  ]
}