{
  "metadata": {
    "python": "3.12.3",
    "platform": "Linux-6.8.0-62-generic-x86_64-with-glibc2.39",
    "torch": "2.8.0+cu128",
    "cuda_available": true,
    "gpu_name": "NVIDIA RTX 6000 Ada Generation",
    "args": {
      "run_name": "pilot_context_trace_reranker_s96",
      "compiler_checkpoint": "large_artifacts/qwen_context_trace_verifier/checkpoints/fixed_mixed_vm_trace_compiler_s512",
      "train_examples": 96,
      "val_examples": 48,
      "eval_examples": 48,
      "eval_pairs": 32,
      "domain_eval_examples": 0,
      "train_min_len": 1,
      "train_max_len": 6,
      "eval_length": 6,
      "val_length": 6,
      "hard_length": 8,
      "harder_length": 10,
      "domains": "arithmetic,calendar,unit,list,boolean,lookup",
      "train_template_mode": "mixed",
      "repair_topk": 3,
      "repair_max_edits": 2,
      "repair_max_pair_arg_slots": 10,
      "qwen_batch_size": 8,
      "max_length": 512,
      "trace_d_model": 96,
      "trace_layers": 2,
      "trace_heads": 4,
      "trace_ff_mult": 4,
      "verifier_dropout": 0.05,
      "verifier_epochs": 8,
      "verifier_lr": 0.001,
      "verifier_weight_decay": 0.0001,
      "positive_label": "answer",
      "no_positive_base_weight": 0.2,
      "base_positive_group_weight": 0.5,
      "repairable_group_weight": 4.0,
      "no_positive_group_weight": 1.0,
      "hard_negative_weight": 0.4,
      "hard_negative_margin": 1.0,
      "hard_negative_prefix_threshold": 0.5,
      "hard_negative_max_edits": 1,
      "pair_train_weight": 0.1,
      "pair_require_state_consistency": false,
      "grad_clip": 1.0,
      "pair_top_m": 20,
      "pair_bonus": 2.0,
      "seed": 211,
      "eval_seed": 94021
    },
    "compiler_args": {
      "model_id": "Qwen/Qwen3-4B",
      "value_modulus": 97,
      "max_steps": 10,
      "head_width": 512,
      "rank_temperature": 1.0,
      "arg_window": 16,
      "arg_distance_temperature": 4.0
    },
    "feature_names": [
      "bias",
      "candidate_prior",
      "prior_delta",
      "candidate_prior_per_slot",
      "prior_delta_per_edit",
      "base_prior",
      "edit_count",
      "changed",
      "init_edit",
      "op_edit_count",
      "arg_edit_count",
      "edit_fraction",
      "first_edit_pos_norm",
      "last_edit_pos_norm",
      "mean_edit_pos_norm",
      "edit_span_norm",
      "mean_new_rank",
      "max_new_rank",
      "mean_changed_margin",
      "min_changed_margin",
      "mean_changed_entropy",
      "mean_logp_delta",
      "min_logp_delta",
      "max_logp_delta",
      "base_init_margin",
      "base_op_margin_mean",
      "base_op_margin_min",
      "base_arg_margin_mean",
      "base_arg_margin_min",
      "base_op_entropy_mean",
      "base_arg_entropy_mean",
      "candidate_answer_norm",
      "candidate_final_soft_logp",
      "candidate_state_soft_logp_mean",
      "candidate_state_soft_logp_min",
      "base_final_soft_logp",
      "base_state_soft_logp_mean",
      "answer_equals_base",
      "state_same_fraction_as_base",
      "state_prefix_same_as_base",
      "mean_old_value_norm",
      "mean_new_value_norm",
      "mean_abs_value_delta_norm",
      "arg_new_value_mean_norm",
      "arg_abs_delta_mean_norm",
      "op_new_add_frac",
      "op_new_sub_frac",
      "op_new_mul_frac",
      "op_new_add7_frac",
      "op_new_sub7_frac",
      "op_new_set_frac",
      "op_new_max_frac",
      "op_new_min_frac",
      "op_new_xor_frac",
      "op_new_gt_frac",
      "candidate_add_frac",
      "candidate_sub_frac",
      "candidate_mul_frac",
      "candidate_add7_frac",
      "candidate_sub7_frac",
      "candidate_set_frac",
      "candidate_max_frac",
      "candidate_min_frac",
      "candidate_xor_frac",
      "candidate_gt_frac",
      "candidate_arg_mean_norm",
      "candidate_arg_std_norm",
      "base_add_frac",
      "base_sub_frac",
      "base_mul_frac",
      "base_add7_frac",
      "base_sub7_frac",
      "base_set_frac",
      "base_max_frac",
      "base_min_frac",
      "base_xor_frac",
      "base_gt_frac"
    ],
    "trace_feature_names": [
      "is_global",
      "is_step",
      "active",
      "step_pos_norm",
      "candidate_prior",
      "prior_delta",
      "edit_count",
      "init_value_norm",
      "init_changed",
      "answer_norm",
      "final_soft_logp",
      "base_final_soft_logp",
      "answer_equals_base",
      "op_add",
      "op_sub",
      "op_mul",
      "op_add7",
      "op_sub7",
      "op_set",
      "op_max",
      "op_min",
      "op_xor",
      "op_gt",
      "arg_norm",
      "state_norm",
      "state_soft_logp",
      "base_op_add",
      "base_op_sub",
      "base_op_mul",
      "base_op_add7",
      "base_op_sub7",
      "base_op_set",
      "base_op_max",
      "base_op_min",
      "base_op_xor",
      "base_op_gt",
      "base_arg_norm",
      "base_state_norm",
      "base_state_soft_logp",
      "op_changed",
      "arg_changed",
      "state_same_as_base",
      "op_logp",
      "arg_logp",
      "op_logp_delta",
      "arg_logp_delta",
      "op_margin",
      "arg_margin",
      "op_entropy",
      "arg_entropy"
    ],
    "train_seconds": 35.00407648086548
  },
  "metrics": {
    "train_len1_6": {
      "n": 96.0,
      "oracle_found_fraction": 1.0,
      "avg_candidates": 55.083333333333336,
      "avg_positive_candidates": 13.166666666666666,
      "avg_state_exact_candidates": 3.5104166666666665,
      "avg_program_exact_candidates": 0.9583333333333334,
      "base_executor_accuracy": 0.75,
      "base_state_exact": 0.6770833333333334,
      "base_program_exact": 0.6354166666666666,
      "base_state_prefix_fraction": 0.8126736111111109,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 16.0,
      "domain_arithmetic_base_executor_accuracy": 0.5,
      "domain_arithmetic_base_state_exact": 0.5,
      "domain_arithmetic_base_state_prefix_fraction": 0.7083333333333333,
      "domain_calendar_n": 16.0,
      "domain_calendar_base_executor_accuracy": 0.875,
      "domain_calendar_base_state_exact": 0.8125,
      "domain_calendar_base_state_prefix_fraction": 0.8854166666666667,
      "domain_unit_n": 16.0,
      "domain_unit_base_executor_accuracy": 0.5,
      "domain_unit_base_state_exact": 0.5,
      "domain_unit_base_state_prefix_fraction": 0.771875,
      "domain_list_n": 16.0,
      "domain_list_base_executor_accuracy": 0.9375,
      "domain_list_base_state_exact": 0.9375,
      "domain_list_base_state_prefix_fraction": 0.9791666666666667,
      "domain_boolean_n": 16.0,
      "domain_boolean_base_executor_accuracy": 0.9375,
      "domain_boolean_base_state_exact": 0.625,
      "domain_boolean_base_state_prefix_fraction": 0.7270833333333334,
      "domain_lookup_n": 16.0,
      "domain_lookup_base_executor_accuracy": 0.75,
      "domain_lookup_base_state_exact": 0.6875,
      "domain_lookup_base_state_prefix_fraction": 0.8041666666666667,
      "prior_executor_accuracy": 0.75,
      "prior_state_exact": 0.6770833333333334,
      "prior_program_exact": 0.6354166666666666,
      "prior_state_prefix_fraction": 0.8126736111111109,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.65625,
      "soft_trace_state_exact": 0.53125,
      "soft_trace_program_exact": 0.5,
      "soft_trace_state_prefix_fraction": 0.735763888888889,
      "soft_trace_changed_fraction": 0.28125,
      "soft_trace_avg_edits": 0.3229166666666667,
      "oracle_executor_accuracy": 1.0,
      "oracle_state_exact": 0.875,
      "oracle_program_exact": 0.8333333333333334,
      "oracle_state_prefix_fraction": 0.9093749999999998,
      "oracle_changed_fraction": 0.25,
      "oracle_avg_edits": 0.3020833333333333,
      "domain_arithmetic_oracle_executor_accuracy": 1.0,
      "domain_arithmetic_oracle_state_exact": 1.0,
      "domain_arithmetic_oracle_state_prefix_fraction": 1.0,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.875,
      "domain_calendar_oracle_state_prefix_fraction": 0.9166666666666666,
      "domain_unit_oracle_executor_accuracy": 1.0,
      "domain_unit_oracle_state_exact": 0.875,
      "domain_unit_oracle_state_prefix_fraction": 0.90625,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 1.0,
      "domain_list_oracle_state_prefix_fraction": 1.0,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.625,
      "domain_boolean_oracle_state_prefix_fraction": 0.7270833333333334,
      "domain_lookup_oracle_executor_accuracy": 1.0,
      "domain_lookup_oracle_state_exact": 0.875,
      "domain_lookup_oracle_state_prefix_fraction": 0.9062500000000001,
      "learned_executor_accuracy": 0.8333333333333334,
      "learned_state_exact": 0.7291666666666666,
      "learned_program_exact": 0.6770833333333334,
      "learned_state_prefix_fraction": 0.8248263888888886,
      "learned_changed_fraction": 0.1875,
      "learned_avg_edits": 0.19791666666666666,
      "domain_arithmetic_learned_executor_accuracy": 0.6875,
      "domain_arithmetic_learned_state_exact": 0.625,
      "domain_arithmetic_learned_state_prefix_fraction": 0.7395833333333333,
      "domain_calendar_learned_executor_accuracy": 0.875,
      "domain_calendar_learned_state_exact": 0.8125,
      "domain_calendar_learned_state_prefix_fraction": 0.8854166666666667,
      "domain_unit_learned_executor_accuracy": 0.6875,
      "domain_unit_learned_state_exact": 0.625,
      "domain_unit_learned_state_prefix_fraction": 0.7760416666666667,
      "domain_list_learned_executor_accuracy": 0.9375,
      "domain_list_learned_state_exact": 0.9375,
      "domain_list_learned_state_prefix_fraction": 0.9791666666666667,
      "domain_boolean_learned_executor_accuracy": 0.9375,
      "domain_boolean_learned_state_exact": 0.625,
      "domain_boolean_learned_state_prefix_fraction": 0.7270833333333334,
      "domain_lookup_learned_executor_accuracy": 0.875,
      "domain_lookup_learned_state_exact": 0.75,
      "domain_lookup_learned_state_prefix_fraction": 0.8416666666666667,
      "learned_oracle_gap_recovered": 0.3333333333333335
    },
    "val_mixed_len6": {
      "n": 48.0,
      "oracle_found_fraction": 0.9791666666666666,
      "avg_candidates": 111.0,
      "avg_positive_candidates": 29.791666666666668,
      "avg_state_exact_candidates": 6.5,
      "avg_program_exact_candidates": 0.9166666666666666,
      "base_executor_accuracy": 0.6041666666666666,
      "base_state_exact": 0.4583333333333333,
      "base_program_exact": 0.375,
      "base_state_prefix_fraction": 0.670138888888889,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 8.0,
      "domain_arithmetic_base_executor_accuracy": 0.25,
      "domain_arithmetic_base_state_exact": 0.25,
      "domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "domain_calendar_n": 8.0,
      "domain_calendar_base_executor_accuracy": 0.5,
      "domain_calendar_base_state_exact": 0.5,
      "domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "domain_unit_n": 8.0,
      "domain_unit_base_executor_accuracy": 0.375,
      "domain_unit_base_state_exact": 0.375,
      "domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "domain_list_n": 8.0,
      "domain_list_base_executor_accuracy": 0.75,
      "domain_list_base_state_exact": 0.5,
      "domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "domain_boolean_n": 8.0,
      "domain_boolean_base_executor_accuracy": 0.875,
      "domain_boolean_base_state_exact": 0.25,
      "domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "domain_lookup_n": 8.0,
      "domain_lookup_base_executor_accuracy": 0.875,
      "domain_lookup_base_state_exact": 0.875,
      "domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "prior_executor_accuracy": 0.6041666666666666,
      "prior_state_exact": 0.4583333333333333,
      "prior_program_exact": 0.375,
      "prior_state_prefix_fraction": 0.670138888888889,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.3958333333333333,
      "soft_trace_state_exact": 0.22916666666666666,
      "soft_trace_program_exact": 0.1875,
      "soft_trace_state_prefix_fraction": 0.5416666666666667,
      "soft_trace_changed_fraction": 0.6666666666666666,
      "soft_trace_avg_edits": 0.8125,
      "oracle_executor_accuracy": 0.9791666666666666,
      "oracle_state_exact": 0.7708333333333334,
      "oracle_program_exact": 0.6666666666666666,
      "oracle_state_prefix_fraction": 0.8263888888888888,
      "oracle_changed_fraction": 0.375,
      "oracle_avg_edits": 0.4375,
      "domain_arithmetic_oracle_executor_accuracy": 0.875,
      "domain_arithmetic_oracle_state_exact": 0.875,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.75,
      "domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "domain_unit_oracle_executor_accuracy": 1.0,
      "domain_unit_oracle_state_exact": 0.875,
      "domain_unit_oracle_state_prefix_fraction": 0.875,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.75,
      "domain_list_oracle_state_prefix_fraction": 0.8125,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.375,
      "domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "domain_lookup_oracle_executor_accuracy": 1.0,
      "domain_lookup_oracle_state_exact": 1.0,
      "domain_lookup_oracle_state_prefix_fraction": 1.0,
      "learned_executor_accuracy": 0.6458333333333334,
      "learned_state_exact": 0.5,
      "learned_program_exact": 0.4166666666666667,
      "learned_state_prefix_fraction": 0.6875,
      "learned_changed_fraction": 0.14583333333333334,
      "learned_avg_edits": 0.14583333333333334,
      "domain_arithmetic_learned_executor_accuracy": 0.375,
      "domain_arithmetic_learned_state_exact": 0.375,
      "domain_arithmetic_learned_state_prefix_fraction": 0.6458333333333334,
      "domain_calendar_learned_executor_accuracy": 0.5,
      "domain_calendar_learned_state_exact": 0.5,
      "domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "domain_unit_learned_executor_accuracy": 0.5,
      "domain_unit_learned_state_exact": 0.5,
      "domain_unit_learned_state_prefix_fraction": 0.6875,
      "domain_list_learned_executor_accuracy": 0.75,
      "domain_list_learned_state_exact": 0.5,
      "domain_list_learned_state_prefix_fraction": 0.7708333333333334,
      "domain_boolean_learned_executor_accuracy": 0.875,
      "domain_boolean_learned_state_exact": 0.25,
      "domain_boolean_learned_state_prefix_fraction": 0.39583333333333337,
      "domain_lookup_learned_executor_accuracy": 0.875,
      "domain_lookup_learned_state_exact": 0.875,
      "domain_lookup_learned_state_prefix_fraction": 0.9166666666666666,
      "learned_oracle_gap_recovered": 0.11111111111111131
    },
    "fresh_standard_len6": {
      "n": 48.0,
      "oracle_found_fraction": 1.0,
      "avg_candidates": 111.0,
      "avg_positive_candidates": 30.333333333333332,
      "avg_state_exact_candidates": 6.229166666666667,
      "avg_program_exact_candidates": 0.8958333333333334,
      "base_executor_accuracy": 0.7708333333333334,
      "base_state_exact": 0.7083333333333334,
      "base_program_exact": 0.625,
      "base_state_prefix_fraction": 0.7881944444444443,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 8.0,
      "domain_arithmetic_base_executor_accuracy": 0.625,
      "domain_arithmetic_base_state_exact": 0.625,
      "domain_arithmetic_base_state_prefix_fraction": 0.7083333333333334,
      "domain_calendar_n": 8.0,
      "domain_calendar_base_executor_accuracy": 0.875,
      "domain_calendar_base_state_exact": 0.875,
      "domain_calendar_base_state_prefix_fraction": 0.9791666666666667,
      "domain_unit_n": 8.0,
      "domain_unit_base_executor_accuracy": 0.5,
      "domain_unit_base_state_exact": 0.5,
      "domain_unit_base_state_prefix_fraction": 0.6458333333333333,
      "domain_list_n": 8.0,
      "domain_list_base_executor_accuracy": 0.875,
      "domain_list_base_state_exact": 0.875,
      "domain_list_base_state_prefix_fraction": 0.8958333333333333,
      "domain_boolean_n": 8.0,
      "domain_boolean_base_executor_accuracy": 0.75,
      "domain_boolean_base_state_exact": 0.5,
      "domain_boolean_base_state_prefix_fraction": 0.6041666666666666,
      "domain_lookup_n": 8.0,
      "domain_lookup_base_executor_accuracy": 1.0,
      "domain_lookup_base_state_exact": 0.875,
      "domain_lookup_base_state_prefix_fraction": 0.8958333333333334,
      "prior_executor_accuracy": 0.7708333333333334,
      "prior_state_exact": 0.7083333333333334,
      "prior_program_exact": 0.625,
      "prior_state_prefix_fraction": 0.7881944444444443,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.4583333333333333,
      "soft_trace_state_exact": 0.3125,
      "soft_trace_program_exact": 0.25,
      "soft_trace_state_prefix_fraction": 0.6493055555555556,
      "soft_trace_changed_fraction": 0.5625,
      "soft_trace_avg_edits": 0.6458333333333334,
      "oracle_executor_accuracy": 1.0,
      "oracle_state_exact": 0.8333333333333334,
      "oracle_program_exact": 0.75,
      "oracle_state_prefix_fraction": 0.8645833333333334,
      "oracle_changed_fraction": 0.22916666666666666,
      "oracle_avg_edits": 0.2708333333333333,
      "domain_arithmetic_oracle_executor_accuracy": 1.0,
      "domain_arithmetic_oracle_state_exact": 0.875,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.8958333333333334,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.875,
      "domain_calendar_oracle_state_prefix_fraction": 0.9375,
      "domain_unit_oracle_executor_accuracy": 1.0,
      "domain_unit_oracle_state_exact": 1.0,
      "domain_unit_oracle_state_prefix_fraction": 1.0,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.875,
      "domain_list_oracle_state_prefix_fraction": 0.8958333333333333,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.5,
      "domain_boolean_oracle_state_prefix_fraction": 0.5625,
      "domain_lookup_oracle_executor_accuracy": 1.0,
      "domain_lookup_oracle_state_exact": 0.875,
      "domain_lookup_oracle_state_prefix_fraction": 0.8958333333333334,
      "learned_executor_accuracy": 0.75,
      "learned_state_exact": 0.6875,
      "learned_program_exact": 0.6041666666666666,
      "learned_state_prefix_fraction": 0.784722222222222,
      "learned_changed_fraction": 0.10416666666666667,
      "learned_avg_edits": 0.10416666666666667,
      "domain_arithmetic_learned_executor_accuracy": 0.5,
      "domain_arithmetic_learned_state_exact": 0.5,
      "domain_arithmetic_learned_state_prefix_fraction": 0.6875,
      "domain_calendar_learned_executor_accuracy": 0.875,
      "domain_calendar_learned_state_exact": 0.875,
      "domain_calendar_learned_state_prefix_fraction": 0.9791666666666667,
      "domain_unit_learned_executor_accuracy": 0.5,
      "domain_unit_learned_state_exact": 0.5,
      "domain_unit_learned_state_prefix_fraction": 0.6458333333333333,
      "domain_list_learned_executor_accuracy": 0.875,
      "domain_list_learned_state_exact": 0.875,
      "domain_list_learned_state_prefix_fraction": 0.8958333333333333,
      "domain_boolean_learned_executor_accuracy": 0.75,
      "domain_boolean_learned_state_exact": 0.5,
      "domain_boolean_learned_state_prefix_fraction": 0.6041666666666666,
      "domain_lookup_learned_executor_accuracy": 1.0,
      "domain_lookup_learned_state_exact": 0.875,
      "domain_lookup_learned_state_prefix_fraction": 0.8958333333333334,
      "learned_oracle_gap_recovered": -0.09090909090909109
    },
    "fresh_paraphrase_len6": {
      "n": 48.0,
      "oracle_found_fraction": 1.0,
      "avg_candidates": 111.0,
      "avg_positive_candidates": 29.0625,
      "avg_state_exact_candidates": 4.270833333333333,
      "avg_program_exact_candidates": 0.8333333333333334,
      "base_executor_accuracy": 0.5625,
      "base_state_exact": 0.4166666666666667,
      "base_program_exact": 0.375,
      "base_state_prefix_fraction": 0.6597222222222222,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 8.0,
      "domain_arithmetic_base_executor_accuracy": 0.625,
      "domain_arithmetic_base_state_exact": 0.625,
      "domain_arithmetic_base_state_prefix_fraction": 0.875,
      "domain_calendar_n": 8.0,
      "domain_calendar_base_executor_accuracy": 0.375,
      "domain_calendar_base_state_exact": 0.375,
      "domain_calendar_base_state_prefix_fraction": 0.6666666666666666,
      "domain_unit_n": 8.0,
      "domain_unit_base_executor_accuracy": 0.125,
      "domain_unit_base_state_exact": 0.125,
      "domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "domain_list_n": 8.0,
      "domain_list_base_executor_accuracy": 0.875,
      "domain_list_base_state_exact": 0.625,
      "domain_list_base_state_prefix_fraction": 0.7291666666666667,
      "domain_boolean_n": 8.0,
      "domain_boolean_base_executor_accuracy": 0.75,
      "domain_boolean_base_state_exact": 0.25,
      "domain_boolean_base_state_prefix_fraction": 0.35416666666666663,
      "domain_lookup_n": 8.0,
      "domain_lookup_base_executor_accuracy": 0.625,
      "domain_lookup_base_state_exact": 0.5,
      "domain_lookup_base_state_prefix_fraction": 0.7291666666666666,
      "prior_executor_accuracy": 0.5625,
      "prior_state_exact": 0.4166666666666667,
      "prior_program_exact": 0.375,
      "prior_state_prefix_fraction": 0.6597222222222222,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.375,
      "soft_trace_state_exact": 0.22916666666666666,
      "soft_trace_program_exact": 0.20833333333333334,
      "soft_trace_state_prefix_fraction": 0.5729166666666666,
      "soft_trace_changed_fraction": 0.5416666666666666,
      "soft_trace_avg_edits": 0.625,
      "oracle_executor_accuracy": 1.0,
      "oracle_state_exact": 0.6666666666666666,
      "oracle_program_exact": 0.5833333333333334,
      "oracle_state_prefix_fraction": 0.7534722222222223,
      "oracle_changed_fraction": 0.4375,
      "oracle_avg_edits": 0.5416666666666666,
      "domain_arithmetic_oracle_executor_accuracy": 1.0,
      "domain_arithmetic_oracle_state_exact": 0.875,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.8958333333333334,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.75,
      "domain_calendar_oracle_state_prefix_fraction": 0.8125000000000001,
      "domain_unit_oracle_executor_accuracy": 1.0,
      "domain_unit_oracle_state_exact": 0.5,
      "domain_unit_oracle_state_prefix_fraction": 0.7083333333333334,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.625,
      "domain_list_oracle_state_prefix_fraction": 0.7291666666666667,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.5,
      "domain_boolean_oracle_state_prefix_fraction": 0.5625,
      "domain_lookup_oracle_executor_accuracy": 1.0,
      "domain_lookup_oracle_state_exact": 0.75,
      "domain_lookup_oracle_state_prefix_fraction": 0.8125,
      "learned_executor_accuracy": 0.5416666666666666,
      "learned_state_exact": 0.3958333333333333,
      "learned_program_exact": 0.3333333333333333,
      "learned_state_prefix_fraction": 0.6527777777777778,
      "learned_changed_fraction": 0.2708333333333333,
      "learned_avg_edits": 0.2708333333333333,
      "domain_arithmetic_learned_executor_accuracy": 0.375,
      "domain_arithmetic_learned_state_exact": 0.375,
      "domain_arithmetic_learned_state_prefix_fraction": 0.7916666666666666,
      "domain_calendar_learned_executor_accuracy": 0.375,
      "domain_calendar_learned_state_exact": 0.375,
      "domain_calendar_learned_state_prefix_fraction": 0.6666666666666666,
      "domain_unit_learned_executor_accuracy": 0.125,
      "domain_unit_learned_state_exact": 0.125,
      "domain_unit_learned_state_prefix_fraction": 0.6041666666666666,
      "domain_list_learned_executor_accuracy": 0.875,
      "domain_list_learned_state_exact": 0.625,
      "domain_list_learned_state_prefix_fraction": 0.7291666666666667,
      "domain_boolean_learned_executor_accuracy": 0.75,
      "domain_boolean_learned_state_exact": 0.25,
      "domain_boolean_learned_state_prefix_fraction": 0.35416666666666663,
      "domain_lookup_learned_executor_accuracy": 0.75,
      "domain_lookup_learned_state_exact": 0.625,
      "domain_lookup_learned_state_prefix_fraction": 0.7708333333333333,
      "learned_oracle_gap_recovered": -0.04761904761904771
    },
    "fresh_paired_len6": {
      "n": 64.0,
      "oracle_found_fraction": 0.984375,
      "avg_candidates": 111.0,
      "avg_positive_candidates": 29.40625,
      "avg_state_exact_candidates": 4.25,
      "avg_program_exact_candidates": 0.96875,
      "base_executor_accuracy": 0.65625,
      "base_state_exact": 0.53125,
      "base_program_exact": 0.46875,
      "base_state_prefix_fraction": 0.7656250000000003,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 12.0,
      "domain_arithmetic_base_executor_accuracy": 0.6666666666666666,
      "domain_arithmetic_base_state_exact": 0.6666666666666666,
      "domain_arithmetic_base_state_prefix_fraction": 0.9027777777777778,
      "domain_calendar_n": 12.0,
      "domain_calendar_base_executor_accuracy": 0.3333333333333333,
      "domain_calendar_base_state_exact": 0.25,
      "domain_calendar_base_state_prefix_fraction": 0.7777777777777778,
      "domain_unit_n": 10.0,
      "domain_unit_base_executor_accuracy": 0.3,
      "domain_unit_base_state_exact": 0.3,
      "domain_unit_base_state_prefix_fraction": 0.7166666666666666,
      "domain_list_n": 10.0,
      "domain_list_base_executor_accuracy": 1.0,
      "domain_list_base_state_exact": 0.8,
      "domain_list_base_state_prefix_fraction": 0.8,
      "domain_boolean_n": 10.0,
      "domain_boolean_base_executor_accuracy": 0.9,
      "domain_boolean_base_state_exact": 0.4,
      "domain_boolean_base_state_prefix_fraction": 0.5666666666666667,
      "domain_lookup_n": 10.0,
      "domain_lookup_base_executor_accuracy": 0.8,
      "domain_lookup_base_state_exact": 0.8,
      "domain_lookup_base_state_prefix_fraction": 0.8,
      "prior_executor_accuracy": 0.65625,
      "prior_state_exact": 0.53125,
      "prior_program_exact": 0.46875,
      "prior_state_prefix_fraction": 0.7656250000000003,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.375,
      "soft_trace_state_exact": 0.21875,
      "soft_trace_program_exact": 0.1875,
      "soft_trace_state_prefix_fraction": 0.5963541666666666,
      "soft_trace_changed_fraction": 0.609375,
      "soft_trace_avg_edits": 0.84375,
      "oracle_executor_accuracy": 0.984375,
      "oracle_state_exact": 0.734375,
      "oracle_program_exact": 0.671875,
      "oracle_state_prefix_fraction": 0.7994791666666666,
      "oracle_changed_fraction": 0.328125,
      "oracle_avg_edits": 0.390625,
      "domain_arithmetic_oracle_executor_accuracy": 1.0,
      "domain_arithmetic_oracle_state_exact": 0.9166666666666666,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.9305555555555555,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.6666666666666666,
      "domain_calendar_oracle_state_prefix_fraction": 0.75,
      "domain_unit_oracle_executor_accuracy": 0.9,
      "domain_unit_oracle_state_exact": 0.6,
      "domain_unit_oracle_state_prefix_fraction": 0.7666666666666666,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.8,
      "domain_list_oracle_state_prefix_fraction": 0.8,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.4,
      "domain_boolean_oracle_state_prefix_fraction": 0.5333333333333334,
      "domain_lookup_oracle_executor_accuracy": 1.0,
      "domain_lookup_oracle_state_exact": 1.0,
      "domain_lookup_oracle_state_prefix_fraction": 1.0,
      "learned_executor_accuracy": 0.578125,
      "learned_state_exact": 0.453125,
      "learned_program_exact": 0.390625,
      "learned_state_prefix_fraction": 0.7213541666666669,
      "learned_changed_fraction": 0.1875,
      "learned_avg_edits": 0.1875,
      "domain_arithmetic_learned_executor_accuracy": 0.25,
      "domain_arithmetic_learned_state_exact": 0.25,
      "domain_arithmetic_learned_state_prefix_fraction": 0.6388888888888887,
      "domain_calendar_learned_executor_accuracy": 0.3333333333333333,
      "domain_calendar_learned_state_exact": 0.25,
      "domain_calendar_learned_state_prefix_fraction": 0.7777777777777778,
      "domain_unit_learned_executor_accuracy": 0.3,
      "domain_unit_learned_state_exact": 0.3,
      "domain_unit_learned_state_prefix_fraction": 0.7499999999999998,
      "domain_list_learned_executor_accuracy": 1.0,
      "domain_list_learned_state_exact": 0.8,
      "domain_list_learned_state_prefix_fraction": 0.8,
      "domain_boolean_learned_executor_accuracy": 0.9,
      "domain_boolean_learned_state_exact": 0.4,
      "domain_boolean_learned_state_prefix_fraction": 0.5666666666666667,
      "domain_lookup_learned_executor_accuracy": 0.8,
      "domain_lookup_learned_state_exact": 0.8,
      "domain_lookup_learned_state_prefix_fraction": 0.8,
      "learned_oracle_gap_recovered": -0.23809523809523808,
      "pair_rerank_executor_accuracy": 0.640625,
      "pair_rerank_state_exact": 0.515625,
      "pair_rerank_program_exact": 0.453125,
      "pair_rerank_state_prefix_fraction": 0.744791666666667,
      "pair_rerank_changed_fraction": 0.21875,
      "pair_rerank_avg_edits": 0.25,
      "domain_arithmetic_pair_rerank_executor_accuracy": 0.5,
      "domain_arithmetic_pair_rerank_state_exact": 0.5,
      "domain_arithmetic_pair_rerank_state_prefix_fraction": 0.75,
      "domain_calendar_pair_rerank_executor_accuracy": 0.3333333333333333,
      "domain_calendar_pair_rerank_state_exact": 0.25,
      "domain_calendar_pair_rerank_state_prefix_fraction": 0.7777777777777778,
      "domain_unit_pair_rerank_executor_accuracy": 0.4,
      "domain_unit_pair_rerank_state_exact": 0.4,
      "domain_unit_pair_rerank_state_prefix_fraction": 0.7666666666666665,
      "domain_list_pair_rerank_executor_accuracy": 1.0,
      "domain_list_pair_rerank_state_exact": 0.8,
      "domain_list_pair_rerank_state_prefix_fraction": 0.8,
      "domain_boolean_pair_rerank_executor_accuracy": 0.9,
      "domain_boolean_pair_rerank_state_exact": 0.4,
      "domain_boolean_pair_rerank_state_prefix_fraction": 0.5666666666666667,
      "domain_lookup_pair_rerank_executor_accuracy": 0.8,
      "domain_lookup_pair_rerank_state_exact": 0.8,
      "domain_lookup_pair_rerank_state_prefix_fraction": 0.8,
      "learned_pair_answer_consistency": 0.59375,
      "learned_pair_both_correct": 0.46875,
      "learned_pair_both_state_exact": 0.375,
      "learned_pair_program_consistency": 0.59375,
      "learned_pair_state_consistency": 0.59375,
      "pair_rerank_pair_answer_consistency": 0.78125,
      "pair_rerank_pair_both_correct": 0.5625,
      "pair_rerank_pair_both_state_exact": 0.46875,
      "pair_rerank_pair_program_consistency": 0.78125,
      "pair_rerank_pair_state_consistency": 0.78125,
      "oracle_pair_answer_consistency": 0.96875,
      "oracle_pair_both_correct": 0.96875,
      "oracle_pair_both_state_exact": 0.6875,
      "oracle_pair_program_consistency": 0.875,
      "oracle_pair_state_consistency": 0.875,
      "base_pair_answer_consistency": 0.625,
      "base_pair_both_correct": 0.5,
      "base_pair_both_state_exact": 0.40625,
      "base_pair_program_consistency": 0.625,
      "base_pair_state_consistency": 0.625
    },
    "hard_standard_len8": {
      "n": 48.0,
      "oracle_found_fraction": 0.9791666666666666,
      "avg_candidates": 179.0,
      "avg_positive_candidates": 51.604166666666664,
      "avg_state_exact_candidates": 6.541666666666667,
      "avg_program_exact_candidates": 0.8125,
      "base_executor_accuracy": 0.5416666666666666,
      "base_state_exact": 0.3333333333333333,
      "base_program_exact": 0.2708333333333333,
      "base_state_prefix_fraction": 0.6484375,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 8.0,
      "domain_arithmetic_base_executor_accuracy": 0.125,
      "domain_arithmetic_base_state_exact": 0.125,
      "domain_arithmetic_base_state_prefix_fraction": 0.4375,
      "domain_calendar_n": 8.0,
      "domain_calendar_base_executor_accuracy": 0.375,
      "domain_calendar_base_state_exact": 0.25,
      "domain_calendar_base_state_prefix_fraction": 0.6875,
      "domain_unit_n": 8.0,
      "domain_unit_base_executor_accuracy": 0.5,
      "domain_unit_base_state_exact": 0.5,
      "domain_unit_base_state_prefix_fraction": 0.75,
      "domain_list_n": 8.0,
      "domain_list_base_executor_accuracy": 0.875,
      "domain_list_base_state_exact": 0.625,
      "domain_list_base_state_prefix_fraction": 0.78125,
      "domain_boolean_n": 8.0,
      "domain_boolean_base_executor_accuracy": 1.0,
      "domain_boolean_base_state_exact": 0.125,
      "domain_boolean_base_state_prefix_fraction": 0.421875,
      "domain_lookup_n": 8.0,
      "domain_lookup_base_executor_accuracy": 0.375,
      "domain_lookup_base_state_exact": 0.375,
      "domain_lookup_base_state_prefix_fraction": 0.8125,
      "prior_executor_accuracy": 0.5416666666666666,
      "prior_state_exact": 0.3333333333333333,
      "prior_program_exact": 0.2708333333333333,
      "prior_state_prefix_fraction": 0.6484375,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.3541666666666667,
      "soft_trace_state_exact": 0.08333333333333333,
      "soft_trace_program_exact": 0.08333333333333333,
      "soft_trace_state_prefix_fraction": 0.4635416666666667,
      "soft_trace_changed_fraction": 0.7916666666666666,
      "soft_trace_avg_edits": 1.0833333333333333,
      "oracle_executor_accuracy": 0.9791666666666666,
      "oracle_state_exact": 0.5833333333333334,
      "oracle_program_exact": 0.5208333333333334,
      "oracle_state_prefix_fraction": 0.7291666666666666,
      "oracle_changed_fraction": 0.4375,
      "oracle_avg_edits": 0.5416666666666666,
      "domain_arithmetic_oracle_executor_accuracy": 1.0,
      "domain_arithmetic_oracle_state_exact": 0.75,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.796875,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.5,
      "domain_calendar_oracle_state_prefix_fraction": 0.703125,
      "domain_unit_oracle_executor_accuracy": 0.875,
      "domain_unit_oracle_state_exact": 0.75,
      "domain_unit_oracle_state_prefix_fraction": 0.8125,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.625,
      "domain_list_oracle_state_prefix_fraction": 0.78125,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.125,
      "domain_boolean_oracle_state_prefix_fraction": 0.421875,
      "domain_lookup_oracle_executor_accuracy": 1.0,
      "domain_lookup_oracle_state_exact": 0.75,
      "domain_lookup_oracle_state_prefix_fraction": 0.859375,
      "learned_executor_accuracy": 0.5625,
      "learned_state_exact": 0.3541666666666667,
      "learned_program_exact": 0.2916666666666667,
      "learned_state_prefix_fraction": 0.6432291666666666,
      "learned_changed_fraction": 0.20833333333333334,
      "learned_avg_edits": 0.20833333333333334,
      "domain_arithmetic_learned_executor_accuracy": 0.25,
      "domain_arithmetic_learned_state_exact": 0.25,
      "domain_arithmetic_learned_state_prefix_fraction": 0.453125,
      "domain_calendar_learned_executor_accuracy": 0.375,
      "domain_calendar_learned_state_exact": 0.25,
      "domain_calendar_learned_state_prefix_fraction": 0.6875,
      "domain_unit_learned_executor_accuracy": 0.5,
      "domain_unit_learned_state_exact": 0.5,
      "domain_unit_learned_state_prefix_fraction": 0.703125,
      "domain_list_learned_executor_accuracy": 0.875,
      "domain_list_learned_state_exact": 0.625,
      "domain_list_learned_state_prefix_fraction": 0.78125,
      "domain_boolean_learned_executor_accuracy": 1.0,
      "domain_boolean_learned_state_exact": 0.125,
      "domain_boolean_learned_state_prefix_fraction": 0.421875,
      "domain_lookup_learned_executor_accuracy": 0.375,
      "domain_lookup_learned_state_exact": 0.375,
      "domain_lookup_learned_state_prefix_fraction": 0.8125,
      "learned_oracle_gap_recovered": 0.04761904761904771
    },
    "hard_paraphrase_len8": {
      "n": 48.0,
      "oracle_found_fraction": 0.8958333333333334,
      "avg_candidates": 179.0,
      "avg_positive_candidates": 44.0625,
      "avg_state_exact_candidates": 5.333333333333333,
      "avg_program_exact_candidates": 0.7083333333333334,
      "base_executor_accuracy": 0.3958333333333333,
      "base_state_exact": 0.25,
      "base_program_exact": 0.1875,
      "base_state_prefix_fraction": 0.5729166666666666,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 8.0,
      "domain_arithmetic_base_executor_accuracy": 0.125,
      "domain_arithmetic_base_state_exact": 0.125,
      "domain_arithmetic_base_state_prefix_fraction": 0.53125,
      "domain_calendar_n": 8.0,
      "domain_calendar_base_executor_accuracy": 0.375,
      "domain_calendar_base_state_exact": 0.0,
      "domain_calendar_base_state_prefix_fraction": 0.46875,
      "domain_unit_n": 8.0,
      "domain_unit_base_executor_accuracy": 0.0,
      "domain_unit_base_state_exact": 0.0,
      "domain_unit_base_state_prefix_fraction": 0.546875,
      "domain_list_n": 8.0,
      "domain_list_base_executor_accuracy": 0.75,
      "domain_list_base_state_exact": 0.625,
      "domain_list_base_state_prefix_fraction": 0.734375,
      "domain_boolean_n": 8.0,
      "domain_boolean_base_executor_accuracy": 0.75,
      "domain_boolean_base_state_exact": 0.375,
      "domain_boolean_base_state_prefix_fraction": 0.578125,
      "domain_lookup_n": 8.0,
      "domain_lookup_base_executor_accuracy": 0.375,
      "domain_lookup_base_state_exact": 0.375,
      "domain_lookup_base_state_prefix_fraction": 0.578125,
      "prior_executor_accuracy": 0.3958333333333333,
      "prior_state_exact": 0.25,
      "prior_program_exact": 0.1875,
      "prior_state_prefix_fraction": 0.5729166666666666,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.2708333333333333,
      "soft_trace_state_exact": 0.0,
      "soft_trace_program_exact": 0.0,
      "soft_trace_state_prefix_fraction": 0.4479166666666667,
      "soft_trace_changed_fraction": 0.9583333333333334,
      "soft_trace_avg_edits": 1.2916666666666667,
      "oracle_executor_accuracy": 0.8958333333333334,
      "oracle_state_exact": 0.5,
      "oracle_program_exact": 0.4375,
      "oracle_state_prefix_fraction": 0.6536458333333334,
      "oracle_changed_fraction": 0.5,
      "oracle_avg_edits": 0.7708333333333334,
      "domain_arithmetic_oracle_executor_accuracy": 0.625,
      "domain_arithmetic_oracle_state_exact": 0.625,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.75,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.25,
      "domain_calendar_oracle_state_prefix_fraction": 0.546875,
      "domain_unit_oracle_executor_accuracy": 0.875,
      "domain_unit_oracle_state_exact": 0.5,
      "domain_unit_oracle_state_prefix_fraction": 0.609375,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.625,
      "domain_list_oracle_state_prefix_fraction": 0.734375,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.375,
      "domain_boolean_oracle_state_prefix_fraction": 0.578125,
      "domain_lookup_oracle_executor_accuracy": 0.875,
      "domain_lookup_oracle_state_exact": 0.625,
      "domain_lookup_oracle_state_prefix_fraction": 0.703125,
      "learned_executor_accuracy": 0.375,
      "learned_state_exact": 0.25,
      "learned_program_exact": 0.1875,
      "learned_state_prefix_fraction": 0.53125,
      "learned_changed_fraction": 0.4166666666666667,
      "learned_avg_edits": 0.4375,
      "domain_arithmetic_learned_executor_accuracy": 0.125,
      "domain_arithmetic_learned_state_exact": 0.125,
      "domain_arithmetic_learned_state_prefix_fraction": 0.40625,
      "domain_calendar_learned_executor_accuracy": 0.25,
      "domain_calendar_learned_state_exact": 0.0,
      "domain_calendar_learned_state_prefix_fraction": 0.46875,
      "domain_unit_learned_executor_accuracy": 0.0,
      "domain_unit_learned_state_exact": 0.0,
      "domain_unit_learned_state_prefix_fraction": 0.421875,
      "domain_list_learned_executor_accuracy": 0.75,
      "domain_list_learned_state_exact": 0.625,
      "domain_list_learned_state_prefix_fraction": 0.734375,
      "domain_boolean_learned_executor_accuracy": 0.75,
      "domain_boolean_learned_state_exact": 0.375,
      "domain_boolean_learned_state_prefix_fraction": 0.578125,
      "domain_lookup_learned_executor_accuracy": 0.375,
      "domain_lookup_learned_state_exact": 0.375,
      "domain_lookup_learned_state_prefix_fraction": 0.578125,
      "learned_oracle_gap_recovered": -0.04166666666666663
    },
    "harder_standard_len10": {
      "n": 48.0,
      "oracle_found_fraction": 0.9375,
      "avg_candidates": 263.0,
      "avg_positive_candidates": 44.625,
      "avg_state_exact_candidates": 2.8333333333333335,
      "avg_program_exact_candidates": 0.4166666666666667,
      "base_executor_accuracy": 0.22916666666666666,
      "base_state_exact": 0.125,
      "base_program_exact": 0.10416666666666667,
      "base_state_prefix_fraction": 0.55625,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 8.0,
      "domain_arithmetic_base_executor_accuracy": 0.125,
      "domain_arithmetic_base_state_exact": 0.125,
      "domain_arithmetic_base_state_prefix_fraction": 0.5,
      "domain_calendar_n": 8.0,
      "domain_calendar_base_executor_accuracy": 0.0,
      "domain_calendar_base_state_exact": 0.0,
      "domain_calendar_base_state_prefix_fraction": 0.525,
      "domain_unit_n": 8.0,
      "domain_unit_base_executor_accuracy": 0.25,
      "domain_unit_base_state_exact": 0.25,
      "domain_unit_base_state_prefix_fraction": 0.7,
      "domain_list_n": 8.0,
      "domain_list_base_executor_accuracy": 0.375,
      "domain_list_base_state_exact": 0.125,
      "domain_list_base_state_prefix_fraction": 0.6875,
      "domain_boolean_n": 8.0,
      "domain_boolean_base_executor_accuracy": 0.5,
      "domain_boolean_base_state_exact": 0.125,
      "domain_boolean_base_state_prefix_fraction": 0.3875,
      "domain_lookup_n": 8.0,
      "domain_lookup_base_executor_accuracy": 0.125,
      "domain_lookup_base_state_exact": 0.125,
      "domain_lookup_base_state_prefix_fraction": 0.5375,
      "prior_executor_accuracy": 0.22916666666666666,
      "prior_state_exact": 0.125,
      "prior_program_exact": 0.10416666666666667,
      "prior_state_prefix_fraction": 0.55625,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.25,
      "soft_trace_state_exact": 0.0,
      "soft_trace_program_exact": 0.0,
      "soft_trace_state_prefix_fraction": 0.3833333333333333,
      "soft_trace_changed_fraction": 0.9791666666666666,
      "soft_trace_avg_edits": 1.375,
      "oracle_executor_accuracy": 0.9375,
      "oracle_state_exact": 0.2916666666666667,
      "oracle_program_exact": 0.22916666666666666,
      "oracle_state_prefix_fraction": 0.5687500000000002,
      "oracle_changed_fraction": 0.7083333333333334,
      "oracle_avg_edits": 1.0208333333333333,
      "domain_arithmetic_oracle_executor_accuracy": 1.0,
      "domain_arithmetic_oracle_state_exact": 0.5,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.625,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.125,
      "domain_calendar_oracle_state_prefix_fraction": 0.475,
      "domain_unit_oracle_executor_accuracy": 0.875,
      "domain_unit_oracle_state_exact": 0.625,
      "domain_unit_oracle_state_prefix_fraction": 0.725,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.25,
      "domain_list_oracle_state_prefix_fraction": 0.7,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.125,
      "domain_boolean_oracle_state_prefix_fraction": 0.3625,
      "domain_lookup_oracle_executor_accuracy": 0.75,
      "domain_lookup_oracle_state_exact": 0.125,
      "domain_lookup_oracle_state_prefix_fraction": 0.5249999999999999,
      "learned_executor_accuracy": 0.25,
      "learned_state_exact": 0.14583333333333334,
      "learned_program_exact": 0.125,
      "learned_state_prefix_fraction": 0.5708333333333334,
      "learned_changed_fraction": 0.22916666666666666,
      "learned_avg_edits": 0.22916666666666666,
      "domain_arithmetic_learned_executor_accuracy": 0.125,
      "domain_arithmetic_learned_state_exact": 0.125,
      "domain_arithmetic_learned_state_prefix_fraction": 0.5125000000000001,
      "domain_calendar_learned_executor_accuracy": 0.0,
      "domain_calendar_learned_state_exact": 0.0,
      "domain_calendar_learned_state_prefix_fraction": 0.525,
      "domain_unit_learned_executor_accuracy": 0.375,
      "domain_unit_learned_state_exact": 0.375,
      "domain_unit_learned_state_prefix_fraction": 0.775,
      "domain_list_learned_executor_accuracy": 0.375,
      "domain_list_learned_state_exact": 0.125,
      "domain_list_learned_state_prefix_fraction": 0.6875,
      "domain_boolean_learned_executor_accuracy": 0.5,
      "domain_boolean_learned_state_exact": 0.125,
      "domain_boolean_learned_state_prefix_fraction": 0.3875,
      "domain_lookup_learned_executor_accuracy": 0.125,
      "domain_lookup_learned_state_exact": 0.125,
      "domain_lookup_learned_state_prefix_fraction": 0.5375,
      "learned_oracle_gap_recovered": 0.029411764705882363
    },
    "harder_paraphrase_len10": {
      "n": 48.0,
      "oracle_found_fraction": 0.9166666666666666,
      "avg_candidates": 263.0,
      "avg_positive_candidates": 58.083333333333336,
      "avg_state_exact_candidates": 3.875,
      "avg_program_exact_candidates": 0.25,
      "base_executor_accuracy": 0.3333333333333333,
      "base_state_exact": 0.14583333333333334,
      "base_program_exact": 0.125,
      "base_state_prefix_fraction": 0.5291666666666667,
      "base_changed_fraction": 0.0,
      "base_avg_edits": 0.0,
      "domain_arithmetic_n": 8.0,
      "domain_arithmetic_base_executor_accuracy": 0.125,
      "domain_arithmetic_base_state_exact": 0.125,
      "domain_arithmetic_base_state_prefix_fraction": 0.47500000000000003,
      "domain_calendar_n": 8.0,
      "domain_calendar_base_executor_accuracy": 0.25,
      "domain_calendar_base_state_exact": 0.125,
      "domain_calendar_base_state_prefix_fraction": 0.5750000000000001,
      "domain_unit_n": 8.0,
      "domain_unit_base_executor_accuracy": 0.0,
      "domain_unit_base_state_exact": 0.0,
      "domain_unit_base_state_prefix_fraction": 0.325,
      "domain_list_n": 8.0,
      "domain_list_base_executor_accuracy": 0.625,
      "domain_list_base_state_exact": 0.25,
      "domain_list_base_state_prefix_fraction": 0.7,
      "domain_boolean_n": 8.0,
      "domain_boolean_base_executor_accuracy": 0.75,
      "domain_boolean_base_state_exact": 0.25,
      "domain_boolean_base_state_prefix_fraction": 0.47500000000000003,
      "domain_lookup_n": 8.0,
      "domain_lookup_base_executor_accuracy": 0.25,
      "domain_lookup_base_state_exact": 0.125,
      "domain_lookup_base_state_prefix_fraction": 0.625,
      "prior_executor_accuracy": 0.3333333333333333,
      "prior_state_exact": 0.14583333333333334,
      "prior_program_exact": 0.125,
      "prior_state_prefix_fraction": 0.5291666666666667,
      "prior_changed_fraction": 0.0,
      "prior_avg_edits": 0.0,
      "soft_trace_executor_accuracy": 0.25,
      "soft_trace_state_exact": 0.0,
      "soft_trace_program_exact": 0.0,
      "soft_trace_state_prefix_fraction": 0.4124999999999998,
      "soft_trace_changed_fraction": 0.9583333333333334,
      "soft_trace_avg_edits": 1.4166666666666667,
      "oracle_executor_accuracy": 0.9166666666666666,
      "oracle_state_exact": 0.22916666666666666,
      "oracle_program_exact": 0.16666666666666666,
      "oracle_state_prefix_fraction": 0.4979166666666666,
      "oracle_changed_fraction": 0.5833333333333334,
      "oracle_avg_edits": 0.8958333333333334,
      "domain_arithmetic_oracle_executor_accuracy": 0.875,
      "domain_arithmetic_oracle_state_exact": 0.25,
      "domain_arithmetic_oracle_state_prefix_fraction": 0.45,
      "domain_calendar_oracle_executor_accuracy": 1.0,
      "domain_calendar_oracle_state_exact": 0.25,
      "domain_calendar_oracle_state_prefix_fraction": 0.5499999999999999,
      "domain_unit_oracle_executor_accuracy": 0.75,
      "domain_unit_oracle_state_exact": 0.0,
      "domain_unit_oracle_state_prefix_fraction": 0.1875,
      "domain_list_oracle_executor_accuracy": 1.0,
      "domain_list_oracle_state_exact": 0.375,
      "domain_list_oracle_state_prefix_fraction": 0.7125,
      "domain_boolean_oracle_executor_accuracy": 1.0,
      "domain_boolean_oracle_state_exact": 0.25,
      "domain_boolean_oracle_state_prefix_fraction": 0.47500000000000003,
      "domain_lookup_oracle_executor_accuracy": 0.875,
      "domain_lookup_oracle_state_exact": 0.25,
      "domain_lookup_oracle_state_prefix_fraction": 0.6124999999999999,
      "learned_executor_accuracy": 0.2708333333333333,
      "learned_state_exact": 0.125,
      "learned_program_exact": 0.10416666666666667,
      "learned_state_prefix_fraction": 0.5208333333333334,
      "learned_changed_fraction": 0.4791666666666667,
      "learned_avg_edits": 0.5,
      "domain_arithmetic_learned_executor_accuracy": 0.125,
      "domain_arithmetic_learned_state_exact": 0.125,
      "domain_arithmetic_learned_state_prefix_fraction": 0.5,
      "domain_calendar_learned_executor_accuracy": 0.25,
      "domain_calendar_learned_state_exact": 0.125,
      "domain_calendar_learned_state_prefix_fraction": 0.5750000000000001,
      "domain_unit_learned_executor_accuracy": 0.0,
      "domain_unit_learned_state_exact": 0.0,
      "domain_unit_learned_state_prefix_fraction": 0.325,
      "domain_list_learned_executor_accuracy": 0.625,
      "domain_list_learned_state_exact": 0.25,
      "domain_list_learned_state_prefix_fraction": 0.7,
      "domain_boolean_learned_executor_accuracy": 0.625,
      "domain_boolean_learned_state_exact": 0.25,
      "domain_boolean_learned_state_prefix_fraction": 0.47500000000000003,
      "domain_lookup_learned_executor_accuracy": 0.0,
      "domain_lookup_learned_state_exact": 0.0,
      "domain_lookup_learned_state_prefix_fraction": 0.5499999999999999,
      "learned_oracle_gap_recovered": -0.10714285714285715
    }
  },
  "train_log": [
    {
      "epoch": 1,
      "train_loss": 4.8858584738330535,
      "hard_negative_loss": 0.3223797279931233,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.6041666666666666,
      "val_learned_state_exact": 0.4166666666666667,
      "val_learned_program_exact": 0.2916666666666667,
      "val_learned_state_prefix_fraction": 0.6180555555555557,
      "val_learned_changed_fraction": 0.3125,
      "val_learned_avg_edits": 0.4166666666666667,
      "val_domain_arithmetic_learned_executor_accuracy": 0.25,
      "val_domain_arithmetic_learned_state_exact": 0.25,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.5,
      "val_domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_learned_executor_accuracy": 0.375,
      "val_domain_unit_learned_state_exact": 0.375,
      "val_domain_unit_learned_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_learned_executor_accuracy": 0.75,
      "val_domain_list_learned_state_exact": 0.5,
      "val_domain_list_learned_state_prefix_fraction": 0.6041666666666667,
      "val_domain_boolean_learned_executor_accuracy": 0.875,
      "val_domain_boolean_learned_state_exact": 0.0,
      "val_domain_boolean_learned_state_prefix_fraction": 0.25,
      "val_domain_lookup_learned_executor_accuracy": 0.875,
      "val_domain_lookup_learned_state_exact": 0.875,
      "val_domain_lookup_learned_state_prefix_fraction": 0.9166666666666666,
      "val_learned_oracle_gap_recovered": 0.0
    },
    {
      "epoch": 2,
      "train_loss": 5.177517458603227,
      "hard_negative_loss": 0.2737237892791587,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.6041666666666666,
      "val_learned_state_exact": 0.20833333333333334,
      "val_learned_program_exact": 0.08333333333333333,
      "val_learned_state_prefix_fraction": 0.4444444444444445,
      "val_learned_changed_fraction": 0.6875,
      "val_learned_avg_edits": 1.3541666666666667,
      "val_domain_arithmetic_learned_executor_accuracy": 0.25,
      "val_domain_arithmetic_learned_state_exact": 0.25,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.25,
      "val_domain_calendar_learned_state_prefix_fraction": 0.4999999999999999,
      "val_domain_unit_learned_executor_accuracy": 0.375,
      "val_domain_unit_learned_state_exact": 0.25,
      "val_domain_unit_learned_state_prefix_fraction": 0.5624999999999999,
      "val_domain_list_learned_executor_accuracy": 0.75,
      "val_domain_list_learned_state_exact": 0.125,
      "val_domain_list_learned_state_prefix_fraction": 0.3333333333333333,
      "val_domain_boolean_learned_executor_accuracy": 0.875,
      "val_domain_boolean_learned_state_exact": 0.0,
      "val_domain_boolean_learned_state_prefix_fraction": 0.14583333333333331,
      "val_domain_lookup_learned_executor_accuracy": 0.875,
      "val_domain_lookup_learned_state_exact": 0.375,
      "val_domain_lookup_learned_state_prefix_fraction": 0.5,
      "val_learned_oracle_gap_recovered": 0.0
    },
    {
      "epoch": 3,
      "train_loss": 4.1534137349063185,
      "hard_negative_loss": 0.2848082439519331,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.4166666666666667,
      "val_learned_state_exact": 0.2916666666666667,
      "val_learned_program_exact": 0.25,
      "val_learned_state_prefix_fraction": 0.6215277777777779,
      "val_learned_changed_fraction": 0.4791666666666667,
      "val_learned_avg_edits": 0.5,
      "val_domain_arithmetic_learned_executor_accuracy": 0.125,
      "val_domain_arithmetic_learned_state_exact": 0.125,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.6041666666666667,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.5,
      "val_domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_learned_executor_accuracy": 0.375,
      "val_domain_unit_learned_state_exact": 0.25,
      "val_domain_unit_learned_state_prefix_fraction": 0.6458333333333334,
      "val_domain_list_learned_executor_accuracy": 0.75,
      "val_domain_list_learned_state_exact": 0.5,
      "val_domain_list_learned_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_learned_executor_accuracy": 0.625,
      "val_domain_boolean_learned_state_exact": 0.25,
      "val_domain_boolean_learned_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_learned_executor_accuracy": 0.125,
      "val_domain_lookup_learned_state_exact": 0.125,
      "val_domain_lookup_learned_state_prefix_fraction": 0.6041666666666667,
      "val_learned_oracle_gap_recovered": -0.49999999999999983
    },
    {
      "epoch": 4,
      "train_loss": 2.8554824198856044,
      "hard_negative_loss": 0.13328888612867218,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.4583333333333333,
      "val_learned_state_exact": 0.375,
      "val_learned_program_exact": 0.3125,
      "val_learned_state_prefix_fraction": 0.6215277777777778,
      "val_learned_changed_fraction": 0.5,
      "val_learned_avg_edits": 0.5,
      "val_domain_arithmetic_learned_executor_accuracy": 0.375,
      "val_domain_arithmetic_learned_state_exact": 0.375,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.6666666666666667,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.5,
      "val_domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_learned_executor_accuracy": 0.25,
      "val_domain_unit_learned_state_exact": 0.25,
      "val_domain_unit_learned_state_prefix_fraction": 0.5833333333333334,
      "val_domain_list_learned_executor_accuracy": 0.75,
      "val_domain_list_learned_state_exact": 0.625,
      "val_domain_list_learned_state_prefix_fraction": 0.7916666666666666,
      "val_domain_boolean_learned_executor_accuracy": 0.625,
      "val_domain_boolean_learned_state_exact": 0.25,
      "val_domain_boolean_learned_state_prefix_fraction": 0.375,
      "val_domain_lookup_learned_executor_accuracy": 0.25,
      "val_domain_lookup_learned_state_exact": 0.25,
      "val_domain_lookup_learned_state_prefix_fraction": 0.6041666666666667,
      "val_learned_oracle_gap_recovered": -0.38888888888888884
    },
    {
      "epoch": 5,
      "train_loss": 2.5932787488215623,
      "hard_negative_loss": 0.1599444488987095,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.4791666666666667,
      "val_learned_state_exact": 0.3541666666666667,
      "val_learned_program_exact": 0.2916666666666667,
      "val_learned_state_prefix_fraction": 0.6145833333333334,
      "val_learned_changed_fraction": 0.3125,
      "val_learned_avg_edits": 0.3125,
      "val_domain_arithmetic_learned_executor_accuracy": 0.125,
      "val_domain_arithmetic_learned_state_exact": 0.125,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.5625,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.5,
      "val_domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_learned_executor_accuracy": 0.125,
      "val_domain_unit_learned_state_exact": 0.125,
      "val_domain_unit_learned_state_prefix_fraction": 0.5,
      "val_domain_list_learned_executor_accuracy": 0.75,
      "val_domain_list_learned_state_exact": 0.625,
      "val_domain_list_learned_state_prefix_fraction": 0.7916666666666666,
      "val_domain_boolean_learned_executor_accuracy": 0.875,
      "val_domain_boolean_learned_state_exact": 0.25,
      "val_domain_boolean_learned_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_learned_executor_accuracy": 0.5,
      "val_domain_lookup_learned_state_exact": 0.5,
      "val_domain_lookup_learned_state_prefix_fraction": 0.7291666666666666,
      "val_learned_oracle_gap_recovered": -0.3333333333333332
    },
    {
      "epoch": 6,
      "train_loss": 2.4620473504872984,
      "hard_negative_loss": 0.09209633552824086,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.5,
      "val_learned_state_exact": 0.3541666666666667,
      "val_learned_program_exact": 0.3125,
      "val_learned_state_prefix_fraction": 0.6250000000000001,
      "val_learned_changed_fraction": 0.3125,
      "val_learned_avg_edits": 0.3125,
      "val_domain_arithmetic_learned_executor_accuracy": 0.375,
      "val_domain_arithmetic_learned_state_exact": 0.375,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.6458333333333334,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.5,
      "val_domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_learned_executor_accuracy": 0.125,
      "val_domain_unit_learned_state_exact": 0.125,
      "val_domain_unit_learned_state_prefix_fraction": 0.5625,
      "val_domain_list_learned_executor_accuracy": 0.75,
      "val_domain_list_learned_state_exact": 0.5,
      "val_domain_list_learned_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_learned_executor_accuracy": 0.875,
      "val_domain_boolean_learned_state_exact": 0.25,
      "val_domain_boolean_learned_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_learned_executor_accuracy": 0.375,
      "val_domain_lookup_learned_state_exact": 0.375,
      "val_domain_lookup_learned_state_prefix_fraction": 0.6666666666666666,
      "val_learned_oracle_gap_recovered": -0.2777777777777777
    },
    {
      "epoch": 7,
      "train_loss": 1.9673389264998333,
      "hard_negative_loss": 0.06664280030955028,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.6458333333333334,
      "val_learned_state_exact": 0.5,
      "val_learned_program_exact": 0.4166666666666667,
      "val_learned_state_prefix_fraction": 0.6875,
      "val_learned_changed_fraction": 0.14583333333333334,
      "val_learned_avg_edits": 0.14583333333333334,
      "val_domain_arithmetic_learned_executor_accuracy": 0.375,
      "val_domain_arithmetic_learned_state_exact": 0.375,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.6458333333333334,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.5,
      "val_domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_learned_executor_accuracy": 0.5,
      "val_domain_unit_learned_state_exact": 0.5,
      "val_domain_unit_learned_state_prefix_fraction": 0.6875,
      "val_domain_list_learned_executor_accuracy": 0.75,
      "val_domain_list_learned_state_exact": 0.5,
      "val_domain_list_learned_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_learned_executor_accuracy": 0.875,
      "val_domain_boolean_learned_state_exact": 0.25,
      "val_domain_boolean_learned_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_learned_executor_accuracy": 0.875,
      "val_domain_lookup_learned_state_exact": 0.875,
      "val_domain_lookup_learned_state_prefix_fraction": 0.9166666666666666,
      "val_learned_oracle_gap_recovered": 0.11111111111111131
    },
    {
      "epoch": 8,
      "train_loss": 2.010377821035745,
      "hard_negative_loss": 0.0799908340188854,
      "pair_train_loss": 0.0,
      "pair_train_batches": 0.0,
      "val_n": 48.0,
      "val_oracle_found_fraction": 0.9791666666666666,
      "val_avg_candidates": 111.0,
      "val_avg_positive_candidates": 29.791666666666668,
      "val_avg_state_exact_candidates": 6.5,
      "val_avg_program_exact_candidates": 0.9166666666666666,
      "val_base_executor_accuracy": 0.6041666666666666,
      "val_base_state_exact": 0.4583333333333333,
      "val_base_program_exact": 0.375,
      "val_base_state_prefix_fraction": 0.670138888888889,
      "val_base_changed_fraction": 0.0,
      "val_base_avg_edits": 0.0,
      "val_domain_arithmetic_n": 8.0,
      "val_domain_arithmetic_base_executor_accuracy": 0.25,
      "val_domain_arithmetic_base_state_exact": 0.25,
      "val_domain_arithmetic_base_state_prefix_fraction": 0.6250000000000001,
      "val_domain_calendar_n": 8.0,
      "val_domain_calendar_base_executor_accuracy": 0.5,
      "val_domain_calendar_base_state_exact": 0.5,
      "val_domain_calendar_base_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_n": 8.0,
      "val_domain_unit_base_executor_accuracy": 0.375,
      "val_domain_unit_base_state_exact": 0.375,
      "val_domain_unit_base_state_prefix_fraction": 0.6041666666666666,
      "val_domain_list_n": 8.0,
      "val_domain_list_base_executor_accuracy": 0.75,
      "val_domain_list_base_state_exact": 0.5,
      "val_domain_list_base_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_n": 8.0,
      "val_domain_boolean_base_executor_accuracy": 0.875,
      "val_domain_boolean_base_state_exact": 0.25,
      "val_domain_boolean_base_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_n": 8.0,
      "val_domain_lookup_base_executor_accuracy": 0.875,
      "val_domain_lookup_base_state_exact": 0.875,
      "val_domain_lookup_base_state_prefix_fraction": 0.9166666666666666,
      "val_prior_executor_accuracy": 0.6041666666666666,
      "val_prior_state_exact": 0.4583333333333333,
      "val_prior_program_exact": 0.375,
      "val_prior_state_prefix_fraction": 0.670138888888889,
      "val_prior_changed_fraction": 0.0,
      "val_prior_avg_edits": 0.0,
      "val_soft_trace_executor_accuracy": 0.3958333333333333,
      "val_soft_trace_state_exact": 0.22916666666666666,
      "val_soft_trace_program_exact": 0.1875,
      "val_soft_trace_state_prefix_fraction": 0.5416666666666667,
      "val_soft_trace_changed_fraction": 0.6666666666666666,
      "val_soft_trace_avg_edits": 0.8125,
      "val_oracle_executor_accuracy": 0.9791666666666666,
      "val_oracle_state_exact": 0.7708333333333334,
      "val_oracle_program_exact": 0.6666666666666666,
      "val_oracle_state_prefix_fraction": 0.8263888888888888,
      "val_oracle_changed_fraction": 0.375,
      "val_oracle_avg_edits": 0.4375,
      "val_domain_arithmetic_oracle_executor_accuracy": 0.875,
      "val_domain_arithmetic_oracle_state_exact": 0.875,
      "val_domain_arithmetic_oracle_state_prefix_fraction": 0.9583333333333334,
      "val_domain_calendar_oracle_executor_accuracy": 1.0,
      "val_domain_calendar_oracle_state_exact": 0.75,
      "val_domain_calendar_oracle_state_prefix_fraction": 0.7916666666666666,
      "val_domain_unit_oracle_executor_accuracy": 1.0,
      "val_domain_unit_oracle_state_exact": 0.875,
      "val_domain_unit_oracle_state_prefix_fraction": 0.875,
      "val_domain_list_oracle_executor_accuracy": 1.0,
      "val_domain_list_oracle_state_exact": 0.75,
      "val_domain_list_oracle_state_prefix_fraction": 0.8125,
      "val_domain_boolean_oracle_executor_accuracy": 1.0,
      "val_domain_boolean_oracle_state_exact": 0.375,
      "val_domain_boolean_oracle_state_prefix_fraction": 0.5208333333333333,
      "val_domain_lookup_oracle_executor_accuracy": 1.0,
      "val_domain_lookup_oracle_state_exact": 1.0,
      "val_domain_lookup_oracle_state_prefix_fraction": 1.0,
      "val_learned_executor_accuracy": 0.4375,
      "val_learned_state_exact": 0.3125,
      "val_learned_program_exact": 0.2708333333333333,
      "val_learned_state_prefix_fraction": 0.607638888888889,
      "val_learned_changed_fraction": 0.3958333333333333,
      "val_learned_avg_edits": 0.3958333333333333,
      "val_domain_arithmetic_learned_executor_accuracy": 0.125,
      "val_domain_arithmetic_learned_state_exact": 0.125,
      "val_domain_arithmetic_learned_state_prefix_fraction": 0.5833333333333334,
      "val_domain_calendar_learned_executor_accuracy": 0.5,
      "val_domain_calendar_learned_state_exact": 0.5,
      "val_domain_calendar_learned_state_prefix_fraction": 0.7083333333333333,
      "val_domain_unit_learned_executor_accuracy": 0.125,
      "val_domain_unit_learned_state_exact": 0.125,
      "val_domain_unit_learned_state_prefix_fraction": 0.5416666666666666,
      "val_domain_list_learned_executor_accuracy": 0.625,
      "val_domain_list_learned_state_exact": 0.5,
      "val_domain_list_learned_state_prefix_fraction": 0.7708333333333334,
      "val_domain_boolean_learned_executor_accuracy": 0.875,
      "val_domain_boolean_learned_state_exact": 0.25,
      "val_domain_boolean_learned_state_prefix_fraction": 0.39583333333333337,
      "val_domain_lookup_learned_executor_accuracy": 0.375,
      "val_domain_lookup_learned_state_exact": 0.375,
      "val_domain_lookup_learned_state_prefix_fraction": 0.6458333333333334,
      "val_learned_oracle_gap_recovered": -0.44444444444444436
    }
  ]
}