{
  "loo": {
    "losses": [
      {
        "heldout_task_id": 11,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.0966894626617432,
            "train_acc": 0.260869562625885
          },
          {
            "epoch": 40,
            "loss": 0.08385508507490158,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.0425223782658577,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.028608452528715134,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.02135499194264412,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 12,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.4101696014404297,
            "train_acc": 0.043478261679410934
          },
          {
            "epoch": 40,
            "loss": 0.10287032276391983,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04991019517183304,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.03293881565332413,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.02435942366719246,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 13,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.3522655963897705,
            "train_acc": 0.043478261679410934
          },
          {
            "epoch": 40,
            "loss": 0.08936627954244614,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.045321960002183914,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.030354835093021393,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.022536765784025192,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 14,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.4128607511520386,
            "train_acc": 0.17391304671764374
          },
          {
            "epoch": 40,
            "loss": 0.10305488854646683,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.052714984863996506,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.03525516018271446,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.026190761476755142,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 15,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.2078962326049805,
            "train_acc": 0.17391304671764374
          },
          {
            "epoch": 40,
            "loss": 0.08476323634386063,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.043342333287000656,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.028830111026763916,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.021320385858416557,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 16,
        "losses": [
          {
            "epoch": 0,
            "loss": 0.8220565915107727,
            "train_acc": 0.739130437374115
          },
          {
            "epoch": 40,
            "loss": 0.06564603745937347,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.032026637345552444,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.020960452035069466,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.015412886627018452,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 17,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.0256191492080688,
            "train_acc": 0.6086956262588501
          },
          {
            "epoch": 40,
            "loss": 0.07249116897583008,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.03576590493321419,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.023656275123357773,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.017484286800026894,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 18,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.0971673727035522,
            "train_acc": 0.3478260934352875
          },
          {
            "epoch": 40,
            "loss": 0.07205915451049805,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.0362706184387207,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.024370091035962105,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.018180597573518753,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 19,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.1259199380874634,
            "train_acc": 0.3913043439388275
          },
          {
            "epoch": 40,
            "loss": 0.08823394030332565,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04490116611123085,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.029919259250164032,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.022172357887029648,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 20,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.344861388206482,
            "train_acc": 0.08695652335882187
          },
          {
            "epoch": 40,
            "loss": 0.10091261565685272,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.05158836022019386,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.0343744121491909,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.02543887123465538,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 21,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.183222770690918,
            "train_acc": 0.30434781312942505
          },
          {
            "epoch": 40,
            "loss": 0.0828934982419014,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04223950207233429,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.02837255410850048,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.021160993725061417,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 22,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.1675180196762085,
            "train_acc": 0.260869562625885
          },
          {
            "epoch": 40,
            "loss": 0.09610958397388458,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04752519726753235,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.03119552880525589,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.022934358566999435,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 23,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.2140471935272217,
            "train_acc": 0.30434781312942505
          },
          {
            "epoch": 40,
            "loss": 0.08798132091760635,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.044005345553159714,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.028925534337759018,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.021235987544059753,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 24,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.3020875453948975,
            "train_acc": 0.260869562625885
          },
          {
            "epoch": 40,
            "loss": 0.09131034463644028,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04606148228049278,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.030681677162647247,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.022744150832295418,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 25,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.220621109008789,
            "train_acc": 0.3913043439388275
          },
          {
            "epoch": 40,
            "loss": 0.04569628834724426,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.02268259972333908,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.015645205974578857,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.011853941716253757,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 26,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.03900945186615,
            "train_acc": 0.3478260934352875
          },
          {
            "epoch": 40,
            "loss": 0.07720594108104706,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.038676049560308456,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.02559259720146656,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.018912184983491898,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 27,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.037787675857544,
            "train_acc": 0.43478259444236755
          },
          {
            "epoch": 40,
            "loss": 0.0779685378074646,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.03948640078306198,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.02638130821287632,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.019619369879364967,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 28,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.0749925374984741,
            "train_acc": 0.3478260934352875
          },
          {
            "epoch": 40,
            "loss": 0.08061319589614868,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04100166633725166,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.02703281305730343,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.019871219992637634,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 29,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.1299148797988892,
            "train_acc": 0.47826087474823
          },
          {
            "epoch": 40,
            "loss": 0.08069156110286713,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04042208194732666,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.02657029591500759,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.019498838111758232,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 30,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.3347798585891724,
            "train_acc": 0.3478260934352875
          },
          {
            "epoch": 40,
            "loss": 0.0980425626039505,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.05024933069944382,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.03332909196615219,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.024597328156232834,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 31,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.3305549621582031,
            "train_acc": 0.30434781312942505
          },
          {
            "epoch": 40,
            "loss": 0.0735006108880043,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.03829648345708847,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.02603207901120186,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.019467171281576157,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 32,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.3662742376327515,
            "train_acc": 0.08695652335882187
          },
          {
            "epoch": 40,
            "loss": 0.09100199490785599,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04294027015566826,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.028332224115729332,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.02096407115459442,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 33,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.0792255401611328,
            "train_acc": 0.260869562625885
          },
          {
            "epoch": 40,
            "loss": 0.08608807623386383,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.04296267405152321,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.02861238643527031,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.02125019021332264,
            "train_acc": 1.0
          }
        ]
      },
      {
        "heldout_task_id": 34,
        "losses": [
          {
            "epoch": 0,
            "loss": 1.196223497390747,
            "train_acc": 0.3913043439388275
          },
          {
            "epoch": 40,
            "loss": 0.07046949863433838,
            "train_acc": 1.0
          },
          {
            "epoch": 80,
            "loss": 0.03553066402673721,
            "train_acc": 1.0
          },
          {
            "epoch": 120,
            "loss": 0.024189922958612442,
            "train_acc": 1.0
          },
          {
            "epoch": 159,
            "loss": 0.01821841485798359,
            "train_acc": 1.0
          }
        ]
      }
    ],
    "oracle_label_counts": {
      "constrained4": 1,
      "hot_next4": 7,
      "stop": 16
    },
    "summary": {
      "action_counts": {
        "constrained4": 2,
        "hot_next4": 6,
        "stop": 16
      },
      "candidate_count_mean": 5.333333333333333,
      "coverage": 0.625,
      "covered_tasks": [
        11,
        12,
        13,
        14,
        17,
        18,
        19,
        22,
        23,
        27,
        28,
        29,
        30,
        32,
        33
      ],
      "distinct_functional_rate_mean": 0.4479166666666667,
      "forward_tokens": 30495,
      "parse_success_mean": 5.125,
      "pass1_proxy": 0.4166666666666667,
      "records": 24,
      "visible_coverage": 0.8333333333333334
    },
    "task_count": 24
  },
  "source": {
    "eval_records": 80,
    "learned_action_counts": {
      "high4": 3,
      "low4": 11,
      "mid4": 11,
      "stop": 55
    },
    "results": [
      {
        "arm_name": "base_prefix_k4",
        "records": {
          "candidate_count_mean": 3.45,
          "coverage": 0.7,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            20,
            21,
            23,
            24,
            25,
            27,
            28,
            29,
            30,
            32,
            33,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            63,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.75625,
          "distinct_functional_rate_mean": 0.4770833333333333,
          "forward_tokens": 62270,
          "hidden_pass_candidates_mean": 1.875,
          "parse_success_mean": 3.3625,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 2.2625,
          "visible_coverage": 0.8375
        }
      },
      {
        "arm_name": "default_extra_k8",
        "records": {
          "candidate_count_mean": 2.375,
          "coverage": 0.05,
          "covered_tasks": [
            22,
            36,
            42,
            67
          ],
          "distinct_behavior_rate_mean": 0.17447916666666669,
          "distinct_functional_rate_mean": 0.07604166666666666,
          "forward_tokens": 48065,
          "hidden_pass_candidates_mean": 0.125,
          "parse_success_mean": 2.225,
          "pass1_proxy": 0.0125,
          "pass1_tasks": [
            22
          ],
          "records": 80,
          "visible_candidates_mean": 0.7125,
          "visible_coverage": 0.125
        }
      },
      {
        "arm_name": "hot_extra_k8",
        "records": {
          "candidate_count_mean": 2.4,
          "coverage": 0.0875,
          "covered_tasks": [
            22,
            35,
            36,
            42,
            55,
            67,
            81
          ],
          "distinct_behavior_rate_mean": 0.240625,
          "distinct_functional_rate_mean": 0.0890625,
          "forward_tokens": 48801,
          "hidden_pass_candidates_mean": 0.1875,
          "parse_success_mean": 2.3125,
          "pass1_proxy": 0.0375,
          "pass1_tasks": [
            35,
            42,
            81
          ],
          "records": 80,
          "visible_candidates_mean": 0.8,
          "visible_coverage": 0.1875
        }
      },
      {
        "arm_name": "diverse_extra_k8",
        "records": {
          "candidate_count_mean": 2.3875,
          "coverage": 0.075,
          "covered_tasks": [
            22,
            35,
            36,
            42,
            67,
            81
          ],
          "distinct_behavior_rate_mean": 0.19464285714285715,
          "distinct_functional_rate_mean": 0.09129464285714287,
          "forward_tokens": 49872,
          "hidden_pass_candidates_mean": 0.1,
          "parse_success_mean": 2.275,
          "pass1_proxy": 0.0125,
          "pass1_tasks": [
            35
          ],
          "records": 80,
          "visible_candidates_mean": 0.65,
          "visible_coverage": 0.1625
        }
      },
      {
        "arm_name": "low_mid_high_static_k8",
        "records": {
          "candidate_count_mean": 3.7375,
          "coverage": 0.7,
          "covered_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            22,
            25,
            28,
            29,
            30,
            32,
            35,
            36,
            37,
            38,
            40,
            41,
            42,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.7348958333333333,
          "distinct_functional_rate_mean": 0.4916666666666667,
          "forward_tokens": 69828,
          "hidden_pass_candidates_mean": 1.2875,
          "parse_success_mean": 3.575,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 1.875,
          "visible_coverage": 0.775
        }
      },
      {
        "arm_name": "source_round_robin_k8",
        "records": {
          "candidate_count_mean": 3.75,
          "coverage": 0.6875,
          "covered_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            22,
            25,
            28,
            29,
            30,
            32,
            35,
            36,
            37,
            38,
            40,
            41,
            42,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            81,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.7243303571428571,
          "distinct_functional_rate_mean": 0.48325892857142855,
          "forward_tokens": 70190,
          "hidden_pass_candidates_mean": 1.2875,
          "parse_success_mean": 3.625,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 1.95,
          "visible_coverage": 0.8
        }
      },
      {
        "arm_name": "prefix2_low4",
        "records": {
          "candidate_count_mean": 3.1,
          "coverage": 0.6375,
          "covered_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            25,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            42,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.72,
          "distinct_functional_rate_mean": 0.49833333333333335,
          "forward_tokens": 57030,
          "hidden_pass_candidates_mean": 1.1875,
          "parse_success_mean": 2.9875,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 1.6625,
          "visible_coverage": 0.75
        }
      },
      {
        "arm_name": "prefix2_mid4",
        "records": {
          "candidate_count_mean": 3.15,
          "coverage": 0.7,
          "covered_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            22,
            25,
            28,
            29,
            30,
            32,
            35,
            36,
            37,
            38,
            40,
            41,
            42,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            67,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            81,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.7704166666666666,
          "distinct_functional_rate_mean": 0.5091666666666667,
          "forward_tokens": 57013,
          "hidden_pass_candidates_mean": 1.3,
          "parse_success_mean": 3.075,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 1.775,
          "visible_coverage": 0.7875
        }
      },
      {
        "arm_name": "prefix2_high4",
        "records": {
          "candidate_count_mean": 3.15,
          "coverage": 0.675,
          "covered_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            22,
            25,
            28,
            29,
            30,
            32,
            36,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.7683333333333333,
          "distinct_functional_rate_mean": 0.5154166666666666,
          "forward_tokens": 58838,
          "hidden_pass_candidates_mean": 1.2375,
          "parse_success_mean": 2.9875,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 1.7375,
          "visible_coverage": 0.7625
        }
      },
      {
        "arm_name": "learned_scheduler_after_prefix2",
        "records": {
          "candidate_count_mean": 2.8,
          "coverage": 0.65,
          "covered_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            25,
            28,
            29,
            30,
            32,
            35,
            37,
            38,
            40,
            41,
            42,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.76875,
          "distinct_functional_rate_mean": 0.5354166666666667,
          "forward_tokens": 51517,
          "hidden_pass_candidates_mean": 1.225,
          "parse_success_mean": 2.725,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 1.425,
          "visible_coverage": 0.7625
        }
      },
      {
        "arm_name": "oracle_best_block_after_prefix2",
        "records": {
          "candidate_count_mean": 2.3,
          "coverage": 0.7125,
          "covered_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            22,
            25,
            28,
            29,
            30,
            32,
            35,
            36,
            37,
            38,
            40,
            41,
            42,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            81,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.7933333333333333,
          "distinct_functional_rate_mean": 0.5841666666666667,
          "forward_tokens": 40154,
          "hidden_pass_candidates_mean": 1.325,
          "parse_success_mean": 2.275,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 1.6,
          "visible_coverage": 0.7875
        }
      },
      {
        "arm_name": "full_union_all_candidates",
        "records": {
          "candidate_count_mean": 24.0875,
          "coverage": 0.8625,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            15,
            17,
            18,
            19,
            20,
            21,
            22,
            23,
            24,
            25,
            27,
            28,
            29,
            30,
            32,
            33,
            34,
            35,
            36,
            37,
            38,
            40,
            41,
            42,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            55,
            56,
            57,
            58,
            59,
            61,
            62,
            63,
            64,
            65,
            66,
            67,
            68,
            69,
            70,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            81,
            82,
            83,
            85,
            86,
            87,
            88,
            89,
            90
          ],
          "distinct_behavior_rate_mean": 0.635992928544019,
          "distinct_functional_rate_mean": 0.34001591234862166,
          "forward_tokens": 504344,
          "hidden_pass_candidates_mean": 2.875,
          "parse_success_mean": 22.225,
          "pass1_proxy": 0.6125,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            20,
            21,
            28,
            29,
            30,
            32,
            37,
            38,
            40,
            41,
            45,
            46,
            47,
            49,
            50,
            51,
            52,
            53,
            54,
            56,
            57,
            58,
            61,
            62,
            64,
            65,
            66,
            68,
            69,
            71,
            72,
            74,
            75,
            76,
            78,
            79,
            80,
            82,
            85,
            86,
            88,
            89,
            90
          ],
          "records": 80,
          "visible_candidates_mean": 7.325,
          "visible_coverage": 0.9625
        }
      }
    ],
    "scheduler": {
      "bias": [
        0.5738916993141174,
        -1.0430265665054321,
        -0.20947179198265076,
        -0.34616273641586304
      ],
      "blocks": [
        "stop",
        "low4",
        "mid4",
        "high4"
      ],
      "feature_mean": [
        1.0,
        0.33895835280418396,
        0.40234375,
        0.981249988079071,
        0.643750011920929,
        0.6875,
        0.612500011920929,
        0.36699217557907104,
        0.4451136589050293
      ],
      "feature_std": [
        9.999999747378752e-05,
        0.09113071858882904,
        0.1361134648323059,
        0.12436547875404358,
        0.4583525061607361,
        0.46643680334091187,
        0.21010847389698029,
        0.24482886493206024,
        0.2779875099658966
      ],
      "label_counts": {
        "high4": 5,
        "low4": 12,
        "mid4": 11,
        "stop": 52
      },
      "losses": [
        {
          "epoch": 0,
          "loss": 1.4374157190322876,
          "train_acc": 0.3375000059604645
        },
        {
          "epoch": 20,
          "loss": 0.6598604321479797,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 40,
          "loss": 0.6314912438392639,
          "train_acc": 0.8500000238418579
        },
        {
          "epoch": 60,
          "loss": 0.6204109787940979,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 80,
          "loss": 0.6136312484741211,
          "train_acc": 0.8500000238418579
        },
        {
          "epoch": 100,
          "loss": 0.6080926060676575,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 120,
          "loss": 0.6032719612121582,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 140,
          "loss": 0.59891277551651,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 160,
          "loss": 0.5949051380157471,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 180,
          "loss": 0.5911927223205566,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 200,
          "loss": 0.5877455472946167,
          "train_acc": 0.862500011920929
        },
        {
          "epoch": 219,
          "loss": 0.5846996307373047,
          "train_acc": 0.862500011920929
        }
      ],
      "weight": [
        [
          -0.12894588708877563,
          0.49376291036605835,
          -0.5977544784545898,
          0.8298785090446472,
          0.14483758807182312,
          2.0718348026275635,
          -0.3015819191932678,
          -1.1536693572998047,
          0.14505767822265625
        ],
        [
          0.1081862673163414,
          -0.40889108180999756,
          0.26743072271347046,
          1.391844630241394,
          0.05916218087077141,
          -1.754076600074768,
          0.016629446297883987,
          -0.32897746562957764,
          1.9655461311340332
        ],
        [
          -0.04371708631515503,
          0.2847643792629242,
          0.11341012269258499,
          -1.1482709646224976,
          0.002074961783364415,
          -1.590429663658142,
          -0.05011879652738571,
          1.4279968738555908,
          -1.1697782278060913
        ],
        [
          0.07653467357158661,
          0.4897516369819641,
          0.5117801427841187,
          1.360503077507019,
          2.262279510498047,
          -3.36057186126709,
          0.24740000069141388,
          -0.033321887254714966,
          0.3704167902469635
        ]
      ]
    },
    "train_records": 80
  },
  "subset": {
    "results": [
      {
        "arm_name": "subset_base_hot_k4",
        "records": {
          "candidate_count_mean": 4.0,
          "coverage": 0.5833333333333334,
          "covered_tasks": [
            12,
            13,
            14,
            17,
            18,
            19,
            21,
            22,
            23,
            27,
            28,
            29,
            30,
            32
          ],
          "distinct_behavior_rate_mean": 0.875,
          "distinct_functional_rate_mean": 0.5104166666666666,
          "forward_tokens": 23434,
          "hidden_pass_candidates_mean": 1.5416666666666667,
          "parse_success_mean": 3.5416666666666665,
          "pass1_proxy": 0.375,
          "pass1_tasks": [
            13,
            17,
            18,
            19,
            22,
            27,
            28,
            29,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 2.2083333333333335,
          "visible_coverage": 0.7916666666666666
        }
      },
      {
        "arm_name": "subset_base_hot_k8",
        "records": {
          "candidate_count_mean": 7.708333333333333,
          "coverage": 0.6666666666666666,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            22,
            23,
            27,
            28,
            29,
            30,
            32,
            33,
            34
          ],
          "distinct_behavior_rate_mean": 0.7182539682539683,
          "distinct_functional_rate_mean": 0.3209325396825397,
          "forward_tokens": 44219,
          "hidden_pass_candidates_mean": 3.0,
          "parse_success_mean": 7.5,
          "pass1_proxy": 0.4166666666666667,
          "pass1_tasks": [
            11,
            12,
            14,
            17,
            18,
            19,
            27,
            28,
            29,
            30
          ],
          "records": 24,
          "visible_candidates_mean": 4.208333333333333,
          "visible_coverage": 0.8333333333333334
        }
      },
      {
        "arm_name": "subset_constrained_k4",
        "records": {
          "candidate_count_mean": 3.8333333333333335,
          "coverage": 0.625,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            22,
            23,
            25,
            27,
            28,
            29,
            30,
            32
          ],
          "distinct_behavior_rate_mean": 0.8541666666666666,
          "distinct_functional_rate_mean": 0.548611111111111,
          "forward_tokens": 21785,
          "hidden_pass_candidates_mean": 1.5416666666666667,
          "parse_success_mean": 3.5833333333333335,
          "pass1_proxy": 0.4166666666666667,
          "pass1_tasks": [
            12,
            14,
            17,
            18,
            19,
            22,
            23,
            27,
            28,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 1.9583333333333333,
          "visible_coverage": 0.7083333333333334
        }
      },
      {
        "arm_name": "subset_hot2_plus_constrained4",
        "records": {
          "candidate_count_mean": 5.833333333333333,
          "coverage": 0.625,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            22,
            23,
            25,
            27,
            28,
            29,
            30,
            32
          ],
          "distinct_behavior_rate_mean": 0.7888888888888889,
          "distinct_functional_rate_mean": 0.41944444444444445,
          "forward_tokens": 33549,
          "hidden_pass_candidates_mean": 2.1666666666666665,
          "parse_success_mean": 5.375,
          "pass1_proxy": 0.375,
          "pass1_tasks": [
            13,
            17,
            18,
            19,
            22,
            27,
            28,
            29,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 3.0,
          "visible_coverage": 0.7916666666666666
        }
      },
      {
        "arm_name": "subset_hot4_plus_constrained4",
        "records": {
          "candidate_count_mean": 7.833333333333333,
          "coverage": 0.6666666666666666,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            21,
            22,
            23,
            25,
            27,
            28,
            29,
            30,
            32
          ],
          "distinct_behavior_rate_mean": 0.7363095238095237,
          "distinct_functional_rate_mean": 0.3410714285714285,
          "forward_tokens": 45219,
          "hidden_pass_candidates_mean": 3.0833333333333335,
          "parse_success_mean": 7.125,
          "pass1_proxy": 0.375,
          "pass1_tasks": [
            13,
            17,
            18,
            19,
            22,
            27,
            28,
            29,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 4.166666666666667,
          "visible_coverage": 0.8333333333333334
        }
      },
      {
        "arm_name": "subset_constrained4_plus_hot4",
        "records": {
          "candidate_count_mean": 7.833333333333333,
          "coverage": 0.6666666666666666,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            21,
            22,
            23,
            25,
            27,
            28,
            29,
            30,
            32
          ],
          "distinct_behavior_rate_mean": 0.7363095238095237,
          "distinct_functional_rate_mean": 0.3410714285714285,
          "forward_tokens": 45219,
          "hidden_pass_candidates_mean": 3.0833333333333335,
          "parse_success_mean": 7.125,
          "pass1_proxy": 0.4166666666666667,
          "pass1_tasks": [
            12,
            14,
            17,
            18,
            19,
            22,
            23,
            27,
            28,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 4.166666666666667,
          "visible_coverage": 0.8333333333333334
        }
      },
      {
        "arm_name": "subset_visible_gate_hot2_then_constrained",
        "records": {
          "candidate_count_mean": 3.3333333333333335,
          "coverage": 0.5416666666666666,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            22,
            27,
            28,
            29,
            30,
            32
          ],
          "distinct_behavior_rate_mean": 0.9305555555555555,
          "distinct_functional_rate_mean": 0.6458333333333334,
          "forward_tokens": 20755,
          "hidden_pass_candidates_mean": 0.875,
          "parse_success_mean": 2.9583333333333335,
          "pass1_proxy": 0.375,
          "pass1_tasks": [
            13,
            17,
            18,
            19,
            22,
            27,
            28,
            29,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 1.375,
          "visible_coverage": 0.7916666666666666
        }
      },
      {
        "arm_name": "subset_visible_gate_hot4_then_constrained",
        "records": {
          "candidate_count_mean": 4.833333333333333,
          "coverage": 0.625,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            21,
            22,
            23,
            27,
            28,
            29,
            30,
            32
          ],
          "distinct_behavior_rate_mean": 0.8541666666666666,
          "distinct_functional_rate_mean": 0.4739583333333333,
          "forward_tokens": 29184,
          "hidden_pass_candidates_mean": 1.5833333333333333,
          "parse_success_mean": 4.291666666666667,
          "pass1_proxy": 0.375,
          "pass1_tasks": [
            13,
            17,
            18,
            19,
            22,
            27,
            28,
            29,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 2.2916666666666665,
          "visible_coverage": 0.8333333333333334
        }
      },
      {
        "arm_name": "subset_oracle_choose_arm",
        "records": {
          "candidate_count_mean": 4.166666666666667,
          "coverage": 0.75,
          "covered_tasks": [
            11,
            12,
            13,
            14,
            17,
            18,
            19,
            21,
            22,
            23,
            25,
            27,
            28,
            29,
            30,
            32,
            33,
            34
          ],
          "distinct_behavior_rate_mean": 0.875,
          "distinct_functional_rate_mean": 0.5381944444444444,
          "forward_tokens": 22223,
          "hidden_pass_candidates_mean": 1.5833333333333333,
          "parse_success_mean": 3.9583333333333335,
          "pass1_proxy": 0.375,
          "pass1_tasks": [
            14,
            17,
            18,
            19,
            22,
            27,
            28,
            29,
            32
          ],
          "records": 24,
          "visible_candidates_mean": 2.1666666666666665,
          "visible_coverage": 0.8333333333333334
        }
      }
    ],
    "task_count": 24
  }
}