{
  "abi": {
    "candidate_count_mean": 5.6,
    "first_visible_accuracy": 0.172,
    "first_visible_heldout": 43,
    "heldout_coverage": 0.18,
    "heldout_covered": 45
  },
  "by_family": [
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "agriculture",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "craigslist_data_wrangler",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 0,
      "direct_parse_rate": 0.0,
      "family": "crime_data_wrangler",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "potters_wheel_divide",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "potters_wheel_fold",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "potters_wheel_fold_2",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "potters_wheel_merge_split",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "potters_wheel_split_fold",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.4,
      "direct_exact": 2,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "potters_wheel_unfold",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "potters_wheel_unfold2",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "proactive_wrangling_complex",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "proactive_wrangling_fold",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "reshape_table_structure_data_wrangler",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_10",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_11",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_12",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_13",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_15",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.6,
      "direct_exact": 3,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_17",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_18",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_19",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_2",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_22",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_24",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_25",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_26",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_27",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_28",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_29",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_3",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_30",
      "n": 5
    },
    {
      "abi_first_visible": 3,
      "abi_first_visible_rate": 0.6,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 4,
      "direct_parse_rate": 0.8,
      "family": "synthetic_33",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_34",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_36",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_37",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_4",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_40",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 2,
      "direct_parse_rate": 0.4,
      "family": "synthetic_41",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_43",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.6,
      "direct_exact": 3,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_44",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_45",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_46",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_47",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.2,
      "direct_exact": 1,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_48",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.6,
      "direct_exact": 3,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_49",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.4,
      "direct_exact": 2,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_5",
      "n": 5
    },
    {
      "abi_first_visible": 5,
      "abi_first_visible_rate": 1.0,
      "abi_oracle": 5,
      "abi_oracle_rate": 1.0,
      "direct_accuracy": 0.4,
      "direct_exact": 2,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_51",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_6",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.8,
      "direct_exact": 4,
      "direct_parse": 5,
      "direct_parse_rate": 1.0,
      "family": "synthetic_7",
      "n": 5
    },
    {
      "abi_first_visible": 0,
      "abi_first_visible_rate": 0.0,
      "abi_oracle": 0,
      "abi_oracle_rate": 0.0,
      "direct_accuracy": 0.0,
      "direct_exact": 0,
      "direct_parse": 0,
      "direct_parse_rate": 0.0,
      "family": "synthetic_8",
      "n": 5
    }
  ],
  "by_num_samples": {
    "1": {
      "abi_oracle": 9,
      "abi_oracle_rate": 0.18,
      "direct_accuracy": 0.38,
      "direct_exact": 19,
      "direct_parse": 47,
      "direct_parse_rate": 0.94,
      "n": 50
    },
    "2": {
      "abi_oracle": 9,
      "abi_oracle_rate": 0.18,
      "direct_accuracy": 0.58,
      "direct_exact": 29,
      "direct_parse": 47,
      "direct_parse_rate": 0.94,
      "n": 50
    },
    "3": {
      "abi_oracle": 9,
      "abi_oracle_rate": 0.18,
      "direct_accuracy": 0.6,
      "direct_exact": 30,
      "direct_parse": 47,
      "direct_parse_rate": 0.94,
      "n": 50
    },
    "4": {
      "abi_oracle": 9,
      "abi_oracle_rate": 0.18,
      "direct_accuracy": 0.62,
      "direct_exact": 31,
      "direct_parse": 47,
      "direct_parse_rate": 0.94,
      "n": 50
    },
    "5": {
      "abi_oracle": 9,
      "abi_oracle_rate": 0.18,
      "direct_accuracy": 0.58,
      "direct_exact": 29,
      "direct_parse": 48,
      "direct_parse_rate": 0.96,
      "n": 50
    }
  },
  "direct": {
    "exact": 138,
    "exact_accuracy": 0.552,
    "max_new_tokens": 768,
    "parse_ok": 236,
    "parse_rate": 0.944
  },
  "n": 250,
  "overlap": {
    "abi_only": 10,
    "both_direct_and_abi": 35,
    "direct_accuracy_on_abi_covered": {
      "count": 35,
      "n": 45,
      "rate": 0.7777777777777778
    },
    "direct_accuracy_on_abi_uncovered": {
      "count": 103,
      "n": 205,
      "rate": 0.5024390243902439
    },
    "direct_only": 103,
    "direct_or_abi_first_visible": 147,
    "direct_or_abi_first_visible_rate": 0.588,
    "direct_or_abi_oracle": 148,
    "direct_or_abi_oracle_rate": 0.592,
    "neither": 102
  },
  "target_char_buckets": {
    "201-500": {
      "count": 53,
      "n": 84,
      "rate": 0.6309523809523809
    },
    "501-1000": {
      "count": 14,
      "n": 18,
      "rate": 0.7777777777777778
    },
    "<=200": {
      "count": 68,
      "n": 131,
      "rate": 0.5190839694656488
    },
    ">1000": {
      "count": 3,
      "n": 17,
      "rate": 0.17647058823529413
    }
  }
}