{
  "by_family": {
    "synthetic_10": {
      "agreement": 4,
      "agreement_correct": 4,
      "agreement_precision": 1.0,
      "agreement_rate": 0.8,
      "any_round_hidden_exact": 4,
      "any_round_hidden_exact_rate": 0.8,
      "any_visible_verified_hidden_exact": 4,
      "any_visible_verified_hidden_exact_rate": 0.8,
      "both_direct_and_program": 4,
      "direct_accuracy": 1.0,
      "direct_exact": 5,
      "direct_only": 1,
      "direct_parse_ok": 5,
      "direct_parse_rate": 1.0,
      "final_hidden_exact": 4,
      "final_program_accuracy": 0.8,
      "final_visible_pass": 4,
      "final_visible_pass_rate": 0.8,
      "hybrid_accuracy": 1.0,
      "hybrid_exact": 5,
      "initial_accuracy": 0.8,
      "initial_hidden_exact": 4,
      "initial_visible_pass": 4,
      "initial_visible_pass_rate": 0.8,
      "mean_round_count": 1.4,
      "n": 5,
      "neither": 0,
      "oracle_union": 5,
      "oracle_union_rate": 1.0,
      "program_only": 0,
      "visible_false_pass": 0,
      "visible_false_pass_rate_among_visible": 0.0
    },
    "synthetic_11": {
      "agreement": 1,
      "agreement_correct": 1,
      "agreement_precision": 1.0,
      "agreement_rate": 1.0,
      "any_round_hidden_exact": 1,
      "any_round_hidden_exact_rate": 1.0,
      "any_visible_verified_hidden_exact": 1,
      "any_visible_verified_hidden_exact_rate": 1.0,
      "both_direct_and_program": 1,
      "direct_accuracy": 1.0,
      "direct_exact": 1,
      "direct_only": 0,
      "direct_parse_ok": 1,
      "direct_parse_rate": 1.0,
      "final_hidden_exact": 1,
      "final_program_accuracy": 1.0,
      "final_visible_pass": 1,
      "final_visible_pass_rate": 1.0,
      "hybrid_accuracy": 1.0,
      "hybrid_exact": 1,
      "initial_accuracy": 1.0,
      "initial_hidden_exact": 1,
      "initial_visible_pass": 1,
      "initial_visible_pass_rate": 1.0,
      "mean_round_count": 1.0,
      "n": 1,
      "neither": 0,
      "oracle_union": 1,
      "oracle_union_rate": 1.0,
      "program_only": 0,
      "visible_false_pass": 0,
      "visible_false_pass_rate_among_visible": 0.0
    }
  },
  "by_num_samples": {
    "1": {
      "agreement": 2,
      "agreement_correct": 2,
      "agreement_precision": 1.0,
      "agreement_rate": 1.0,
      "any_round_hidden_exact": 2,
      "any_round_hidden_exact_rate": 1.0,
      "any_visible_verified_hidden_exact": 2,
      "any_visible_verified_hidden_exact_rate": 1.0,
      "both_direct_and_program": 2,
      "direct_accuracy": 1.0,
      "direct_exact": 2,
      "direct_only": 0,
      "direct_parse_ok": 2,
      "direct_parse_rate": 1.0,
      "final_hidden_exact": 2,
      "final_program_accuracy": 1.0,
      "final_visible_pass": 2,
      "final_visible_pass_rate": 1.0,
      "hybrid_accuracy": 1.0,
      "hybrid_exact": 2,
      "initial_accuracy": 1.0,
      "initial_hidden_exact": 2,
      "initial_visible_pass": 2,
      "initial_visible_pass_rate": 1.0,
      "mean_round_count": 1.0,
      "n": 2,
      "neither": 0,
      "oracle_union": 2,
      "oracle_union_rate": 1.0,
      "program_only": 0,
      "visible_false_pass": 0,
      "visible_false_pass_rate_among_visible": 0.0
    },
    "2": {
      "agreement": 1,
      "agreement_correct": 1,
      "agreement_precision": 1.0,
      "agreement_rate": 1.0,
      "any_round_hidden_exact": 1,
      "any_round_hidden_exact_rate": 1.0,
      "any_visible_verified_hidden_exact": 1,
      "any_visible_verified_hidden_exact_rate": 1.0,
      "both_direct_and_program": 1,
      "direct_accuracy": 1.0,
      "direct_exact": 1,
      "direct_only": 0,
      "direct_parse_ok": 1,
      "direct_parse_rate": 1.0,
      "final_hidden_exact": 1,
      "final_program_accuracy": 1.0,
      "final_visible_pass": 1,
      "final_visible_pass_rate": 1.0,
      "hybrid_accuracy": 1.0,
      "hybrid_exact": 1,
      "initial_accuracy": 1.0,
      "initial_hidden_exact": 1,
      "initial_visible_pass": 1,
      "initial_visible_pass_rate": 1.0,
      "mean_round_count": 1.0,
      "n": 1,
      "neither": 0,
      "oracle_union": 1,
      "oracle_union_rate": 1.0,
      "program_only": 0,
      "visible_false_pass": 0,
      "visible_false_pass_rate_among_visible": 0.0
    },
    "3": {
      "agreement": 1,
      "agreement_correct": 1,
      "agreement_precision": 1.0,
      "agreement_rate": 1.0,
      "any_round_hidden_exact": 1,
      "any_round_hidden_exact_rate": 1.0,
      "any_visible_verified_hidden_exact": 1,
      "any_visible_verified_hidden_exact_rate": 1.0,
      "both_direct_and_program": 1,
      "direct_accuracy": 1.0,
      "direct_exact": 1,
      "direct_only": 0,
      "direct_parse_ok": 1,
      "direct_parse_rate": 1.0,
      "final_hidden_exact": 1,
      "final_program_accuracy": 1.0,
      "final_visible_pass": 1,
      "final_visible_pass_rate": 1.0,
      "hybrid_accuracy": 1.0,
      "hybrid_exact": 1,
      "initial_accuracy": 1.0,
      "initial_hidden_exact": 1,
      "initial_visible_pass": 1,
      "initial_visible_pass_rate": 1.0,
      "mean_round_count": 1.0,
      "n": 1,
      "neither": 0,
      "oracle_union": 1,
      "oracle_union_rate": 1.0,
      "program_only": 0,
      "visible_false_pass": 0,
      "visible_false_pass_rate_among_visible": 0.0
    },
    "4": {
      "agreement": 1,
      "agreement_correct": 1,
      "agreement_precision": 1.0,
      "agreement_rate": 1.0,
      "any_round_hidden_exact": 1,
      "any_round_hidden_exact_rate": 1.0,
      "any_visible_verified_hidden_exact": 1,
      "any_visible_verified_hidden_exact_rate": 1.0,
      "both_direct_and_program": 1,
      "direct_accuracy": 1.0,
      "direct_exact": 1,
      "direct_only": 0,
      "direct_parse_ok": 1,
      "direct_parse_rate": 1.0,
      "final_hidden_exact": 1,
      "final_program_accuracy": 1.0,
      "final_visible_pass": 1,
      "final_visible_pass_rate": 1.0,
      "hybrid_accuracy": 1.0,
      "hybrid_exact": 1,
      "initial_accuracy": 1.0,
      "initial_hidden_exact": 1,
      "initial_visible_pass": 1,
      "initial_visible_pass_rate": 1.0,
      "mean_round_count": 1.0,
      "n": 1,
      "neither": 0,
      "oracle_union": 1,
      "oracle_union_rate": 1.0,
      "program_only": 0,
      "visible_false_pass": 0,
      "visible_false_pass_rate_among_visible": 0.0
    },
    "5": {
      "agreement": 0,
      "agreement_correct": 0,
      "agreement_precision": 0.0,
      "agreement_rate": 0.0,
      "any_round_hidden_exact": 0,
      "any_round_hidden_exact_rate": 0.0,
      "any_visible_verified_hidden_exact": 0,
      "any_visible_verified_hidden_exact_rate": 0.0,
      "both_direct_and_program": 0,
      "direct_accuracy": 1.0,
      "direct_exact": 1,
      "direct_only": 1,
      "direct_parse_ok": 1,
      "direct_parse_rate": 1.0,
      "final_hidden_exact": 0,
      "final_program_accuracy": 0.0,
      "final_visible_pass": 0,
      "final_visible_pass_rate": 0.0,
      "hybrid_accuracy": 1.0,
      "hybrid_exact": 1,
      "initial_accuracy": 0.0,
      "initial_hidden_exact": 0,
      "initial_visible_pass": 0,
      "initial_visible_pass_rate": 0.0,
      "mean_round_count": 3.0,
      "n": 1,
      "neither": 0,
      "oracle_union": 1,
      "oracle_union_rate": 1.0,
      "program_only": 0,
      "visible_false_pass": 0,
      "visible_false_pass_rate_among_visible": 0.0
    }
  },
  "overall": {
    "agreement": 5,
    "agreement_correct": 5,
    "agreement_precision": 1.0,
    "agreement_rate": 0.8333333333333334,
    "any_round_hidden_exact": 5,
    "any_round_hidden_exact_rate": 0.8333333333333334,
    "any_visible_verified_hidden_exact": 5,
    "any_visible_verified_hidden_exact_rate": 0.8333333333333334,
    "both_direct_and_program": 5,
    "direct_accuracy": 1.0,
    "direct_exact": 6,
    "direct_only": 1,
    "direct_parse_ok": 6,
    "direct_parse_rate": 1.0,
    "final_hidden_exact": 5,
    "final_program_accuracy": 0.8333333333333334,
    "final_visible_pass": 5,
    "final_visible_pass_rate": 0.8333333333333334,
    "hybrid_accuracy": 1.0,
    "hybrid_exact": 6,
    "initial_accuracy": 0.8333333333333334,
    "initial_hidden_exact": 5,
    "initial_visible_pass": 5,
    "initial_visible_pass_rate": 0.8333333333333334,
    "mean_round_count": 1.3333333333333333,
    "n": 6,
    "neither": 0,
    "oracle_union": 6,
    "oracle_union_rate": 1.0,
    "program_only": 0,
    "visible_false_pass": 0,
    "visible_false_pass_rate_among_visible": 0.0
  },
  "round_stats": {
    "0": {
      "attempted": 6,
      "hidden_exact": 5,
      "visible_pass": 5,
      "visible_verified_hidden_exact": 5
    },
    "1": {
      "attempted": 1,
      "hidden_exact": 0,
      "visible_pass": 0,
      "visible_verified_hidden_exact": 0
    },
    "2": {
      "attempted": 1,
      "hidden_exact": 0,
      "visible_pass": 0,
      "visible_verified_hidden_exact": 0
    }
  }
}