{ "arms": { "base": { "all": { "abstention_rate": 0.0, "answer_tokens": 18125, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": 0.0, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.003472222222222222, "injected_tokens": 576, "logical_model_input_tokens": 534768, "macro_use_rate_correct_candidates": 0.0, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 4.994318181818182, "mean_sampled_tokens_per_completion": 255.54861111111111, "mean_sampled_tokens_per_task": 6133.166666666667, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 4.994318181818182, "n_samples": 288, "n_tasks": 12, "oracle_hidden_coverage": 0.08333333333333333, "parse_rate": 0.6111111111111112, "sampled_plus_unique_prompt_tokens": 83575, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.08333333333333333, "selected_visible_pass_rate": 0.08333333333333333, "thinking_tokens": 55296, "total_sampled_tokens": 73598, "total_unique_prompt_tokens": 9977, "truncation_rate": 0.3854166666666667, "valid_program_rate": 0.6111111111111112, "visible_pass_rate": 0.003472222222222222 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 9422, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": 0.0, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.006944444444444444, "injected_tokens": 288, "logical_model_input_tokens": 267696, "macro_use_rate_correct_candidates": 0.0, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 258.0208333333333, "mean_sampled_tokens_per_task": 6192.5, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 144, "n_tasks": 6, "oracle_hidden_coverage": 0.16666666666666666, "parse_rate": 0.5902777777777778, "sampled_plus_unique_prompt_tokens": 42150, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.16666666666666666, "selected_visible_pass_rate": 0.16666666666666666, "thinking_tokens": 27648, "total_sampled_tokens": 37155, "total_unique_prompt_tokens": 4995, "truncation_rate": 0.4097222222222222, "valid_program_rate": 0.5902777777777778, "visible_pass_rate": 0.006944444444444444 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 8703, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 267072, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 4.989010989010989, "mean_sampled_tokens_per_completion": 253.07638888888889, "mean_sampled_tokens_per_task": 6073.833333333333, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 4.989010989010989, "n_samples": 144, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.6319444444444444, "sampled_plus_unique_prompt_tokens": 41425, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 36443, "total_unique_prompt_tokens": 4982, "truncation_rate": 0.3611111111111111, "valid_program_rate": 0.6319444444444444, "visible_pass_rate": 0.0 } }, "designed_ceiling": { "all": { "abstention_rate": 0.0, "answer_tokens": 9653, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 283800, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 4.916666666666667, "mean_expanded_depth_valid": 4.975, "mean_sampled_tokens_per_completion": 259.6041666666667, "mean_sampled_tokens_per_task": 3115.25, "mean_surface_depth_selected": 4.916666666666667, "mean_surface_depth_valid": 4.975, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5694444444444444, "sampled_plus_unique_prompt_tokens": 48044, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 37383, "total_unique_prompt_tokens": 10661, "truncation_rate": 0.4305555555555556, "valid_program_rate": 0.5555555555555556, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 4873, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142056, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 4.833333333333333, "mean_expanded_depth_valid": 4.975609756097561, "mean_sampled_tokens_per_completion": 260.25, "mean_sampled_tokens_per_task": 3123.0, "mean_surface_depth_selected": 4.833333333333333, "mean_surface_depth_valid": 4.975609756097561, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5694444444444444, "sampled_plus_unique_prompt_tokens": 24075, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18738, "total_unique_prompt_tokens": 5337, "truncation_rate": 0.4305555555555556, "valid_program_rate": 0.5694444444444444, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 4780, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 141744, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 4.9743589743589745, "mean_sampled_tokens_per_completion": 258.9583333333333, "mean_sampled_tokens_per_task": 3107.5, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 4.9743589743589745, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5694444444444444, "sampled_plus_unique_prompt_tokens": 23969, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18645, "total_unique_prompt_tokens": 5324, "truncation_rate": 0.4305555555555556, "valid_program_rate": 0.5416666666666666, "visible_pass_rate": 0.0 } }, "mined": { "all": { "abstention_rate": 0.0, "answer_tokens": 9639, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 284088, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 4.916666666666667, "mean_expanded_depth_valid": 4.987654320987654, "mean_sampled_tokens_per_completion": 259.50694444444446, "mean_sampled_tokens_per_task": 3114.0833333333335, "mean_surface_depth_selected": 4.916666666666667, "mean_surface_depth_valid": 4.987654320987654, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5694444444444444, "sampled_plus_unique_prompt_tokens": 48042, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 37369, "total_unique_prompt_tokens": 10673, "truncation_rate": 0.4305555555555556, "valid_program_rate": 0.5625, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 5097, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142200, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 4.833333333333333, "mean_expanded_depth_valid": 4.973684210526316, "mean_sampled_tokens_per_completion": 263.31944444444446, "mean_sampled_tokens_per_task": 3159.8333333333335, "mean_surface_depth_selected": 4.833333333333333, "mean_surface_depth_valid": 4.973684210526316, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5277777777777778, "sampled_plus_unique_prompt_tokens": 24302, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18959, "total_unique_prompt_tokens": 5343, "truncation_rate": 0.4722222222222222, "valid_program_rate": 0.5277777777777778, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 4542, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 141888, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 255.69444444444446, "mean_sampled_tokens_per_task": 3068.3333333333335, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.6111111111111112, "sampled_plus_unique_prompt_tokens": 23740, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18410, "total_unique_prompt_tokens": 5330, "truncation_rate": 0.3888888888888889, "valid_program_rate": 0.5972222222222222, "visible_pass_rate": 0.0 } }, "mined_hint": { "all": { "abstention_rate": 0.0, "answer_tokens": 10314, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 285528, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 264.15972222222223, "mean_sampled_tokens_per_task": 3169.9166666666665, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5277777777777778, "sampled_plus_unique_prompt_tokens": 48772, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 38039, "total_unique_prompt_tokens": 10733, "truncation_rate": 0.4652777777777778, "valid_program_rate": 0.5277777777777778, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 5000, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142920, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 262.0, "mean_sampled_tokens_per_task": 3144.0, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5555555555555556, "sampled_plus_unique_prompt_tokens": 24237, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18864, "total_unique_prompt_tokens": 5373, "truncation_rate": 0.4444444444444444, "valid_program_rate": 0.5555555555555556, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 5314, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142608, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 266.31944444444446, "mean_sampled_tokens_per_task": 3195.8333333333335, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5, "sampled_plus_unique_prompt_tokens": 24535, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 19175, "total_unique_prompt_tokens": 5360, "truncation_rate": 0.4861111111111111, "valid_program_rate": 0.5, "visible_pass_rate": 0.0 } }, "random_0": { "all": { "abstention_rate": 0.0, "answer_tokens": 10097, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 284376, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 262.65972222222223, "mean_sampled_tokens_per_task": 3151.9166666666665, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5347222222222222, "sampled_plus_unique_prompt_tokens": 48508, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 37823, "total_unique_prompt_tokens": 10685, "truncation_rate": 0.4583333333333333, "valid_program_rate": 0.5347222222222222, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 4859, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142344, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 260.05555555555554, "mean_sampled_tokens_per_task": 3120.6666666666665, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5555555555555556, "sampled_plus_unique_prompt_tokens": 24073, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18724, "total_unique_prompt_tokens": 5349, "truncation_rate": 0.4305555555555556, "valid_program_rate": 0.5555555555555556, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 5238, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142032, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 265.2638888888889, "mean_sampled_tokens_per_task": 3183.1666666666665, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5138888888888888, "sampled_plus_unique_prompt_tokens": 24435, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 19099, "total_unique_prompt_tokens": 5336, "truncation_rate": 0.4861111111111111, "valid_program_rate": 0.5138888888888888, "visible_pass_rate": 0.0 } }, "random_1": { "all": { "abstention_rate": 0.0, "answer_tokens": 9196, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 284088, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 256.46527777777777, "mean_sampled_tokens_per_task": 3077.5833333333335, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.6041666666666666, "sampled_plus_unique_prompt_tokens": 47604, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 36931, "total_unique_prompt_tokens": 10673, "truncation_rate": 0.3958333333333333, "valid_program_rate": 0.6041666666666666, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 4983, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142200, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 261.7638888888889, "mean_sampled_tokens_per_task": 3141.1666666666665, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5555555555555556, "sampled_plus_unique_prompt_tokens": 24190, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18847, "total_unique_prompt_tokens": 5343, "truncation_rate": 0.4444444444444444, "valid_program_rate": 0.5555555555555556, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 4213, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 141888, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 251.16666666666666, "mean_sampled_tokens_per_task": 3014.0, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.6527777777777778, "sampled_plus_unique_prompt_tokens": 23414, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18084, "total_unique_prompt_tokens": 5330, "truncation_rate": 0.3472222222222222, "valid_program_rate": 0.6527777777777778, "visible_pass_rate": 0.0 } }, "random_2": { "all": { "abstention_rate": 0.0, "answer_tokens": 9349, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 284088, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 4.988095238095238, "mean_sampled_tokens_per_completion": 257.5138888888889, "mean_sampled_tokens_per_task": 3090.1666666666665, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 4.988095238095238, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5902777777777778, "sampled_plus_unique_prompt_tokens": 47755, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 37082, "total_unique_prompt_tokens": 10673, "truncation_rate": 0.4097222222222222, "valid_program_rate": 0.5833333333333334, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 4697, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142200, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 4.975609756097561, "mean_sampled_tokens_per_completion": 257.81944444444446, "mean_sampled_tokens_per_task": 3093.8333333333335, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 4.975609756097561, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5833333333333334, "sampled_plus_unique_prompt_tokens": 23906, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18563, "total_unique_prompt_tokens": 5343, "truncation_rate": 0.4166666666666667, "valid_program_rate": 0.5694444444444444, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 4652, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 141888, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 257.2083333333333, "mean_sampled_tokens_per_task": 3086.5, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5972222222222222, "sampled_plus_unique_prompt_tokens": 23849, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18519, "total_unique_prompt_tokens": 5330, "truncation_rate": 0.4027777777777778, "valid_program_rate": 0.5972222222222222, "visible_pass_rate": 0.0 } }, "random_3": { "all": { "abstention_rate": 0.0, "answer_tokens": 9946, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 284088, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 4.9875, "mean_sampled_tokens_per_completion": 261.63194444444446, "mean_sampled_tokens_per_task": 3139.5833333333335, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 4.9875, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5555555555555556, "sampled_plus_unique_prompt_tokens": 48348, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 37675, "total_unique_prompt_tokens": 10673, "truncation_rate": 0.4375, "valid_program_rate": 0.5555555555555556, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 4872, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142200, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 4.975609756097561, "mean_sampled_tokens_per_completion": 260.25, "mean_sampled_tokens_per_task": 3123.0, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 4.975609756097561, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5694444444444444, "sampled_plus_unique_prompt_tokens": 24081, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18738, "total_unique_prompt_tokens": 5343, "truncation_rate": 0.4166666666666667, "valid_program_rate": 0.5694444444444444, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 5074, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 141888, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 263.0138888888889, "mean_sampled_tokens_per_task": 3156.1666666666665, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5416666666666666, "sampled_plus_unique_prompt_tokens": 24267, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18937, "total_unique_prompt_tokens": 5330, "truncation_rate": 0.4583333333333333, "valid_program_rate": 0.5416666666666666, "visible_pass_rate": 0.0 } }, "random_4": { "all": { "abstention_rate": 0.0, "answer_tokens": 9591, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 288, "logical_model_input_tokens": 284376, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 259.1875, "mean_sampled_tokens_per_task": 3110.25, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 144, "n_tasks": 12, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5833333333333334, "sampled_plus_unique_prompt_tokens": 48008, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 27648, "total_sampled_tokens": 37323, "total_unique_prompt_tokens": 10685, "truncation_rate": 0.4166666666666667, "valid_program_rate": 0.5833333333333334, "visible_pass_rate": 0.0 }, "smoke_no_reuse": { "abstention_rate": 0.0, "answer_tokens": 4310, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142344, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 252.5, "mean_sampled_tokens_per_task": 3030.0, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.6388888888888888, "sampled_plus_unique_prompt_tokens": 23529, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 18180, "total_unique_prompt_tokens": 5349, "truncation_rate": 0.3611111111111111, "valid_program_rate": 0.6388888888888888, "visible_pass_rate": 0.0 }, "smoke_reuse": { "abstention_rate": 0.0, "answer_tokens": 5281, "false_visible_pass_count": 0, "false_visible_pass_rate_all_samples": 0.0, "false_visible_pass_rate_given_visible_pass": null, "forced_close_rate": 1.0, "hidden_pass_sample_rate": 0.0, "injected_tokens": 144, "logical_model_input_tokens": 142032, "macro_use_rate_correct_candidates": null, "macro_use_rate_selected": 0.0, "macro_use_rate_valid_candidates": 0.0, "mean_expanded_depth_selected": 5.0, "mean_expanded_depth_valid": 5.0, "mean_sampled_tokens_per_completion": 265.875, "mean_sampled_tokens_per_task": 3190.5, "mean_surface_depth_selected": 5.0, "mean_surface_depth_valid": 5.0, "n_samples": 72, "n_tasks": 6, "oracle_hidden_coverage": 0.0, "parse_rate": 0.5277777777777778, "sampled_plus_unique_prompt_tokens": 24479, "selected_false_visible_pass_rate": 0.0, "selected_hidden_accuracy": 0.0, "selected_visible_pass_rate": 0.0, "thinking_tokens": 13824, "total_sampled_tokens": 19143, "total_unique_prompt_tokens": 5336, "truncation_rate": 0.4722222222222222, "valid_program_rate": 0.5277777777777778, "visible_pass_rate": 0.0 } } }, "bootstrap_repetitions": 10000, "bootstrap_seed": 8102, "run": "smoke", "runner_metadata": { "base": { "counts": { "completions": 288, "injected_tokens": 576, "logical_model_input_tokens": 534768, "requests": 12, "sampled_tokens": 73598, "stage1_logical_prompt_tokens": 239448, "stage2_logical_prompt_tokens": 295320, "unique_input_prompt_tokens": 9977 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 24, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 67.48177834041417, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 1090.6351582605357 } }, "designed_ceiling": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 283800, "requests": 12, "sampled_tokens": 37383, "stage1_logical_prompt_tokens": 127932, "stage2_logical_prompt_tokens": 155868, "unique_input_prompt_tokens": 10661 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 38.00302366912365, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 983.6848858521907 } }, "mined": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 284088, "requests": 12, "sampled_tokens": 37369, "stage1_logical_prompt_tokens": 128076, "stage2_logical_prompt_tokens": 156012, "unique_input_prompt_tokens": 10673 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 37.87069963198155, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 986.7523009382729 } }, "mined_hint": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 285528, "requests": 12, "sampled_tokens": 38039, "stage1_logical_prompt_tokens": 128796, "stage2_logical_prompt_tokens": 156732, "unique_input_prompt_tokens": 10733 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 38.379088904708624, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 991.1386925949954 } }, "random_0": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 284376, "requests": 12, "sampled_tokens": 37823, "stage1_logical_prompt_tokens": 128220, "stage2_logical_prompt_tokens": 156156, "unique_input_prompt_tokens": 10685 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 38.35911696776748, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 986.0237406346458 } }, "random_1": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 284088, "requests": 12, "sampled_tokens": 36931, "stage1_logical_prompt_tokens": 128076, "stage2_logical_prompt_tokens": 156012, "unique_input_prompt_tokens": 10673 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 38.153216446749866, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 967.9655724844141 } }, "random_2": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 284088, "requests": 12, "sampled_tokens": 37082, "stage1_logical_prompt_tokens": 128076, "stage2_logical_prompt_tokens": 156012, "unique_input_prompt_tokens": 10673 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 38.048863338306546, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 974.5889034920752 } }, "random_3": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 284088, "requests": 12, "sampled_tokens": 37675, "stage1_logical_prompt_tokens": 128076, "stage2_logical_prompt_tokens": 156012, "unique_input_prompt_tokens": 10673 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 37.90050444472581, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 994.0500938436139 } }, "random_4": { "counts": { "completions": 144, "injected_tokens": 288, "logical_model_input_tokens": 284376, "requests": 12, "sampled_tokens": 37323, "stage1_logical_prompt_tokens": 128220, "stage2_logical_prompt_tokens": 156156, "unique_input_prompt_tokens": 10685 }, "engine": { "adapter": null, "enable_prefix_caching": false, "enforce_eager": false, "gpu_memory_utilization": 0.9, "max_model_len": 16384, "max_num_batched_tokens": 32768, "max_num_seqs": 128 }, "model": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "runner_sha256": "022f956ca5185fb67d5913b21465d1e465f6a1fee09067846731c87718d4527c", "sampling": { "allow_custom_prompts": false, "answer_max_tokens": 128, "frequency_penalty": 0.0, "greedy": false, "logprob_token_ids": [], "logprobs": null, "max_tokens": 128, "min_p": 0.0, "n": 12, "presence_penalty": 0.0, "prompt_logprobs": null, "repetition_penalty": 1.0, "run_seed": 2701, "shuffle_thinking": false, "temperature": 0.6, "thinking": "budget", "thinking_budget": 192, "top_k": 20, "top_p": 0.95 }, "schema_version": 2, "timing": { "generation_seconds": 37.87951398547739, "model_load_seconds": 17.697869495488703, "sampled_tokens_per_second": 985.3083124115386 } } }, "schema_version": 1, "selection_rule": "earliest valid sample among maximal visible-example score; hidden labels grade only", "smoke_gate": { "gates": { "answer_truncation": false, "base_parse": true, "designed_oracle_not_below_base": false, "designed_parse": true, "designed_valid_macro_candidates": false, "overall_parse": true }, "metrics": { "base_oracle_hidden_coverage": 0.08333333333333333, "base_parse_rate": 0.6111111111111112, "designed_oracle_hidden_coverage": 0.0, "designed_parse_rate": 0.5694444444444444, "designed_valid_macro_using_candidates": 0, "overall_parse_rate": 0.5972222222222222, "overall_truncation_rate": 0.40046296296296297 }, "pass": false, "reasons": [ "designed_valid_macro_candidates", "answer_truncation", "designed_oracle_not_below_base" ], "required_arms_present": true } }