{ "args": { "runs": "main_no_state_selected_s900,main_state_w025_selected_s900,main_state_l12_off_long_selected_s900,main_state_l12_w025_long_selected_s900", "eval_size": 256, "eval_lengths": "24", "eval_seed": 91001, "eval_batch_size": 8, "output_csv": "experiments/qwen_checkpoint_scheduled_state_compiler/analysis/selected_retest_metrics.csv", "output_json": "experiments/qwen_checkpoint_scheduled_state_compiler/analysis/selected_retest_results.json" }, "rows": [ { "run": "main_no_state_selected_s900", "variant": "copy_trace", "selected_step": 800, "selected_tag": "eval_trace_long", "state_loss_schedule": "none", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.2734375, "split": "fresh_standard_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.3203125, "executor_target_mass": 0.010309316596249118, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.9986979166666666, "arg_accuracy": 0.9482421875, "op_pos_accuracy": 0.9996744791666666, "arg_pos_accuracy": 0.94921875, "program_exact": 0.3125, "state_accuracy": 0.5812174479166666, "state_all_exact": 0.3125, "state_prefix_fraction": 0.57373046875, "state_mean_correct_prefix": 13.76953125 }, { "run": "main_no_state_selected_s900", "variant": "copy_trace", "selected_step": 800, "selected_tag": "eval_trace_long", "state_loss_schedule": "none", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.2734375, "split": "fresh_paraphrase_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.1953125, "executor_target_mass": 0.01030928417458199, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.9964192708333334, "arg_accuracy": 0.9275716145833334, "op_pos_accuracy": 0.9978841145833334, "arg_pos_accuracy": 0.9305013020833334, "program_exact": 0.18359375, "state_accuracy": 0.5472005208333334, "state_all_exact": 0.18359375, "state_prefix_fraction": 0.54443359375, "state_mean_correct_prefix": 13.06640625 }, { "run": "main_no_state_selected_s900", "variant": "copy_trace", "selected_step": 800, "selected_tag": "eval_trace_long", "state_loss_schedule": "none", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.2734375, "split": "fresh_paired_len24", "n": 512.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.25, "executor_target_mass": 0.01030928133695852, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.99755859375, "arg_accuracy": 0.9390462239583334, "op_pos_accuracy": 0.9984537760416666, "arg_pos_accuracy": 0.939697265625, "program_exact": 0.2421875, "state_accuracy": 0.5533040364583334, "state_all_exact": 0.2421875, "state_prefix_fraction": 0.551025390625, "state_mean_correct_prefix": 13.224609375, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.40625, "executor_pair_both_correct": 0.171875, "compiler_pair_program_consistency": 0.40625, "compiler_pair_state_consistency": 0.40625 }, { "run": "main_state_w025_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "constant", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.3046875, "split": "fresh_standard_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.33984375, "executor_target_mass": 0.010309299657819793, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 0.9514973958333334, "op_pos_accuracy": 1.0, "arg_pos_accuracy": 0.95166015625, "program_exact": 0.3359375, "state_accuracy": 0.5940755208333334, "state_all_exact": 0.3359375, "state_prefix_fraction": 0.5906575520833334, "state_mean_correct_prefix": 14.17578125 }, { "run": "main_state_w025_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "constant", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.3046875, "split": "fresh_paraphrase_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.3125, "executor_target_mass": 0.010309282632078975, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.9998372395833334, "arg_accuracy": 0.9488932291666666, "op_pos_accuracy": 1.0, "arg_pos_accuracy": 0.94482421875, "program_exact": 0.30078125, "state_accuracy": 0.58056640625, "state_all_exact": 0.30078125, "state_prefix_fraction": 0.578125, "state_mean_correct_prefix": 13.875 }, { "run": "main_state_w025_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "constant", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.3046875, "split": "fresh_paired_len24", "n": 512.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.328125, "executor_target_mass": 0.010309276593034156, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.9999186197916666, "arg_accuracy": 0.94970703125, "op_pos_accuracy": 1.0, "arg_pos_accuracy": 0.9479166666666666, "program_exact": 0.318359375, "state_accuracy": 0.583251953125, "state_all_exact": 0.318359375, "state_prefix_fraction": 0.5782877604166666, "state_mean_correct_prefix": 13.87890625, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.56640625, "executor_pair_both_correct": 0.296875, "compiler_pair_program_consistency": 0.55859375, "compiler_pair_state_consistency": 0.55859375 }, { "run": "main_state_l12_off_long_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "short=1.0,medium=1.0,train=1.0,long=0.0", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.2265625, "split": "fresh_standard_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.3125, "executor_target_mass": 0.01030931354034692, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.9990234375, "arg_accuracy": 0.9475911458333334, "op_pos_accuracy": 0.9996744791666666, "arg_pos_accuracy": 0.9493815104166666, "program_exact": 0.296875, "state_accuracy": 0.5921223958333334, "state_all_exact": 0.296875, "state_prefix_fraction": 0.5807291666666666, "state_mean_correct_prefix": 13.9375 }, { "run": "main_state_l12_off_long_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "short=1.0,medium=1.0,train=1.0,long=0.0", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.2265625, "split": "fresh_paraphrase_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.07421875, "executor_target_mass": 0.01030929441913031, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.9803059895833334, "arg_accuracy": 0.9033203125, "op_pos_accuracy": 0.9765625, "arg_pos_accuracy": 0.9060872395833334, "program_exact": 0.0703125, "state_accuracy": 0.5304361979166666, "state_all_exact": 0.0703125, "state_prefix_fraction": 0.5279947916666666, "state_mean_correct_prefix": 12.671875 }, { "run": "main_state_l12_off_long_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "short=1.0,medium=1.0,train=1.0,long=0.0", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.2265625, "split": "fresh_paired_len24", "n": 512.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.19140625, "executor_target_mass": 0.01030928289401345, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 0.991455078125, "arg_accuracy": 0.9261067708333334, "op_pos_accuracy": 0.9901529947916666, "arg_pos_accuracy": 0.9296061197916666, "program_exact": 0.18359375, "state_accuracy": 0.5425618489583334, "state_all_exact": 0.18359375, "state_prefix_fraction": 0.541748046875, "state_mean_correct_prefix": 13.001953125, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.296875, "executor_pair_both_correct": 0.0625, "compiler_pair_program_consistency": 0.29296875, "compiler_pair_state_consistency": 0.29296875 }, { "run": "main_state_l12_w025_long_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "short=1.0,medium=1.0,train=1.0,long=0.25", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.3046875, "split": "fresh_standard_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.33984375, "executor_target_mass": 0.010309303004760295, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 0.95068359375, "op_pos_accuracy": 1.0, "arg_pos_accuracy": 0.9513346354166666, "program_exact": 0.328125, "state_accuracy": 0.6027018229166666, "state_all_exact": 0.328125, "state_prefix_fraction": 0.59423828125, "state_mean_correct_prefix": 14.26171875 }, { "run": "main_state_l12_w025_long_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "short=1.0,medium=1.0,train=1.0,long=0.25", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.3046875, "split": "fresh_paraphrase_len24", "n": 256.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.28125, "executor_target_mass": 0.010309282661182806, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 0.9485677083333334, "op_pos_accuracy": 1.0, "arg_pos_accuracy": 0.9493815104166666, "program_exact": 0.27734375, "state_accuracy": 0.5774739583333334, "state_all_exact": 0.27734375, "state_prefix_fraction": 0.5768229166666666, "state_mean_correct_prefix": 13.84375 }, { "run": "main_state_l12_w025_long_selected_s900", "variant": "copy_trace_state_scheduled", "selected_step": 800, "selected_tag": "eval_trace_state_scheduled_long", "state_loss_schedule": "short=1.0,medium=1.0,train=1.0,long=0.25", "selection_metric": "paired_len24_executor_accuracy", "selection_value": 0.3046875, "split": "fresh_paired_len24", "n": 512.0, "direct_accuracy": NaN, "direct_target_mass": NaN, "executor_accuracy": 0.322265625, "executor_target_mass": 0.010309282530215569, "init_accuracy": 1.0, "init_pos_accuracy": 1.0, "op_accuracy": 1.0, "arg_accuracy": 0.9493001302083334, "op_pos_accuracy": 1.0, "arg_pos_accuracy": 0.94921875, "program_exact": 0.314453125, "state_accuracy": 0.5730794270833334, "state_all_exact": 0.314453125, "state_prefix_fraction": 0.5712890625, "state_mean_correct_prefix": 13.7109375, "pair_true_answer_consistency": 1.0, "pair_true_length_consistency": 1.0, "executor_pair_answer_consistency": 0.70703125, "executor_pair_both_correct": 0.30078125, "compiler_pair_program_consistency": 0.70703125, "compiler_pair_state_consistency": 0.70703125 } ] }