{ "dataset": "humaneval", "n": 164, "model": "base", "model_id": "Qwen/Qwen3.5-4B", "model_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "pass_at_1": 0.8963414634146342, "passed": 147, "total": 164, "per_problem": [ { "task_id": "HumanEval/0", "passed": true, "parse_ok": true, "n_thinking_tokens": 624, "n_answer_tokens": 165, "n_completion_tokens": 790, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/1", "passed": true, "parse_ok": true, "n_thinking_tokens": 673, "n_answer_tokens": 225, "n_completion_tokens": 899, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/2", "passed": true, "parse_ok": true, "n_thinking_tokens": 498, "n_answer_tokens": 94, "n_completion_tokens": 593, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/3", "passed": true, "parse_ok": true, "n_thinking_tokens": 286, "n_answer_tokens": 149, "n_completion_tokens": 436, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/4", "passed": true, "parse_ok": true, "n_thinking_tokens": 441, "n_answer_tokens": 178, "n_completion_tokens": 620, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/5", "passed": true, "parse_ok": true, "n_thinking_tokens": 556, "n_answer_tokens": 81, "n_completion_tokens": 638, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/6", "passed": true, "parse_ok": true, "n_thinking_tokens": 771, "n_answer_tokens": 210, "n_completion_tokens": 982, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/7", "passed": true, "parse_ok": true, "n_thinking_tokens": 360, "n_answer_tokens": 104, "n_completion_tokens": 465, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/8", "passed": true, "parse_ok": true, "n_thinking_tokens": 179, "n_answer_tokens": 76, "n_completion_tokens": 256, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/9", "passed": true, "parse_ok": true, "n_thinking_tokens": 385, "n_answer_tokens": 137, "n_completion_tokens": 523, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/10", "passed": true, "parse_ok": true, "n_thinking_tokens": 804, "n_answer_tokens": 199, "n_completion_tokens": 1004, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/11", "passed": true, "parse_ok": true, "n_thinking_tokens": 476, "n_answer_tokens": 106, "n_completion_tokens": 583, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/12", "passed": true, "parse_ok": true, "n_thinking_tokens": 272, "n_answer_tokens": 63, "n_completion_tokens": 336, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/13", "passed": true, "parse_ok": true, "n_thinking_tokens": 502, "n_answer_tokens": 92, "n_completion_tokens": 595, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/14", "passed": true, "parse_ok": true, "n_thinking_tokens": 223, "n_answer_tokens": 53, "n_completion_tokens": 277, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/15", "passed": true, "parse_ok": true, "n_thinking_tokens": 267, "n_answer_tokens": 89, "n_completion_tokens": 357, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/16", "passed": true, "parse_ok": true, "n_thinking_tokens": 276, "n_answer_tokens": 78, "n_completion_tokens": 355, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/17", "passed": true, "parse_ok": true, "n_thinking_tokens": 501, "n_answer_tokens": 248, "n_completion_tokens": 750, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/18", "passed": true, "parse_ok": true, "n_thinking_tokens": 558, "n_answer_tokens": 171, "n_completion_tokens": 730, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/19", "passed": true, "parse_ok": true, "n_thinking_tokens": 431, "n_answer_tokens": 228, "n_completion_tokens": 660, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/20", "passed": true, "parse_ok": true, "n_thinking_tokens": 759, "n_answer_tokens": 122, "n_completion_tokens": 882, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/21", "passed": true, "parse_ok": true, "n_thinking_tokens": 715, "n_answer_tokens": 182, "n_completion_tokens": 898, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/22", "passed": true, "parse_ok": true, "n_thinking_tokens": 314, "n_answer_tokens": 109, "n_completion_tokens": 424, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/23", "passed": true, "parse_ok": true, "n_thinking_tokens": 119, "n_answer_tokens": 50, "n_completion_tokens": 170, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/24", "passed": true, "parse_ok": true, "n_thinking_tokens": 876, "n_answer_tokens": 136, "n_completion_tokens": 1013, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/25", "passed": true, "parse_ok": true, "n_thinking_tokens": 524, "n_answer_tokens": 269, "n_completion_tokens": 794, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/26", "passed": true, "parse_ok": true, "n_thinking_tokens": 423, "n_answer_tokens": 115, "n_completion_tokens": 539, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/27", "passed": true, "parse_ok": true, "n_thinking_tokens": 291, "n_answer_tokens": 55, "n_completion_tokens": 347, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/28", "passed": true, "parse_ok": true, "n_thinking_tokens": 201, "n_answer_tokens": 67, "n_completion_tokens": 269, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/29", "passed": true, "parse_ok": true, "n_thinking_tokens": 210, "n_answer_tokens": 99, "n_completion_tokens": 310, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/30", "passed": true, "parse_ok": true, "n_thinking_tokens": 307, "n_answer_tokens": 136, "n_completion_tokens": 444, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/31", "passed": true, "parse_ok": true, "n_thinking_tokens": 402, "n_answer_tokens": 192, "n_completion_tokens": 595, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/32", "passed": false, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 193, "n_completion_tokens": 8387, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/33", "passed": true, "parse_ok": true, "n_thinking_tokens": 1017, "n_answer_tokens": 263, "n_completion_tokens": 1281, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/34", "passed": true, "parse_ok": true, "n_thinking_tokens": 203, "n_answer_tokens": 85, "n_completion_tokens": 289, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/35", "passed": true, "parse_ok": true, "n_thinking_tokens": 198, "n_answer_tokens": 94, "n_completion_tokens": 293, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/36", "passed": true, "parse_ok": true, "n_thinking_tokens": 3156, "n_answer_tokens": 141, "n_completion_tokens": 3298, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/37", "passed": true, "parse_ok": true, "n_thinking_tokens": 790, "n_answer_tokens": 233, "n_completion_tokens": 1024, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/38", "passed": false, "parse_ok": true, "n_thinking_tokens": 580, "n_answer_tokens": 146, "n_completion_tokens": 727, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/39", "passed": true, "parse_ok": true, "n_thinking_tokens": 1669, "n_answer_tokens": 161, "n_completion_tokens": 1831, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/40", "passed": true, "parse_ok": true, "n_thinking_tokens": 829, "n_answer_tokens": 84, "n_completion_tokens": 914, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/41", "passed": true, "parse_ok": true, "n_thinking_tokens": 389, "n_answer_tokens": 165, "n_completion_tokens": 555, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/42", "passed": true, "parse_ok": true, "n_thinking_tokens": 185, "n_answer_tokens": 129, "n_completion_tokens": 315, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/43", "passed": true, "parse_ok": true, "n_thinking_tokens": 1182, "n_answer_tokens": 45, "n_completion_tokens": 1228, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/44", "passed": true, "parse_ok": true, "n_thinking_tokens": 634, "n_answer_tokens": 157, "n_completion_tokens": 792, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/45", "passed": true, "parse_ok": true, "n_thinking_tokens": 156, "n_answer_tokens": 57, "n_completion_tokens": 214, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/46", "passed": true, "parse_ok": true, "n_thinking_tokens": 737, "n_answer_tokens": 323, "n_completion_tokens": 1061, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/47", "passed": true, "parse_ok": true, "n_thinking_tokens": 893, "n_answer_tokens": 152, "n_completion_tokens": 1046, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/48", "passed": true, "parse_ok": true, "n_thinking_tokens": 322, "n_answer_tokens": 84, "n_completion_tokens": 407, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/49", "passed": true, "parse_ok": true, "n_thinking_tokens": 441, "n_answer_tokens": 131, "n_completion_tokens": 573, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/50", "passed": false, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 65, "n_completion_tokens": 8259, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/51", "passed": true, "parse_ok": true, "n_thinking_tokens": 330, "n_answer_tokens": 162, "n_completion_tokens": 493, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/52", "passed": true, "parse_ok": true, "n_thinking_tokens": 409, "n_answer_tokens": 97, "n_completion_tokens": 507, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/53", "passed": true, "parse_ok": true, "n_thinking_tokens": 132, "n_answer_tokens": 60, "n_completion_tokens": 193, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/54", "passed": true, "parse_ok": true, "n_thinking_tokens": 2398, "n_answer_tokens": 158, "n_completion_tokens": 2557, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/55", "passed": true, "parse_ok": true, "n_thinking_tokens": 515, "n_answer_tokens": 123, "n_completion_tokens": 639, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/56", "passed": true, "parse_ok": true, "n_thinking_tokens": 366, "n_answer_tokens": 154, "n_completion_tokens": 521, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/57", "passed": true, "parse_ok": true, "n_thinking_tokens": 501, "n_answer_tokens": 165, "n_completion_tokens": 667, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/58", "passed": true, "parse_ok": true, "n_thinking_tokens": 419, "n_answer_tokens": 140, "n_completion_tokens": 560, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/59", "passed": true, "parse_ok": true, "n_thinking_tokens": 590, "n_answer_tokens": 198, "n_completion_tokens": 789, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/60", "passed": true, "parse_ok": true, "n_thinking_tokens": 431, "n_answer_tokens": 122, "n_completion_tokens": 554, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/61", "passed": true, "parse_ok": true, "n_thinking_tokens": 267, "n_answer_tokens": 71, "n_completion_tokens": 339, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/62", "passed": true, "parse_ok": true, "n_thinking_tokens": 619, "n_answer_tokens": 138, "n_completion_tokens": 758, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/63", "passed": true, "parse_ok": true, "n_thinking_tokens": 601, "n_answer_tokens": 274, "n_completion_tokens": 876, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/64", "passed": true, "parse_ok": true, "n_thinking_tokens": 789, "n_answer_tokens": 188, "n_completion_tokens": 978, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/65", "passed": true, "parse_ok": true, "n_thinking_tokens": 791, "n_answer_tokens": 135, "n_completion_tokens": 927, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/66", "passed": true, "parse_ok": true, "n_thinking_tokens": 411, "n_answer_tokens": 39, "n_completion_tokens": 451, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/67", "passed": true, "parse_ok": true, "n_thinking_tokens": 472, "n_answer_tokens": 58, "n_completion_tokens": 531, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/68", "passed": true, "parse_ok": true, "n_thinking_tokens": 555, "n_answer_tokens": 432, "n_completion_tokens": 988, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/69", "passed": true, "parse_ok": true, "n_thinking_tokens": 529, "n_answer_tokens": 66, "n_completion_tokens": 596, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/70", "passed": true, "parse_ok": true, "n_thinking_tokens": 706, "n_answer_tokens": 112, "n_completion_tokens": 819, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/71", "passed": true, "parse_ok": true, "n_thinking_tokens": 399, "n_answer_tokens": 211, "n_completion_tokens": 611, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/72", "passed": true, "parse_ok": true, "n_thinking_tokens": 350, "n_answer_tokens": 267, "n_completion_tokens": 618, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/73", "passed": true, "parse_ok": true, "n_thinking_tokens": 627, "n_answer_tokens": 133, "n_completion_tokens": 761, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/74", "passed": true, "parse_ok": true, "n_thinking_tokens": 612, "n_answer_tokens": 140, "n_completion_tokens": 753, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/75", "passed": true, "parse_ok": true, "n_thinking_tokens": 960, "n_answer_tokens": 280, "n_completion_tokens": 1241, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/76", "passed": true, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 232, "n_completion_tokens": 8426, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/77", "passed": true, "parse_ok": true, "n_thinking_tokens": 631, "n_answer_tokens": 98, "n_completion_tokens": 730, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/78", "passed": true, "parse_ok": true, "n_thinking_tokens": 966, "n_answer_tokens": 385, "n_completion_tokens": 1352, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/79", "passed": true, "parse_ok": true, "n_thinking_tokens": 519, "n_answer_tokens": 173, "n_completion_tokens": 693, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/80", "passed": true, "parse_ok": true, "n_thinking_tokens": 619, "n_answer_tokens": 83, "n_completion_tokens": 703, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/81", "passed": true, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 467, "n_completion_tokens": 8661, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/82", "passed": true, "parse_ok": true, "n_thinking_tokens": 352, "n_answer_tokens": 138, "n_completion_tokens": 491, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/83", "passed": true, "parse_ok": true, "n_thinking_tokens": 1301, "n_answer_tokens": 77, "n_completion_tokens": 1379, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/84", "passed": true, "parse_ok": true, "n_thinking_tokens": 667, "n_answer_tokens": 183, "n_completion_tokens": 851, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/85", "passed": true, "parse_ok": true, "n_thinking_tokens": 422, "n_answer_tokens": 60, "n_completion_tokens": 483, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/86", "passed": true, "parse_ok": true, "n_thinking_tokens": 683, "n_answer_tokens": 58, "n_completion_tokens": 742, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/87", "passed": true, "parse_ok": true, "n_thinking_tokens": 609, "n_answer_tokens": 341, "n_completion_tokens": 951, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/88", "passed": true, "parse_ok": true, "n_thinking_tokens": 1074, "n_answer_tokens": 172, "n_completion_tokens": 1247, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/89", "passed": true, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 246, "n_completion_tokens": 8440, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/90", "passed": true, "parse_ok": true, "n_thinking_tokens": 514, "n_answer_tokens": 179, "n_completion_tokens": 694, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/91", "passed": true, "parse_ok": true, "n_thinking_tokens": 651, "n_answer_tokens": 125, "n_completion_tokens": 777, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/92", "passed": true, "parse_ok": true, "n_thinking_tokens": 567, "n_answer_tokens": 212, "n_completion_tokens": 780, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/93", "passed": false, "parse_ok": true, "n_thinking_tokens": 730, "n_answer_tokens": 303, "n_completion_tokens": 1034, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/94", "passed": true, "parse_ok": true, "n_thinking_tokens": 954, "n_answer_tokens": 201, "n_completion_tokens": 1156, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/95", "passed": true, "parse_ok": true, "n_thinking_tokens": 815, "n_answer_tokens": 116, "n_completion_tokens": 932, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/96", "passed": true, "parse_ok": true, "n_thinking_tokens": 796, "n_answer_tokens": 252, "n_completion_tokens": 1049, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/97", "passed": true, "parse_ok": true, "n_thinking_tokens": 415, "n_answer_tokens": 136, "n_completion_tokens": 552, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/98", "passed": true, "parse_ok": true, "n_thinking_tokens": 573, "n_answer_tokens": 121, "n_completion_tokens": 695, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/99", "passed": true, "parse_ok": true, "n_thinking_tokens": 560, "n_answer_tokens": 104, "n_completion_tokens": 665, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/100", "passed": true, "parse_ok": true, "n_thinking_tokens": 765, "n_answer_tokens": 200, "n_completion_tokens": 966, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/101", "passed": true, "parse_ok": true, "n_thinking_tokens": 459, "n_answer_tokens": 144, "n_completion_tokens": 604, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/102", "passed": true, "parse_ok": true, "n_thinking_tokens": 717, "n_answer_tokens": 173, "n_completion_tokens": 891, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/103", "passed": true, "parse_ok": true, "n_thinking_tokens": 607, "n_answer_tokens": 193, "n_completion_tokens": 801, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/104", "passed": true, "parse_ok": true, "n_thinking_tokens": 846, "n_answer_tokens": 186, "n_completion_tokens": 1033, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/105", "passed": true, "parse_ok": true, "n_thinking_tokens": 977, "n_answer_tokens": 215, "n_completion_tokens": 1193, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/106", "passed": true, "parse_ok": true, "n_thinking_tokens": 813, "n_answer_tokens": 233, "n_completion_tokens": 1047, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/107", "passed": true, "parse_ok": true, "n_thinking_tokens": 626, "n_answer_tokens": 159, "n_completion_tokens": 786, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/108", "passed": false, "parse_ok": true, "n_thinking_tokens": 1145, "n_answer_tokens": 129, "n_completion_tokens": 1275, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/109", "passed": false, "parse_ok": true, "n_thinking_tokens": 1061, "n_answer_tokens": 154, "n_completion_tokens": 1216, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/110", "passed": true, "parse_ok": true, "n_thinking_tokens": 745, "n_answer_tokens": 86, "n_completion_tokens": 832, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/111", "passed": true, "parse_ok": true, "n_thinking_tokens": 781, "n_answer_tokens": 114, "n_completion_tokens": 896, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/112", "passed": true, "parse_ok": true, "n_thinking_tokens": 654, "n_answer_tokens": 75, "n_completion_tokens": 730, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/113", "passed": true, "parse_ok": true, "n_thinking_tokens": 714, "n_answer_tokens": 262, "n_completion_tokens": 977, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/114", "passed": true, "parse_ok": true, "n_thinking_tokens": 509, "n_answer_tokens": 140, "n_completion_tokens": 650, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/115", "passed": true, "parse_ok": true, "n_thinking_tokens": 1053, "n_answer_tokens": 62, "n_completion_tokens": 1116, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/116", "passed": true, "parse_ok": true, "n_thinking_tokens": 1059, "n_answer_tokens": 102, "n_completion_tokens": 1162, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/117", "passed": true, "parse_ok": true, "n_thinking_tokens": 3173, "n_answer_tokens": 103, "n_completion_tokens": 3277, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/118", "passed": true, "parse_ok": true, "n_thinking_tokens": 1002, "n_answer_tokens": 265, "n_completion_tokens": 1268, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/119", "passed": true, "parse_ok": true, "n_thinking_tokens": 568, "n_answer_tokens": 131, "n_completion_tokens": 700, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/120", "passed": false, "parse_ok": true, "n_thinking_tokens": 669, "n_answer_tokens": 63, "n_completion_tokens": 733, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/121", "passed": true, "parse_ok": true, "n_thinking_tokens": 492, "n_answer_tokens": 151, "n_completion_tokens": 644, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/122", "passed": false, "parse_ok": true, "n_thinking_tokens": 952, "n_answer_tokens": 96, "n_completion_tokens": 1049, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/123", "passed": true, "parse_ok": true, "n_thinking_tokens": 859, "n_answer_tokens": 146, "n_completion_tokens": 1006, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/124", "passed": true, "parse_ok": true, "n_thinking_tokens": 787, "n_answer_tokens": 615, "n_completion_tokens": 1403, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/125", "passed": false, "parse_ok": true, "n_thinking_tokens": 661, "n_answer_tokens": 80, "n_completion_tokens": 742, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/126", "passed": true, "parse_ok": true, "n_thinking_tokens": 1655, "n_answer_tokens": 112, "n_completion_tokens": 1768, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/127", "passed": true, "parse_ok": true, "n_thinking_tokens": 1058, "n_answer_tokens": 222, "n_completion_tokens": 1281, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/128", "passed": true, "parse_ok": true, "n_thinking_tokens": 511, "n_answer_tokens": 224, "n_completion_tokens": 736, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/129", "passed": true, "parse_ok": true, "n_thinking_tokens": 1179, "n_answer_tokens": 321, "n_completion_tokens": 1501, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/130", "passed": false, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 1024, "n_completion_tokens": 9218, "thinking_closed": true, "forced_close": true, "truncated": true }, { "task_id": "HumanEval/131", "passed": false, "parse_ok": true, "n_thinking_tokens": 556, "n_answer_tokens": 143, "n_completion_tokens": 700, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/132", "passed": false, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 479, "n_completion_tokens": 8673, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/133", "passed": true, "parse_ok": true, "n_thinking_tokens": 479, "n_answer_tokens": 187, "n_completion_tokens": 667, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/134", "passed": false, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 275, "n_completion_tokens": 8469, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/135", "passed": true, "parse_ok": true, "n_thinking_tokens": 514, "n_answer_tokens": 150, "n_completion_tokens": 665, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/136", "passed": true, "parse_ok": true, "n_thinking_tokens": 476, "n_answer_tokens": 235, "n_completion_tokens": 712, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/137", "passed": true, "parse_ok": true, "n_thinking_tokens": 572, "n_answer_tokens": 245, "n_completion_tokens": 818, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/138", "passed": true, "parse_ok": true, "n_thinking_tokens": 411, "n_answer_tokens": 54, "n_completion_tokens": 466, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/139", "passed": true, "parse_ok": true, "n_thinking_tokens": 584, "n_answer_tokens": 140, "n_completion_tokens": 725, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/140", "passed": false, "parse_ok": true, "n_thinking_tokens": 757, "n_answer_tokens": 85, "n_completion_tokens": 843, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/141", "passed": true, "parse_ok": true, "n_thinking_tokens": 761, "n_answer_tokens": 204, "n_completion_tokens": 966, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/142", "passed": true, "parse_ok": true, "n_thinking_tokens": 713, "n_answer_tokens": 76, "n_completion_tokens": 790, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/143", "passed": true, "parse_ok": true, "n_thinking_tokens": 539, "n_answer_tokens": 132, "n_completion_tokens": 672, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/144", "passed": true, "parse_ok": true, "n_thinking_tokens": 607, "n_answer_tokens": 144, "n_completion_tokens": 752, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/145", "passed": false, "parse_ok": true, "n_thinking_tokens": 3326, "n_answer_tokens": 46, "n_completion_tokens": 3373, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/146", "passed": true, "parse_ok": true, "n_thinking_tokens": 822, "n_answer_tokens": 105, "n_completion_tokens": 928, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/147", "passed": true, "parse_ok": true, "n_thinking_tokens": 1218, "n_answer_tokens": 420, "n_completion_tokens": 1639, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/148", "passed": true, "parse_ok": true, "n_thinking_tokens": 527, "n_answer_tokens": 133, "n_completion_tokens": 661, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/149", "passed": true, "parse_ok": true, "n_thinking_tokens": 658, "n_answer_tokens": 256, "n_completion_tokens": 915, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/150", "passed": true, "parse_ok": true, "n_thinking_tokens": 431, "n_answer_tokens": 197, "n_completion_tokens": 629, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/151", "passed": true, "parse_ok": true, "n_thinking_tokens": 855, "n_answer_tokens": 197, "n_completion_tokens": 1053, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/152", "passed": true, "parse_ok": true, "n_thinking_tokens": 495, "n_answer_tokens": 254, "n_completion_tokens": 750, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/153", "passed": true, "parse_ok": true, "n_thinking_tokens": 567, "n_answer_tokens": 116, "n_completion_tokens": 684, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/154", "passed": true, "parse_ok": true, "n_thinking_tokens": 8192, "n_answer_tokens": 117, "n_completion_tokens": 8311, "thinking_closed": true, "forced_close": true, "truncated": false }, { "task_id": "HumanEval/155", "passed": true, "parse_ok": true, "n_thinking_tokens": 328, "n_answer_tokens": 151, "n_completion_tokens": 480, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/156", "passed": true, "parse_ok": true, "n_thinking_tokens": 819, "n_answer_tokens": 206, "n_completion_tokens": 1026, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/157", "passed": true, "parse_ok": true, "n_thinking_tokens": 473, "n_answer_tokens": 208, "n_completion_tokens": 682, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/158", "passed": true, "parse_ok": true, "n_thinking_tokens": 846, "n_answer_tokens": 90, "n_completion_tokens": 937, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/159", "passed": true, "parse_ok": true, "n_thinking_tokens": 513, "n_answer_tokens": 361, "n_completion_tokens": 875, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/160", "passed": false, "parse_ok": true, "n_thinking_tokens": 746, "n_answer_tokens": 320, "n_completion_tokens": 1067, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/161", "passed": true, "parse_ok": true, "n_thinking_tokens": 350, "n_answer_tokens": 136, "n_completion_tokens": 487, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/162", "passed": false, "parse_ok": true, "n_thinking_tokens": 509, "n_answer_tokens": 122, "n_completion_tokens": 632, "thinking_closed": true, "forced_close": false, "truncated": false }, { "task_id": "HumanEval/163", "passed": true, "parse_ok": true, "n_thinking_tokens": 2131, "n_answer_tokens": 146, "n_completion_tokens": 2278, "thinking_closed": true, "forced_close": false, "truncated": false } ], "greedy": true, "seed": 0, "thinking": "budget", "thinking_budget": 8192, "answer_max_tokens": 1024, "max_new_tokens": 9216, "max_model_len": 16384, "visible_tests": 1, "timeout_s": 5.0, "mbpp_config": null, "offset": 0, "n_parse_failed": 0, "n_truncated": 1, "n_forced_close": 9, "runner_sha256": "cf1a766a70df8e3aacae18fc52a1a2186f2f8fc20161a6ef679b3dd51f60fcd2", "resolved_sampling": { "temperature": 0.0, "top_p": 1.0, "top_k": 0, "min_p": 0.0, "presence_penalty": 0.0, "frequency_penalty": 0.0, "repetition_penalty": 1.0 }, "generation_seconds": 322.18399737682194, "sampled_tokens_per_second": 646.2425250639673, "model_load_seconds": 115.05109119601548, "wall_seconds": 459.6423298199661 }