[ { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.227040999999998, "cost_usd": 6.227040999999998, "denominator": 20, "duration_basis": "run", "duration_ms": 346973, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 37, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3055311, "tokens_cache_write": 96347, "tokens_input": 834, "tokens_output": 24729, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 39, "vendor_mcp": 17 }, "tool_result_tokens": 42350, "tool_time_ms": 5707, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.361023999999999, "cost_usd": 6.361023999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 439392, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 50, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3868274, "tokens_cache_write": 70743, "tokens_input": 94, "tokens_output": 21539, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 48, "vendor_mcp": 10 }, "tool_result_tokens": 28047, "tool_time_ms": 5078, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.316288, "cost_usd": 5.316288, "denominator": 20, "duration_basis": "run", "duration_ms": 442948, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 36, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2610418, "tokens_cache_write": 68937, "tokens_input": 68, "tokens_output": 26529, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 38, "vendor_mcp": 10 }, "tool_result_tokens": 25209, "tool_time_ms": 8315, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.173740500000001, "cost_usd": 6.173740500000001, "denominator": 20, "duration_basis": "run", "duration_ms": 357240, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 66, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3271670, "tokens_cache_write": 169518, "tokens_input": 127, "tokens_output": 30494, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 89, "subagent": 3, "vendor_mcp": 11 }, "tool_result_tokens": 61174, "tool_time_ms": 26697, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.822244000000001, "cost_usd": 5.822244000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 336164, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 48, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3441574, "tokens_cache_write": 65611, "tokens_input": 90, "tokens_output": 21351, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 52, "vendor_mcp": 11 }, "tool_result_tokens": 24464, "tool_time_ms": 5533, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.28863425, "cost_usd": 10.28863425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 780911, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 90, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7177552, "tokens_cache_write": 86299, "tokens_input": 171, "tokens_output": 31050, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 66, "vendor_mcp": 7 }, "tool_result_tokens": 27994, "tool_time_ms": 34401, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.112191999999999, "cost_usd": 9.112191999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 489615, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 89, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6402352, "tokens_cache_write": 68960, "tokens_input": 169, "tokens_output": 26579, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 71, "vendor_mcp": 1 }, "tool_result_tokens": 22316, "tool_time_ms": 8891, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.000022000000001, "cost_usd": 10.000022000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 643810, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 82, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6749092, "tokens_cache_write": 89579, "tokens_input": 155, "tokens_output": 29156, "tool_calls": 68, "tool_calls_by_class": { "cc_builtin": 60, "vendor_mcp": 8 }, "tool_result_tokens": 31555, "tool_time_ms": 7551, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.117322999999999, "cost_usd": 10.117322999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 538007, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 83, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6878943, "tokens_cache_write": 85563, "tokens_input": 157, "tokens_output": 30511, "tool_calls": 68, "tool_calls_by_class": { "cc_builtin": 64, "vendor_mcp": 4 }, "tool_result_tokens": 29723, "tool_time_ms": 8987, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.545640500000003, "cost_usd": 9.545640500000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 561567, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 88, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6515996, "tokens_cache_write": 104196, "tokens_input": 166, "tokens_output": 28570, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 61, "vendor_mcp": 11 }, "tool_result_tokens": 26166, "tool_time_ms": 34414, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.221667000000002, "cost_usd": 6.221667000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 339614, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 78, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4324347, "tokens_cache_write": 51214, "tokens_input": 149, "tokens_output": 17431, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 48 }, "tool_result_tokens": 14683, "tool_time_ms": 3696, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.339457000000001, "cost_usd": 6.339457000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 378765, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 79, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4418877, "tokens_cache_write": 49496, "tokens_input": 151, "tokens_output": 18583, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 14518, "tool_time_ms": 3137, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.315607999999999, "cost_usd": 6.315607999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 362380, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4440038, "tokens_cache_write": 46507, "tokens_input": 153, "tokens_output": 18878, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 13304, "tool_time_ms": 2847, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.79191, "cost_usd": 5.79191, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 350036, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 75, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4067180, "tokens_cache_write": 43875, "tokens_input": 143, "tokens_output": 16916, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 12466, "tool_time_ms": 2954, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.182381, "cost_usd": 6.182381, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 357100, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 77, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4289101, "tokens_cache_write": 48193, "tokens_input": 147, "tokens_output": 18559, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 47 }, "tool_result_tokens": 14184, "tool_time_ms": 3203, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.5403595, "cost_usd": 10.5403595, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 910956, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 142, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5394789, "tokens_cache_write": 182101, "tokens_input": 274, "tokens_output": 47979, "tool_calls": 160, "tool_calls_by_class": { "cc_builtin": 112, "vendor_mcp": 48 }, "tool_result_tokens": 52537, "tool_time_ms": 16973, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.156063, "cost_usd": 10.156063, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 814982, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 134, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5139752, "tokens_cache_write": 183582, "tokens_input": 295, "tokens_output": 44787, "tool_calls": 145, "tool_calls_by_class": { "cc_builtin": 112, "vendor_mcp": 33 }, "tool_result_tokens": 65074, "tool_time_ms": 11262, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.144782000000001, "cost_usd": 9.144782000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 740560, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 129, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4886284, "tokens_cache_write": 153165, "tokens_input": 250, "tokens_output": 41691, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 100, "vendor_mcp": 40 }, "tool_result_tokens": 50573, "tool_time_ms": 15316, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.187615, "cost_usd": 9.187615, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 831987, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 138, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5234485, "tokens_cache_write": 132535, "tokens_input": 269, "tokens_output": 42930, "tool_calls": 143, "tool_calls_by_class": { "cc_builtin": 111, "vendor_mcp": 32 }, "tool_result_tokens": 52592, "tool_time_ms": 11550, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.247797499999999, "cost_usd": 9.247797499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 810780, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 125, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4781523, "tokens_cache_write": 156016, "tokens_input": 244, "tokens_output": 42999, "tool_calls": 135, "tool_calls_by_class": { "cc_builtin": 100, "vendor_mcp": 35 }, "tool_result_tokens": 63164, "tool_time_ms": 11071, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.530197, "cost_usd": 2.530197, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 83006, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 22, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 733507, "tokens_cache_write": 80765, "tokens_input": 44, "tokens_output": 3619, "tool_calls": 18, "tool_calls_by_class": { "cc_builtin": 11, "vendor_mcp": 7 }, "tool_result_tokens": 12385, "tool_time_ms": 1523, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.958046, "cost_usd": 1.958046, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 86024, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 24, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 855186, "tokens_cache_write": 45614, "tokens_input": 48, "tokens_output": 3802, "tool_calls": 22, "tool_calls_by_class": { "cc_builtin": 14, "vendor_mcp": 8 }, "tool_result_tokens": 10543, "tool_time_ms": 1687, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.818076, "cost_usd": 1.818076, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 89683, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 27, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 954726, "tokens_cache_write": 32373, "tokens_input": 54, "tokens_output": 4307, "tool_calls": 22, "tool_calls_by_class": { "cc_builtin": 14, "vendor_mcp": 8 }, "tool_result_tokens": 3338, "tool_time_ms": 1634, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.3945619999999999, "cost_usd": 1.3945619999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 97068, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 22, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 775952, "tokens_cache_write": 22146, "tokens_input": 44, "tokens_output": 3505, "tool_calls": 19, "tool_calls_by_class": { "cc_builtin": 15, "vendor_mcp": 4 }, "tool_result_tokens": 2678, "tool_time_ms": 1414, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.91797, "cost_usd": 1.91797, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 94440, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 27, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 975030, "tokens_cache_write": 36665, "tokens_input": 54, "tokens_output": 4182, "tool_calls": 22, "tool_calls_by_class": { "cc_builtin": 15, "vendor_mcp": 7 }, "tool_result_tokens": 12232, "tool_time_ms": 1778, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.629149, "cost_usd": 11.629149, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 750828, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 147, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5556959, "tokens_cache_write": 209307, "tokens_input": 290, "tokens_output": 37663, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 111, "vendor_mcp": 37 }, "tool_result_tokens": 59578, "tool_time_ms": 10698, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.961219999999999, "cost_usd": 10.961219999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 750907, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 154, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5700280, "tokens_cache_write": 168822, "tokens_input": 305, "tokens_output": 37629, "tool_calls": 150, "tool_calls_by_class": { "cc_builtin": 117, "vendor_mcp": 33 }, "tool_result_tokens": 47037, "tool_time_ms": 9799, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.288636, "cost_usd": 10.288636, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 732182, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 140, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5161276, "tokens_cache_write": 165309, "tokens_input": 278, "tokens_output": 36368, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 107, "vendor_mcp": 33 }, "tool_result_tokens": 51746, "tool_time_ms": 9233, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.970573, "cost_usd": 11.970573, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 795654, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 158, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5943183, "tokens_cache_write": 198981, "tokens_input": 312, "tokens_output": 40893, "tool_calls": 159, "tool_calls_by_class": { "cc_builtin": 117, "vendor_mcp": 42 }, "tool_result_tokens": 71153, "tool_time_ms": 11878, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.733429, "cost_usd": 10.733429, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 747639, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 152, "model": "claude-fable-5", "path": "runs/cbm/claude-fable-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5878489, "tokens_cache_write": 145992, "tokens_input": 300, "tokens_output": 38642, "tool_calls": 151, "tool_calls_by_class": { "cc_builtin": 113, "vendor_mcp": 38 }, "tool_result_tokens": 55767, "tool_time_ms": 10917, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8699886, "cost_usd": 5.8699886, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 782409, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 215, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 25466666, "tokens_cache_write": 1578490, "tokens_input": 1282, "tokens_output": 33012, "tool_calls": 202, "tool_calls_by_class": { "cc_builtin": 135, "harness_builtin": 3, "vendor_mcp": 64 }, "tool_result_tokens": 95203, "tool_time_ms": 10726, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.01355775, "cost_usd": 6.01355775, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 786127, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 213, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 21558193, "tokens_cache_write": 1737089, "tokens_input": 1293, "tokens_output": 33434, "tool_calls": 212, "tool_calls_by_class": { "cc_builtin": 136, "harness_builtin": 2, "vendor_mcp": 74 }, "tool_result_tokens": 106947, "tool_time_ms": 13380, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.79009925, "cost_usd": 4.79009925, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 660895, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 171, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19574224, "tokens_cache_write": 1327565, "tokens_input": 1054, "tokens_output": 27490, "tool_calls": 172, "tool_calls_by_class": { "cc_builtin": 103, "harness_builtin": 1, "vendor_mcp": 68 }, "tool_result_tokens": 123676, "tool_time_ms": 12597, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.107863649999999, "cost_usd": 6.107863649999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 866737, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 218, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 22604664, "tokens_cache_write": 1694256, "tokens_input": 1297, "tokens_output": 38105, "tool_calls": 225, "tool_calls_by_class": { "cc_builtin": 86, "harness_builtin": 3, "vendor_mcp": 136 }, "tool_result_tokens": 247773, "tool_time_ms": 17397, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.1808815500000005, "cost_usd": 5.1808815500000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 803924, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 8, "family": "cc_competitor", "llm_calls": 212, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 21079434, "tokens_cache_write": 1422091, "tokens_input": 1277, "tokens_output": 37538, "tool_calls": 226, "tool_calls_by_class": { "cc_builtin": 79, "harness_builtin": 1, "vendor_mcp": 146 }, "tool_result_tokens": 143679, "tool_time_ms": 20418, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.796869699999999, "cost_usd": 4.796869699999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 656385, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 172, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 18798247, "tokens_cache_write": 1383183, "tokens_input": 1069, "tokens_output": 29922, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 139, "harness_builtin": 2, "vendor_mcp": 43 }, "tool_result_tokens": 78277, "tool_time_ms": 14462, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.321819950000001, "cost_usd": 5.321819950000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 737703, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 194, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 21862330, "tokens_cache_write": 1378974, "tokens_input": 1189, "tokens_output": 32669, "tool_calls": 207, "tool_calls_by_class": { "cc_builtin": 114, "harness_builtin": 2, "vendor_mcp": 91 }, "tool_result_tokens": 104652, "tool_time_ms": 16366, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.973709700000001, "cost_usd": 4.973709700000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 630841, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 164, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19211867, "tokens_cache_write": 1457430, "tokens_input": 1028, "tokens_output": 27327, "tool_calls": 166, "tool_calls_by_class": { "cc_builtin": 122, "harness_builtin": 1, "vendor_mcp": 43 }, "tool_result_tokens": 97605, "tool_time_ms": 9987, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2638969, "cost_usd": 5.2638969, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 741108, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 219, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 25017849, "tokens_cache_write": 1304038, "tokens_input": 1391, "tokens_output": 30529, "tool_calls": 197, "tool_calls_by_class": { "cc_builtin": 186, "harness_builtin": 1, "vendor_mcp": 10 }, "tool_result_tokens": 75257, "tool_time_ms": 8439, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.230229600000001, "cost_usd": 4.230229600000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 571861, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 175, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 20204626, "tokens_cache_write": 1045048, "tokens_input": 1121, "tokens_output": 23710, "tool_calls": 147, "tool_calls_by_class": { "cc_builtin": 137, "harness_builtin": 2, "vendor_mcp": 8 }, "tool_result_tokens": 80108, "tool_time_ms": 7557, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7839581000000004, "cost_usd": 3.7839581000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 627051, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 190, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17826811, "tokens_cache_write": 932438, "tokens_input": 1191, "tokens_output": 27042, "tool_calls": 161, "tool_calls_by_class": { "cc_builtin": 138, "harness_builtin": 1, "vendor_mcp": 22 }, "tool_result_tokens": 65416, "tool_time_ms": 11170, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.9756203999999995, "cost_usd": 4.9756203999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 718114, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 211, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 23740414, "tokens_cache_write": 1229984, "tokens_input": 1306, "tokens_output": 28061, "tool_calls": 181, "tool_calls_by_class": { "cc_builtin": 157, "harness_builtin": 2, "vendor_mcp": 22 }, "tool_result_tokens": 83497, "tool_time_ms": 9849, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0150940999999998, "cost_usd": 3.0150940999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 549735, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 175, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15421111, "tokens_cache_write": 677043, "tokens_input": 1122, "tokens_output": 23555, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 137, "vendor_mcp": 11 }, "tool_result_tokens": 56466, "tool_time_ms": 8584, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6573670000000003, "cost_usd": 3.6573670000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 630634, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 179, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19833860, "tokens_cache_write": 771085, "tokens_input": 1121, "tokens_output": 26138, "tool_calls": 149, "tool_calls_by_class": { "cc_builtin": 112, "harness_builtin": 2, "vendor_mcp": 35 }, "tool_result_tokens": 91178, "tool_time_ms": 8819, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.3887297, "cost_usd": 5.3887297, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 783738, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 200, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 24708664, "tokens_cache_write": 1356176, "tokens_input": 1255, "tokens_output": 32722, "tool_calls": 170, "tool_calls_by_class": { "cc_builtin": 145, "harness_builtin": 2, "vendor_mcp": 23 }, "tool_result_tokens": 99623, "tool_time_ms": 8717, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.759007899999999, "cost_usd": 4.759007899999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 610578, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 169, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19842899, "tokens_cache_write": 1318906, "tokens_input": 1111, "tokens_output": 27159, "tool_calls": 173, "tool_calls_by_class": { "cc_builtin": 162, "harness_builtin": 1, "vendor_mcp": 10 }, "tool_result_tokens": 83852, "tool_time_ms": 7933, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9482292, "cost_usd": 2.9482292, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 994981, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 305, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14977962, "tokens_cache_write": 596753, "tokens_input": 1852, "tokens_output": 51015, "tool_calls": 357, "tool_calls_by_class": { "cc_builtin": 122, "harness_builtin": 4, "vendor_mcp": 231 }, "tool_result_tokens": 344850, "tool_time_ms": 38204, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0052348, "cost_usd": 3.0052348, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1004097, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 310, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14980708, "tokens_cache_write": 619638, "tokens_input": 5028, "tokens_output": 52572, "tool_calls": 383, "tool_calls_by_class": { "cc_builtin": 159, "harness_builtin": 1, "vendor_mcp": 223 }, "tool_result_tokens": 327997, "tool_time_ms": 43855, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0013039000000004, "cost_usd": 3.0013039000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1126838, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 346, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16206954, "tokens_cache_write": 558704, "tokens_input": 2137, "tokens_output": 57486, "tool_calls": 424, "tool_calls_by_class": { "cc_builtin": 202, "harness_builtin": 2, "subagent": 1, "vendor_mcp": 219 }, "tool_result_tokens": 284123, "tool_time_ms": 40240, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8689722, "cost_usd": 2.8689722, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1004846, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 326, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15335402, "tokens_cache_write": 536925, "tokens_input": 1972, "tokens_output": 51922, "tool_calls": 369, "tool_calls_by_class": { "cc_builtin": 156, "harness_builtin": 2, "vendor_mcp": 211 }, "tool_result_tokens": 307782, "tool_time_ms": 41057, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8528922000000003, "cost_usd": 2.8528922000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 992244, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 336, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15655962, "tokens_cache_write": 512190, "tokens_input": 2021, "tokens_output": 52179, "tool_calls": 390, "tool_calls_by_class": { "cc_builtin": 164, "harness_builtin": 2, "vendor_mcp": 224 }, "tool_result_tokens": 274415, "tool_time_ms": 40980, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5205056000000003, "cost_usd": 2.5205056000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 908287, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 282, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12893466, "tokens_cache_write": 494986, "tokens_input": 1727, "tokens_output": 47892, "tool_calls": 346, "tool_calls_by_class": { "cc_builtin": 116, "harness_builtin": 4, "vendor_mcp": 226 }, "tool_result_tokens": 267248, "tool_time_ms": 42528, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1045472, "cost_usd": 3.1045472, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1125366, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 372, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17882122, "tokens_cache_write": 513953, "tokens_input": 2234, "tokens_output": 57239, "tool_calls": 423, "tool_calls_by_class": { "cc_builtin": 176, "harness_builtin": 3, "vendor_mcp": 244 }, "tool_result_tokens": 273873, "tool_time_ms": 46601, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6358067000000003, "cost_usd": 2.6358067000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 933555, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 297, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14081747, "tokens_cache_write": 491127, "tokens_input": 1813, "tokens_output": 48713, "tool_calls": 359, "tool_calls_by_class": { "cc_builtin": 143, "harness_builtin": 3, "vendor_mcp": 213 }, "tool_result_tokens": 262496, "tool_time_ms": 40205, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6955156, "cost_usd": 2.6955156, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 947604, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 293, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13773476, "tokens_cache_write": 536944, "tokens_input": 1790, "tokens_output": 48498, "tool_calls": 351, "tool_calls_by_class": { "cc_builtin": 152, "vendor_mcp": 199 }, "tool_result_tokens": 300121, "tool_time_ms": 40066, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4062222, "cost_usd": 3.4062222, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1193045, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 385, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19349572, "tokens_cache_write": 578521, "tokens_input": 2313, "tokens_output": 62382, "tool_calls": 443, "tool_calls_by_class": { "cc_builtin": 195, "harness_builtin": 6, "vendor_mcp": 242 }, "tool_result_tokens": 315532, "tool_time_ms": 56034, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.931374, "cost_usd": 2.931374, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 906823, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 313, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13625300, "tokens_cache_write": 663642, "tokens_input": 1915, "tokens_output": 47929, "tool_calls": 343, "tool_calls_by_class": { "cc_builtin": 126, "harness_builtin": 2, "vendor_mcp": 215 }, "tool_result_tokens": 317612, "tool_time_ms": 44883, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.012918800000001, "cost_usd": 3.012918800000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1010391, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 346, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15076608, "tokens_cache_write": 618760, "tokens_input": 2108, "tokens_output": 53126, "tool_calls": 403, "tool_calls_by_class": { "cc_builtin": 173, "harness_builtin": 3, "vendor_mcp": 227 }, "tool_result_tokens": 301510, "tool_time_ms": 50966, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.936308900000001, "cost_usd": 2.936308900000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1012807, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 338, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14095889, "tokens_cache_write": 631229, "tokens_input": 2052, "tokens_output": 52442, "tool_calls": 378, "tool_calls_by_class": { "cc_builtin": 161, "harness_builtin": 2, "vendor_mcp": 215 }, "tool_result_tokens": 313237, "tool_time_ms": 52492, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7761480999999995, "cost_usd": 2.7761480999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 892455, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 299, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13088631, "tokens_cache_write": 614426, "tokens_input": 3278, "tokens_output": 47031, "tool_calls": 347, "tool_calls_by_class": { "cc_builtin": 143, "vendor_mcp": 204 }, "tool_result_tokens": 298355, "tool_time_ms": 42773, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9594721999999996, "cost_usd": 2.9594721999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 957645, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 343, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15101592, "tokens_cache_write": 596316, "tokens_input": 2111, "tokens_output": 50914, "tool_calls": 390, "tool_calls_by_class": { "cc_builtin": 190, "harness_builtin": 3, "vendor_mcp": 197 }, "tool_result_tokens": 302457, "tool_time_ms": 50600, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0082682999999997, "cost_usd": 3.0082682999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 995884, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 342, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15535433, "tokens_cache_write": 590897, "tokens_input": 2116, "tokens_output": 54163, "tool_calls": 420, "tool_calls_by_class": { "cc_builtin": 179, "harness_builtin": 4, "vendor_mcp": 237 }, "tool_result_tokens": 330374, "tool_time_ms": 53300, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9105910999999995, "cost_usd": 2.9105910999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 998209, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 353, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15221001, "tokens_cache_write": 562053, "tokens_input": 2140, "tokens_output": 52449, "tool_calls": 390, "tool_calls_by_class": { "cc_builtin": 172, "vendor_mcp": 218 }, "tool_result_tokens": 321015, "tool_time_ms": 52836, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1290359999999997, "cost_usd": 3.1290359999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1037705, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 365, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16681910, "tokens_cache_write": 587903, "tokens_input": 2204, "tokens_output": 56567, "tool_calls": 407, "tool_calls_by_class": { "cc_builtin": 174, "harness_builtin": 1, "vendor_mcp": 232 }, "tool_result_tokens": 342825, "tool_time_ms": 54739, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6527195, "cost_usd": 2.6527195, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 953269, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 324, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14121745, "tokens_cache_write": 498099, "tokens_input": 1967, "tokens_output": 48476, "tool_calls": 354, "tool_calls_by_class": { "cc_builtin": 153, "harness_builtin": 3, "vendor_mcp": 198 }, "tool_result_tokens": 288478, "tool_time_ms": 50498, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.611979, "cost_usd": 2.611979, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 860285, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 296, "model": "claude-haiku-4-5", "path": "runs/cbm/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12707120, "tokens_cache_write": 552167, "tokens_input": 1823, "tokens_output": 47022, "tool_calls": 347, "tool_calls_by_class": { "cc_builtin": 119, "harness_builtin": 3, "vendor_mcp": 225 }, "tool_result_tokens": 321956, "tool_time_ms": 53955, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.033888, "cost_usd": 5.033888, "denominator": 20, "duration_basis": "run", "duration_ms": 539802, "exam": "HumanExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3582861, "tokens_cache_write": 286660, "tokens_input": 4971, "tokens_output": 18831, "tool_calls": 221, "tool_calls_by_class": { "cc_builtin": 199, "subagent": 8, "vendor_mcp": 14 }, "tool_result_tokens": 124072, "tool_time_ms": 43337, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2357395, "cost_usd": 5.2357395, "denominator": 20, "duration_basis": "run", "duration_ms": 576127, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4018069, "tokens_cache_write": 287943, "tokens_input": 15959, "tokens_output": 16173, "tool_calls": 212, "tool_calls_by_class": { "cc_builtin": 143, "subagent": 6, "vendor_mcp": 63 }, "tool_result_tokens": 133033, "tool_time_ms": 1152209, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.383142500000002, "cost_usd": 5.383142500000002, "denominator": 20, "duration_basis": "run", "duration_ms": 736010, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3825055, "tokens_cache_write": 323965, "tokens_input": 32410, "tokens_output": 15045, "tool_calls": 210, "tool_calls_by_class": { "cc_builtin": 174, "subagent": 6, "vendor_mcp": 30 }, "tool_result_tokens": 162372, "tool_time_ms": 647294, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.160284, "cost_usd": 6.160284, "denominator": 20, "duration_basis": "run", "duration_ms": 540338, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 130, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5476838, "tokens_cache_write": 271811, "tokens_input": 166, "tokens_output": 28778, "tool_calls": 238, "tool_calls_by_class": { "cc_builtin": 203, "subagent": 9, "vendor_mcp": 26 }, "tool_result_tokens": 114936, "tool_time_ms": 27481, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.861353249999998, "cost_usd": 7.861353249999998, "denominator": 20, "duration_basis": "run", "duration_ms": 551439, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 160, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7827509, "tokens_cache_write": 337234, "tokens_input": 4588, "tokens_output": 29997, "tool_calls": 319, "tool_calls_by_class": { "cc_builtin": 285, "subagent": 10, "vendor_mcp": 24 }, "tool_result_tokens": 171709, "tool_time_ms": 77521, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.210036249999997, "cost_usd": 5.210036249999997, "denominator": 20, "duration_basis": "run", "duration_ms": 562179, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 111, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4350865, "tokens_cache_write": 253657, "tokens_input": 2925, "tokens_output": 20859, "tool_calls": 230, "tool_calls_by_class": { "cc_builtin": 211, "subagent": 8, "vendor_mcp": 11 }, "tool_result_tokens": 120017, "tool_time_ms": 40490, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.873653749999999, "cost_usd": 5.873653749999999, "denominator": 20, "duration_basis": "run", "duration_ms": 473757, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4887175, "tokens_cache_write": 316353, "tokens_input": 2443, "tokens_output": 20111, "tool_calls": 255, "tool_calls_by_class": { "cc_builtin": 197, "subagent": 8, "vendor_mcp": 50 }, "tool_result_tokens": 166328, "tool_time_ms": 20519, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.2033615, "cost_usd": 4.2033615, "denominator": 20, "duration_basis": "run", "duration_ms": 359532, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2766383, "tokens_cache_write": 254407, "tokens_input": 152, "tokens_output": 16993, "tool_calls": 203, "tool_calls_by_class": { "cc_builtin": 179, "subagent": 8, "vendor_mcp": 16 }, "tool_result_tokens": 124594, "tool_time_ms": 12166, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.043239249999999, "cost_usd": 6.043239249999999, "denominator": 20, "duration_basis": "run", "duration_ms": 606441, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 129, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5540876, "tokens_cache_write": 282394, "tokens_input": 163, "tokens_output": 31609, "tool_calls": 287, "tool_calls_by_class": { "cc_builtin": 276, "subagent": 8, "vendor_mcp": 3 }, "tool_result_tokens": 153727, "tool_time_ms": 26088, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.679036000000001, "cost_usd": 6.679036000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 398552, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 127, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5641652, "tokens_cache_write": 362375, "tokens_input": 8843, "tokens_output": 27300, "tool_calls": 287, "tool_calls_by_class": { "cc_builtin": 250, "subagent": 11, "vendor_mcp": 26 }, "tool_result_tokens": 203768, "tool_time_ms": 16001, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.609462749999999, "cost_usd": 4.609462749999999, "denominator": 30, "duration_basis": "run", "duration_ms": 363133, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 68, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4153543, "tokens_cache_write": 232926, "tokens_input": 76, "tokens_output": 18808, "tool_calls": 230, "tool_calls_by_class": { "cc_builtin": 226, "subagent": 3, "vendor_mcp": 1 }, "tool_result_tokens": 125735, "tool_time_ms": 8272, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.081969249999999, "cost_usd": 7.081969249999999, "denominator": 30, "duration_basis": "run", "duration_ms": 302811, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 143, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4560006, "tokens_cache_write": 455237, "tokens_input": 4055, "tokens_output": 33652, "tool_calls": 354, "tool_calls_by_class": { "cc_builtin": 330, "subagent": 18, "vendor_mcp": 6 }, "tool_result_tokens": 224292, "tool_time_ms": 10317, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.505807249999998, "cost_usd": 5.505807249999998, "denominator": 30, "duration_basis": "run", "duration_ms": 299346, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 95, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4001877, "tokens_cache_write": 332436, "tokens_input": 212, "tokens_output": 24513, "tool_calls": 275, "tool_calls_by_class": { "cc_builtin": 258, "subagent": 11, "vendor_mcp": 6 }, "tool_result_tokens": 171196, "tool_time_ms": 8598, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.620831750000003, "cost_usd": 6.620831750000003, "denominator": 30, "duration_basis": "run", "duration_ms": 415821, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 149, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6530466, "tokens_cache_write": 307854, "tokens_input": 958, "tokens_output": 31811, "tool_calls": 275, "tool_calls_by_class": { "cc_builtin": 261, "subagent": 11, "vendor_mcp": 3 }, "tool_result_tokens": 150139, "tool_time_ms": 6720, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.247497500000006, "cost_usd": 7.247497500000006, "denominator": 30, "duration_basis": "run", "duration_ms": 408313, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 161, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6817055, "tokens_cache_write": 349466, "tokens_input": 664, "tokens_output": 31772, "tool_calls": 322, "tool_calls_by_class": { "cc_builtin": 309, "subagent": 11, "vendor_mcp": 2 }, "tool_result_tokens": 175662, "tool_time_ms": 10502, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.421325749999996, "cost_usd": 9.421325749999996, "denominator": 30, "duration_basis": "run", "duration_ms": 340142, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 195, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7788684, "tokens_cache_write": 532255, "tokens_input": 2096, "tokens_output": 40478, "tool_calls": 471, "tool_calls_by_class": { "cc_builtin": 434, "subagent": 19, "vendor_mcp": 18 }, "tool_result_tokens": 284807, "tool_time_ms": 17604, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.346090249999996, "cost_usd": 8.346090249999996, "denominator": 30, "duration_basis": "run", "duration_ms": 334866, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 172, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5503253, "tokens_cache_write": 535739, "tokens_input": 2190, "tokens_output": 35550, "tool_calls": 451, "tool_calls_by_class": { "cc_builtin": 413, "subagent": 24, "vendor_mcp": 14 }, "tool_result_tokens": 285032, "tool_time_ms": 14506, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8394525, "cost_usd": 5.8394525, "denominator": 30, "duration_basis": "run", "duration_ms": 422054, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4508470, "tokens_cache_write": 335027, "tokens_input": 12080, "tokens_output": 25076, "tool_calls": 204, "tool_calls_by_class": { "cc_builtin": 198, "subagent": 6 }, "tool_result_tokens": 174022, "tool_time_ms": 91850, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.496595500000001, "cost_usd": 4.496595500000001, "denominator": 30, "duration_basis": "run", "duration_ms": 386332, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 67, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2432706, "tokens_cache_write": 318281, "tokens_input": 14087, "tokens_output": 21372, "tool_calls": 231, "tool_calls_by_class": { "cc_builtin": 220, "subagent": 7, "vendor_mcp": 4 }, "tool_result_tokens": 135060, "tool_time_ms": 199300, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.640569999999999, "cost_usd": 5.640569999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 627274, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 122, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6218705, "tokens_cache_write": 234549, "tokens_input": 6916, "tokens_output": 23631, "tool_calls": 206, "tool_calls_by_class": { "cc_builtin": 199, "subagent": 5, "vendor_mcp": 2 }, "tool_result_tokens": 122305, "tool_time_ms": 188863, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.353812499999999, "cost_usd": 6.353812499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 855649, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 132, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9357435, "tokens_cache_write": 106551, "tokens_input": 172, "tokens_output": 24349, "tool_calls": 125, "tool_calls_by_class": { "cc_builtin": 124, "vendor_mcp": 1 }, "tool_result_tokens": 31682, "tool_time_ms": 33525, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.591654500000001, "cost_usd": 7.591654500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 915769, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 138, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11897839, "tokens_cache_write": 98992, "tokens_input": 178, "tokens_output": 26077, "tool_calls": 138, "tool_calls_by_class": { "cc_builtin": 137, "vendor_mcp": 1 }, "tool_result_tokens": 46910, "tool_time_ms": 36429, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.430651999999999, "cost_usd": 5.430651999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 652677, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 109, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8350664, "tokens_cache_write": 81240, "tokens_input": 149, "tokens_output": 17687, "tool_calls": 97, "tool_calls_by_class": { "cc_builtin": 96, "vendor_mcp": 1 }, "tool_result_tokens": 40815, "tool_time_ms": 12211, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.279072000000001, "cost_usd": 7.279072000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 874876, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 140, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11621774, "tokens_cache_write": 87771, "tokens_input": 180, "tokens_output": 23583, "tool_calls": 133, "tool_calls_by_class": { "cc_builtin": 131, "vendor_mcp": 2 }, "tool_result_tokens": 39853, "tool_time_ms": 12949, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.3706395, "cost_usd": 7.3706395, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1124060, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11864249, "tokens_cache_write": 80915, "tokens_input": 198, "tokens_output": 25135, "tool_calls": 151, "tool_calls_by_class": { "cc_builtin": 150, "vendor_mcp": 1 }, "tool_result_tokens": 33217, "tool_time_ms": 16928, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.886719499999999, "cost_usd": 7.886719499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 823731, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12092229, "tokens_cache_write": 130733, "tokens_input": 165, "tokens_output": 21298, "tool_calls": 119, "tool_calls_by_class": { "cc_builtin": 118, "vendor_mcp": 1 }, "tool_result_tokens": 60173, "tool_time_ms": 30416, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5741405, "cost_usd": 5.5741405, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 665558, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8653931, "tokens_cache_write": 79962, "tokens_input": 146, "tokens_output": 17873, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 97, "vendor_mcp": 1 }, "tool_result_tokens": 44753, "tool_time_ms": 16258, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.7833685, "cost_usd": 7.7833685, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 901292, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 139, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12523627, "tokens_cache_write": 94136, "tokens_input": 359, "tokens_output": 23136, "tool_calls": 133, "tool_calls_by_class": { "cc_builtin": 132, "vendor_mcp": 1 }, "tool_result_tokens": 50770, "tool_time_ms": 24471, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.567278999999999, "cost_usd": 7.567278999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 973454, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 149, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11985708, "tokens_cache_write": 92168, "tokens_input": 189, "tokens_output": 26072, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 135, "vendor_mcp": 2 }, "tool_result_tokens": 41904, "tool_time_ms": 28025, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.619237999999999, "cost_usd": 7.619237999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 884558, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 140, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12324316, "tokens_cache_write": 87298, "tokens_input": 180, "tokens_output": 23328, "tool_calls": 139, "tool_calls_by_class": { "cc_builtin": 138, "vendor_mcp": 1 }, "tool_result_tokens": 44313, "tool_time_ms": 15086, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9453634999999996, "cost_usd": 3.9453634999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 539143, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5677177, "tokens_cache_write": 77409, "tokens_input": 167, "tokens_output": 13274, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 88 }, "tool_result_tokens": 18052, "tool_time_ms": 3465, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.021366, "cost_usd": 4.021366, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 579085, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 113, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6240872, "tokens_cache_write": 53279, "tokens_input": 173, "tokens_output": 14691, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 98 }, "tool_result_tokens": 20442, "tool_time_ms": 3705, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7237465, "cost_usd": 3.7237465, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 526566, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5767543, "tokens_cache_write": 49987, "tokens_input": 166, "tokens_output": 13571, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 90 }, "tool_result_tokens": 19192, "tool_time_ms": 3884, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6164639999999997, "cost_usd": 3.6164639999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 588973, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5620548, "tokens_cache_write": 47367, "tokens_input": 164, "tokens_output": 13268, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 89 }, "tool_result_tokens": 16842, "tool_time_ms": 3412, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5275335000000005, "cost_usd": 3.5275335000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 521731, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 105, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5477447, "tokens_cache_write": 45176, "tokens_input": 165, "tokens_output": 13449, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 90 }, "tool_result_tokens": 15241, "tool_time_ms": 3720, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6884775, "cost_usd": 3.6884775, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 551882, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5741765, "tokens_cache_write": 49071, "tokens_input": 167, "tokens_output": 13042, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 88 }, "tool_result_tokens": 18647, "tool_time_ms": 3644, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5044205, "cost_usd": 3.5044205, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 488636, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5363861, "tokens_cache_write": 50742, "tokens_input": 159, "tokens_output": 12571, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 84 }, "tool_result_tokens": 19998, "tool_time_ms": 3496, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.642919, "cost_usd": 3.642919, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 526876, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 109, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5808828, "tokens_cache_write": 42336, "tokens_input": 169, "tokens_output": 12572, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 86 }, "tool_result_tokens": 19021, "tool_time_ms": 3092, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4738664999999997, "cost_usd": 3.4738664999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 486718, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5416643, "tokens_cache_write": 46616, "tokens_input": 162, "tokens_output": 11943, "tool_calls": 80, "tool_calls_by_class": { "cc_builtin": 80 }, "tool_result_tokens": 17588, "tool_time_ms": 3267, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3802935000000005, "cost_usd": 3.3802935000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 501857, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5237167, "tokens_cache_write": 45499, "tokens_input": 159, "tokens_output": 12237, "tool_calls": 79, "tool_calls_by_class": { "cc_builtin": 79 }, "tool_result_tokens": 16492, "tool_time_ms": 3343, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.588298, "cost_usd": 6.588298, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1130302, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 168, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6490896, "tokens_cache_write": 245903, "tokens_input": 499, "tokens_output": 35253, "tool_calls": 260, "tool_calls_by_class": { "cc_builtin": 232, "vendor_mcp": 28 }, "tool_result_tokens": 97790, "tool_time_ms": 12969, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.512395249999999, "cost_usd": 7.512395249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1292256, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 171, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6325183, "tokens_cache_write": 368561, "tokens_input": 1276, "tokens_output": 38466, "tool_calls": 280, "tool_calls_by_class": { "cc_builtin": 246, "subagent": 3, "vendor_mcp": 31 }, "tool_result_tokens": 143957, "tool_time_ms": 10645, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.688038, "cost_usd": 6.688038, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1182791, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 163, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6169411, "tokens_cache_write": 290262, "tokens_input": 205, "tokens_output": 37078, "tool_calls": 256, "tool_calls_by_class": { "cc_builtin": 219, "subagent": 1, "vendor_mcp": 36 }, "tool_result_tokens": 133675, "tool_time_ms": 29284, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.337607, "cost_usd": 9.337607, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1515338, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 209, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8042974, "tokens_cache_write": 443469, "tokens_input": 306, "tokens_output": 45303, "tool_calls": 393, "tool_calls_by_class": { "cc_builtin": 357, "subagent": 3, "vendor_mcp": 33 }, "tool_result_tokens": 195566, "tool_time_ms": 31315, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.928595000000002, "cost_usd": 6.928595000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1028902, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 161, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6022720, "tokens_cache_write": 315873, "tokens_input": 344, "tokens_output": 36003, "tool_calls": 294, "tool_calls_by_class": { "cc_builtin": 268, "subagent": 2, "vendor_mcp": 24 }, "tool_result_tokens": 135661, "tool_time_ms": 20919, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.6060965, "cost_usd": 6.6060965, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1257198, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 170, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6479443, "tokens_cache_write": 251397, "tokens_input": 212, "tokens_output": 39622, "tool_calls": 258, "tool_calls_by_class": { "cc_builtin": 223, "subagent": 1, "vendor_mcp": 34 }, "tool_result_tokens": 105602, "tool_time_ms": 11166, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.960916, "cost_usd": 6.960916, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1329453, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 171, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6643087, "tokens_cache_write": 277501, "tokens_input": 7127, "tokens_output": 37780, "tool_calls": 273, "tool_calls_by_class": { "cc_builtin": 243, "subagent": 1, "vendor_mcp": 29 }, "tool_result_tokens": 128398, "tool_time_ms": 21348, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.150099, "cost_usd": 8.150099, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1407486, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 184, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7331908, "tokens_cache_write": 379983, "tokens_input": 228, "tokens_output": 44561, "tool_calls": 307, "tool_calls_by_class": { "cc_builtin": 264, "subagent": 2, "vendor_mcp": 41 }, "tool_result_tokens": 182511, "tool_time_ms": 10515, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.40803325, "cost_usd": 7.40803325, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1288832, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 167, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5980189, "tokens_cache_write": 400544, "tokens_input": 10990, "tokens_output": 35374, "tool_calls": 308, "tool_calls_by_class": { "cc_builtin": 280, "subagent": 4, "vendor_mcp": 24 }, "tool_result_tokens": 168098, "tool_time_ms": 10964, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.164670000000001, "cost_usd": 7.164670000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1354544, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 179, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6959780, "tokens_cache_write": 264100, "tokens_input": 766, "tokens_output": 39970, "tool_calls": 276, "tool_calls_by_class": { "cc_builtin": 244, "subagent": 1, "vendor_mcp": 31 }, "tool_result_tokens": 104702, "tool_time_ms": 25970, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.24020925, "cost_usd": 6.24020925, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1002193, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 154, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5669551, "tokens_cache_write": 280331, "tokens_input": 4896, "tokens_output": 29749, "tool_calls": 254, "tool_calls_by_class": { "cc_builtin": 228, "subagent": 1, "vendor_mcp": 25 }, "tool_result_tokens": 96044, "tool_time_ms": 12231, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.564474749999999, "cost_usd": 6.564474749999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1017420, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 155, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5504847, "tokens_cache_write": 322012, "tokens_input": 4320, "tokens_output": 31606, "tool_calls": 262, "tool_calls_by_class": { "cc_builtin": 233, "subagent": 2, "vendor_mcp": 27 }, "tool_result_tokens": 106638, "tool_time_ms": 12413, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.4786545, "cost_usd": 6.4786545, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1002769, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 151, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5352274, "tokens_cache_write": 327104, "tokens_input": 2607, "tokens_output": 30089, "tool_calls": 255, "tool_calls_by_class": { "cc_builtin": 226, "subagent": 2, "vendor_mcp": 27 }, "tool_result_tokens": 115967, "tool_time_ms": 10423, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.1739932500000005, "cost_usd": 7.1739932500000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1157548, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 178, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6339819, "tokens_cache_write": 352097, "tokens_input": 244, "tokens_output": 31489, "tool_calls": 271, "tool_calls_by_class": { "cc_builtin": 243, "subagent": 3, "vendor_mcp": 25 }, "tool_result_tokens": 116650, "tool_time_ms": 12864, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.19820725, "cost_usd": 7.19820725, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1212860, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 173, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6280302, "tokens_cache_write": 356181, "tokens_input": 4438, "tokens_output": 30912, "tool_calls": 297, "tool_calls_by_class": { "cc_builtin": 270, "subagent": 3, "vendor_mcp": 24 }, "tool_result_tokens": 139824, "tool_time_ms": 12490, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.3653597500000005, "cost_usd": 6.3653597500000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1113797, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 160, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5704832, "tokens_cache_write": 297075, "tokens_input": 5252, "tokens_output": 28981, "tool_calls": 249, "tool_calls_by_class": { "cc_builtin": 224, "subagent": 2, "vendor_mcp": 23 }, "tool_result_tokens": 116996, "tool_time_ms": 11990, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.32961525, "cost_usd": 6.32961525, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1096444, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 170, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6145298, "tokens_cache_write": 269541, "tokens_input": 6383, "tokens_output": 27638, "tool_calls": 260, "tool_calls_by_class": { "cc_builtin": 234, "subagent": 2, "vendor_mcp": 24 }, "tool_result_tokens": 96839, "tool_time_ms": 11875, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.1265455, "cost_usd": 6.1265455, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1012361, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5781781, "tokens_cache_write": 261368, "tokens_input": 4277, "tokens_output": 29195, "tool_calls": 246, "tool_calls_by_class": { "cc_builtin": 222, "subagent": 1, "vendor_mcp": 23 }, "tool_result_tokens": 108976, "tool_time_ms": 11737, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.3449335, "cost_usd": 6.3449335, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1072747, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 162, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5942162, "tokens_cache_write": 290373, "tokens_input": 226, "tokens_output": 29720, "tool_calls": 248, "tool_calls_by_class": { "cc_builtin": 222, "subagent": 2, "vendor_mcp": 24 }, "tool_result_tokens": 124950, "tool_time_ms": 13799, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.070655000000001, "cost_usd": 6.070655000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1044812, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 155, "model": "claude-opus-4-6", "path": "runs/cbm/claude-opus-4-6/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5515675, "tokens_cache_write": 281330, "tokens_input": 219, "tokens_output": 29793, "tool_calls": 259, "tool_calls_by_class": { "cc_builtin": 233, "subagent": 2, "vendor_mcp": 24 }, "tool_result_tokens": 116948, "tool_time_ms": 14033, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.021963, "cost_usd": 5.021963, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 557999, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6314516, "tokens_cache_write": 110629, "tokens_input": 163, "tokens_output": 30304, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 62, "vendor_mcp": 9 }, "tool_result_tokens": 25273, "tool_time_ms": 7200, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8548565, "cost_usd": 4.8548565, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 615676, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6461953, "tokens_cache_write": 79656, "tokens_input": 174, "tokens_output": 33058, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 58, "vendor_mcp": 14 }, "tool_result_tokens": 22978, "tool_time_ms": 4891, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7970855, "cost_usd": 4.7970855, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 589332, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 90, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6397521, "tokens_cache_write": 79648, "tokens_input": 169, "tokens_output": 32040, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 66, "vendor_mcp": 4 }, "tool_result_tokens": 22699, "tool_time_ms": 6934, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.861997, "cost_usd": 5.861997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 694746, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8059394, "tokens_cache_write": 89031, "tokens_input": 193, "tokens_output": 37641, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 78, "vendor_mcp": 5 }, "tool_result_tokens": 24758, "tool_time_ms": 8361, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.702016, "cost_usd": 4.702016, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 574616, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6346802, "tokens_cache_write": 77550, "tokens_input": 168, "tokens_output": 30091, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 69, "vendor_mcp": 1 }, "tool_result_tokens": 22786, "tool_time_ms": 7781, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.064521000000001, "cost_usd": 5.064521000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 628999, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6573832, "tokens_cache_write": 89184, "tokens_input": 153, "tokens_output": 35400, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 9 }, "tool_result_tokens": 28929, "tool_time_ms": 8162, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.559938, "cost_usd": 5.559938, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 639539, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7777246, "tokens_cache_write": 85652, "tokens_input": 184, "tokens_output": 32555, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 76, "vendor_mcp": 1 }, "tool_result_tokens": 29483, "tool_time_ms": 9312, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.08419, "cost_usd": 5.08419, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 531416, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 76, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6558360, "tokens_cache_write": 104215, "tokens_input": 142, "tokens_output": 30486, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 60, "vendor_mcp": 3 }, "tool_result_tokens": 38888, "tool_time_ms": 7774, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2315685, "cost_usd": 5.2315685, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 659743, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 90, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6847067, "tokens_cache_write": 86506, "tokens_input": 170, "tokens_output": 37685, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 58, "vendor_mcp": 12 }, "tool_result_tokens": 24385, "tool_time_ms": 8342, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.143277499999998, "cost_usd": 5.143277499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 622807, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 90, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6800535, "tokens_cache_write": 87888, "tokens_input": 2486, "tokens_output": 34068, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 62, "vendor_mcp": 11 }, "tool_result_tokens": 26243, "tool_time_ms": 6146, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2336654999999994, "cost_usd": 3.2336654999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 342827, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4126861, "tokens_cache_write": 74472, "tokens_input": 158, "tokens_output": 16989, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 52, "vendor_mcp": 1 }, "tool_result_tokens": 10235, "tool_time_ms": 3362, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.498532, "cost_usd": 3.498532, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 395269, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4953744, "tokens_cache_write": 51159, "tokens_input": 179, "tokens_output": 20367, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 11911, "tool_time_ms": 3946, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2872689999999998, "cost_usd": 3.2872689999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 380969, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4656708, "tokens_cache_write": 49270, "tokens_input": 168, "tokens_output": 18615, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 57, "vendor_mcp": 1 }, "tool_result_tokens": 11897, "tool_time_ms": 3983, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.396573, "cost_usd": 3.396573, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 391437, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4769066, "tokens_cache_write": 52716, "tokens_input": 166, "tokens_output": 19362, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 13435, "tool_time_ms": 3631, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4402659999999994, "cost_usd": 3.4402659999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 354216, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 85, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4881282, "tokens_cache_write": 55599, "tokens_input": 162, "tokens_output": 17713, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 54, "vendor_mcp": 1 }, "tool_result_tokens": 15834, "tool_time_ms": 4027, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2107589999999995, "cost_usd": 3.2107589999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 392269, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4512888, "tokens_cache_write": 49274, "tokens_input": 160, "tokens_output": 18431, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 54, "vendor_mcp": 1 }, "tool_result_tokens": 12271, "tool_time_ms": 3613, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5306645000000008, "cost_usd": 3.5306645000000008, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 409131, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5051089, "tokens_cache_write": 50816, "tokens_input": 177, "tokens_output": 19843, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 62, "vendor_mcp": 1 }, "tool_result_tokens": 14641, "tool_time_ms": 4095, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2296989999999997, "cost_usd": 3.2296989999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 452251, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4580928, "tokens_cache_write": 48193, "tokens_input": 166, "tokens_output": 18259, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 1 }, "tool_result_tokens": 11623, "tool_time_ms": 3409, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8899825, "cost_usd": 2.8899825, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 339314, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4131735, "tokens_cache_write": 43016, "tokens_input": 156, "tokens_output": 15727, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 10227, "tool_time_ms": 3042, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8575645000000005, "cost_usd": 2.8575645000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 334090, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4120159, "tokens_cache_write": 40665, "tokens_input": 152, "tokens_output": 15603, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 50, "vendor_mcp": 1 }, "tool_result_tokens": 11607, "tool_time_ms": 3006, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.4245565, "cost_usd": 0.4245565, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 70672, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 385553, "tokens_cache_write": 16292, "tokens_input": 22, "tokens_output": 2750, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 2 }, "tool_result_tokens": 5406, "tool_time_ms": 3381, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.42632250000000005, "cost_usd": 0.42632250000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 63073, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 385065, "tokens_cache_write": 16453, "tokens_input": 22, "tokens_output": 2766, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 2 }, "tool_result_tokens": 5357, "tool_time_ms": 2611, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.3578705, "cost_usd": 0.3578705, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 62066, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 9, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 305421, "tokens_cache_write": 14455, "tokens_input": 17, "tokens_output": 2421, "tool_calls": 9, "tool_calls_by_class": { "cc_builtin": 7, "vendor_mcp": 2 }, "tool_result_tokens": 3179, "tool_time_ms": 5124, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.402952, "cost_usd": 0.402952, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 64608, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 421884, "tokens_cache_write": 12449, "tokens_input": 24, "tokens_output": 2696, "tool_calls": 13, "tool_calls_by_class": { "cc_builtin": 11, "vendor_mcp": 2 }, "tool_result_tokens": 5110, "tool_time_ms": 6141, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.40594349999999996, "cost_usd": 0.40594349999999996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 55356, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 345917, "tokens_cache_write": 17449, "tokens_input": 19, "tokens_output": 2336, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 2 }, "tool_result_tokens": 5061, "tool_time_ms": 5845, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.454304, "cost_usd": 0.454304, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 65856, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 418448, "tokens_cache_write": 17351, "tokens_input": 24, "tokens_output": 2858, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 2 }, "tool_result_tokens": 5094, "tool_time_ms": 3748, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.396971, "cost_usd": 0.396971, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 68671, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 9, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 307292, "tokens_cache_write": 16816, "tokens_input": 18, "tokens_output": 3003, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "vendor_mcp": 2 }, "tool_result_tokens": 4496, "tool_time_ms": 3206, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.37477499999999997, "cost_usd": 0.37477499999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 73258, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 348650, "tokens_cache_write": 13033, "tokens_input": 19, "tokens_output": 2801, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "vendor_mcp": 2 }, "tool_result_tokens": 4995, "tool_time_ms": 8257, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.3706945, "cost_usd": 0.3706945, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 78360, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 348339, "tokens_cache_write": 12835, "tokens_input": 20, "tokens_output": 2723, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "vendor_mcp": 2 }, "tool_result_tokens": 5065, "tool_time_ms": 5347, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.38877500000000004, "cost_usd": 0.38877500000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 70814, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 430400, "tokens_cache_write": 11041, "tokens_input": 23, "tokens_output": 2522, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "vendor_mcp": 2 }, "tool_result_tokens": 5736, "tool_time_ms": 5198, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.1118245, "cost_usd": 6.1118245, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 886456, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 165, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|11", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6039919, "tokens_cache_write": 187343, "tokens_input": 322, "tokens_output": 48673, "tool_calls": 161, "tool_calls_by_class": { "cc_builtin": 136, "vendor_mcp": 25 }, "tool_result_tokens": 54581, "tool_time_ms": 12541, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.885116999999999, "cost_usd": 5.885116999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 863462, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 147, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|12", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5385114, "tokens_cache_write": 197053, "tokens_input": 286, "tokens_output": 48824, "tool_calls": 146, "tool_calls_by_class": { "cc_builtin": 113, "vendor_mcp": 33 }, "tool_result_tokens": 61734, "tool_time_ms": 12294, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.9405155, "cost_usd": 5.9405155, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 903765, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 154, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|13", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5623591, "tokens_cache_write": 189767, "tokens_input": 300, "tokens_output": 49182, "tool_calls": 158, "tool_calls_by_class": { "cc_builtin": 131, "vendor_mcp": 27 }, "tool_result_tokens": 55466, "tool_time_ms": 12985, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.066716999999998, "cost_usd": 6.066716999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 859949, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|14", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5813404, "tokens_cache_write": 198901, "tokens_input": 306, "tokens_output": 46779, "tool_calls": 156, "tool_calls_by_class": { "cc_builtin": 123, "vendor_mcp": 33 }, "tool_result_tokens": 62226, "tool_time_ms": 9326, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.0495874999999995, "cost_usd": 6.0495874999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 951367, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 165, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|15", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5989765, "tokens_cache_write": 184751, "tokens_input": 319, "tokens_output": 48224, "tool_calls": 163, "tool_calls_by_class": { "cc_builtin": 133, "vendor_mcp": 30 }, "tool_result_tokens": 54372, "tool_time_ms": 12089, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.7881677499999995, "cost_usd": 6.7881677499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 854257, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 169, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep16", "query_calls": 0, "rep": 16, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|16", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6049533, "tokens_cache_write": 245162, "tokens_input": 326, "tokens_output": 53948, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 149, "subagent": 1, "vendor_mcp": 32 }, "tool_result_tokens": 75766, "tool_time_ms": 11778, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5352645, "cost_usd": 5.5352645, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 814738, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 139, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep17", "query_calls": 0, "rep": 17, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|17", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5014909, "tokens_cache_write": 189033, "tokens_input": 271, "tokens_output": 45445, "tool_calls": 153, "tool_calls_by_class": { "cc_builtin": 122, "vendor_mcp": 31 }, "tool_result_tokens": 57483, "tool_time_ms": 15994, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2660735, "cost_usd": 6.2660735, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 911221, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 161, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep18", "query_calls": 0, "rep": 18, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|18", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5922167, "tokens_cache_write": 200241, "tokens_input": 2096, "tokens_output": 51684, "tool_calls": 159, "tool_calls_by_class": { "cc_builtin": 133, "vendor_mcp": 26 }, "tool_result_tokens": 62182, "tool_time_ms": 13867, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.4867415, "cost_usd": 6.4867415, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1005664, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 175, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep19", "query_calls": 0, "rep": 19, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|19", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6437653, "tokens_cache_write": 191047, "tokens_input": 909, "tokens_output": 54116, "tool_calls": 171, "tool_calls_by_class": { "cc_builtin": 135, "vendor_mcp": 36 }, "tool_result_tokens": 56991, "tool_time_ms": 13337, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.760951999999998, "cost_usd": 5.760951999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 887590, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 146, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos-hard/rep20", "query_calls": 0, "rep": 20, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|20", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5356694, "tokens_cache_write": 185631, "tokens_input": 284, "tokens_output": 48995, "tool_calls": 150, "tool_calls_by_class": { "cc_builtin": 119, "vendor_mcp": 31 }, "tool_result_tokens": 59483, "tool_time_ms": 12877, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.9745904999999999, "cost_usd": 0.9745904999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 53265, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 344851, "tokens_cache_write": 75912, "tokens_input": 24, "tokens_output": 1717, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 2 }, "tool_result_tokens": 1581, "tool_time_ms": 1009, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.37770950000000003, "cost_usd": 0.37770950000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 38496, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 369229, "tokens_cache_write": 15766, "tokens_input": 22, "tokens_output": 1413, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 7, "vendor_mcp": 3 }, "tool_result_tokens": 1334, "tool_time_ms": 867, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.39744, "cost_usd": 0.39744, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 54511, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 404200, "tokens_cache_write": 15752, "tokens_input": 24, "tokens_output": 1508, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 1 }, "tool_result_tokens": 1344, "tool_time_ms": 941, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.40497649999999996, "cost_usd": 0.40497649999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 48373, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 13, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 437383, "tokens_cache_write": 14988, "tokens_input": 26, "tokens_output": 1451, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 7, "vendor_mcp": 3 }, "tool_result_tokens": 947, "tool_time_ms": 1053, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.331504, "cost_usd": 0.331504, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 41241, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 373908, "tokens_cache_write": 10669, "tokens_input": 22, "tokens_output": 1510, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 1 }, "tool_result_tokens": 1010, "tool_time_ms": 678, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.415204, "cost_usd": 0.415204, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 41387, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 13, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 438038, "tokens_cache_write": 15828, "tokens_input": 26, "tokens_output": 1511, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 2 }, "tool_result_tokens": 1371, "tool_time_ms": 1102, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.376231, "cost_usd": 0.376231, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 41253, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 406362, "tokens_cache_write": 13863, "tokens_input": 24, "tokens_output": 1372, "tool_calls": 9, "tool_calls_by_class": { "cc_builtin": 7, "vendor_mcp": 2 }, "tool_result_tokens": 2015, "tool_time_ms": 877, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.4529645, "cost_usd": 0.4529645, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 51700, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 13, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 433169, "tokens_cache_write": 20010, "tokens_input": 26, "tokens_output": 1446, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 8, "vendor_mcp": 2 }, "tool_result_tokens": 1100, "tool_time_ms": 1126, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.3756965, "cost_usd": 0.3756965, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 37451, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 334403, "tokens_cache_write": 17197, "tokens_input": 20, "tokens_output": 1457, "tool_calls": 9, "tool_calls_by_class": { "cc_builtin": 7, "vendor_mcp": 2 }, "tool_result_tokens": 1982, "tool_time_ms": 1937, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.31921999999999995, "cost_usd": 0.31921999999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 36667, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/cbm/claude-opus-4-8/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 376900, "tokens_cache_write": 9021, "tokens_input": 22, "tokens_output": 1618, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 1 }, "tool_result_tokens": 1715, "tool_time_ms": 763, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5958914999999996, "cost_usd": 3.5958914999999996, "denominator": 20, "duration_basis": "run", "duration_ms": 388784, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 44, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3570423, "tokens_cache_write": 122799, "tokens_input": 993, "tokens_output": 23109, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 57, "vendor_mcp": 6 }, "tool_result_tokens": 39790, "tool_time_ms": 18163, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.577925499999999, "cost_usd": 4.577925499999999, "denominator": 20, "duration_basis": "run", "duration_ms": 580851, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 51, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5229431, "tokens_cache_write": 125768, "tokens_input": 96, "tokens_output": 28202, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 13 }, "tool_result_tokens": 56540, "tool_time_ms": 159783, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3003715000000002, "cost_usd": 3.3003715000000002, "denominator": 20, "duration_basis": "run", "duration_ms": 376680, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 38, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3294353, "tokens_cache_write": 100172, "tokens_input": 70, "tokens_output": 26045, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 53, "vendor_mcp": 12 }, "tool_result_tokens": 41296, "tool_time_ms": 10336, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9975920000000005, "cost_usd": 2.9975920000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 356727, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 37, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3064424, "tokens_cache_write": 84771, "tokens_input": 69, "tokens_output": 24693, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 46, "vendor_mcp": 12 }, "tool_result_tokens": 38749, "tool_time_ms": 8627, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8891599999999986, "cost_usd": 3.8891599999999986, "denominator": 20, "duration_basis": "run", "duration_ms": 661099, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 46, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3870090, "tokens_cache_write": 126208, "tokens_input": 15327, "tokens_output": 24616, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 55, "vendor_mcp": 11 }, "tool_result_tokens": 41301, "tool_time_ms": 303389, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.890259, "cost_usd": 3.890259, "denominator": 20, "duration_basis": "run", "duration_ms": 455010, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 52, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4569648, "tokens_cache_write": 93299, "tokens_input": 99, "tokens_output": 26878, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 46, "vendor_mcp": 11 }, "tool_result_tokens": 44019, "tool_time_ms": 11720, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.297233999999999, "cost_usd": 3.297233999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 387034, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 38, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3347008, "tokens_cache_write": 96365, "tokens_input": 71, "tokens_output": 26389, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 46, "vendor_mcp": 13 }, "tool_result_tokens": 39733, "tool_time_ms": 9869, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.030578, "cost_usd": 4.030578, "denominator": 20, "duration_basis": "run", "duration_ms": 440120, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 50, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4727196, "tokens_cache_write": 105556, "tokens_input": 94, "tokens_output": 24438, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 6 }, "tool_result_tokens": 46271, "tool_time_ms": 51342, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.933, "cost_usd": 2.933, "denominator": 20, "duration_basis": "run", "duration_ms": 326682, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 44, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3333790, "tokens_cache_write": 76169, "tokens_input": 3633, "tokens_output": 19450, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 47, "vendor_mcp": 5 }, "tool_result_tokens": 31574, "tool_time_ms": 7129, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.516027999999998, "cost_usd": 4.516027999999998, "denominator": 20, "duration_basis": "run", "duration_ms": 517173, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 56, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5400806, "tokens_cache_write": 109082, "tokens_input": 106, "tokens_output": 28971, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 59, "vendor_mcp": 10 }, "tool_result_tokens": 45897, "tool_time_ms": 60018, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.916572, "cost_usd": 1.916572, "denominator": 30, "duration_basis": "run", "duration_ms": 232952, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 26, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1746044, "tokens_cache_write": 66276, "tokens_input": 48, "tokens_output": 15222, "tool_calls": 39, "tool_calls_by_class": { "cc_builtin": 37, "vendor_mcp": 2 }, "tool_result_tokens": 27743, "tool_time_ms": 2833, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9747155, "cost_usd": 1.9747155, "denominator": 30, "duration_basis": "run", "duration_ms": 265572, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 29, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1846811, "tokens_cache_write": 62544, "tokens_input": 54, "tokens_output": 17024, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 46, "vendor_mcp": 1 }, "tool_result_tokens": 22780, "tool_time_ms": 2516, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4669504999999994, "cost_usd": 2.4669504999999994, "denominator": 30, "duration_basis": "run", "duration_ms": 300347, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 33, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2355881, "tokens_cache_write": 81468, "tokens_input": 61, "tokens_output": 18961, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 55, "vendor_mcp": 4 }, "tool_result_tokens": 27607, "tool_time_ms": 3051, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2559280000000004, "cost_usd": 2.2559280000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 242892, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 29, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2119116, "tokens_cache_write": 78637, "tokens_input": 2955, "tokens_output": 15809, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 47, "vendor_mcp": 2 }, "tool_result_tokens": 32924, "tool_time_ms": 3347, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9669904999999996, "cost_usd": 1.9669904999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 237533, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 27, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1756071, "tokens_cache_write": 65508, "tokens_input": 50, "tokens_output": 17345, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 46, "vendor_mcp": 2 }, "tool_result_tokens": 24097, "tool_time_ms": 2915, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.170252, "cost_usd": 2.170252, "denominator": 30, "duration_basis": "run", "duration_ms": 265281, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 29, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1976934, "tokens_cache_write": 73232, "tokens_input": 53, "tokens_output": 17968, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 53, "vendor_mcp": 2 }, "tool_result_tokens": 28475, "tool_time_ms": 3047, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1399885, "cost_usd": 2.1399885, "denominator": 30, "duration_basis": "run", "duration_ms": 279254, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 30, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2003807, "tokens_cache_write": 69598, "tokens_input": 56, "tokens_output": 17673, "tool_calls": 46, "tool_calls_by_class": { "cc_builtin": 44, "vendor_mcp": 2 }, "tool_result_tokens": 27607, "tool_time_ms": 3491, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6295105, "cost_usd": 1.6295105, "denominator": 30, "duration_basis": "run", "duration_ms": 204338, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1437441, "tokens_cache_write": 56122, "tokens_input": 44, "tokens_output": 13974, "tool_calls": 43, "tool_calls_by_class": { "cc_builtin": 41, "vendor_mcp": 2 }, "tool_result_tokens": 20497, "tool_time_ms": 3354, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1911069999999997, "cost_usd": 2.1911069999999997, "denominator": 30, "duration_basis": "run", "duration_ms": 273168, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 33, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2084494, "tokens_cache_write": 69763, "tokens_input": 61, "tokens_output": 18037, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 55, "vendor_mcp": 2 }, "tool_result_tokens": 26342, "tool_time_ms": 3395, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6361140000000003, "cost_usd": 1.6361140000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 203588, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 22, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1372988, "tokens_cache_write": 57294, "tokens_input": 41, "tokens_output": 15059, "tool_calls": 37, "tool_calls_by_class": { "cc_builtin": 35, "vendor_mcp": 2 }, "tool_result_tokens": 21210, "tool_time_ms": 2488, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.3471934999999995, "cost_usd": 7.3471934999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 782807, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9817857, "tokens_cache_write": 146234, "tokens_input": 195, "tokens_output": 38998, "tool_calls": 92, "tool_calls_by_class": { "cc_builtin": 91, "vendor_mcp": 1 }, "tool_result_tokens": 39231, "tool_time_ms": 94433, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.516368499999997, "cost_usd": 8.516368499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 951855, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 114, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11840217, "tokens_cache_write": 135825, "tokens_input": 217, "tokens_output": 49477, "tool_calls": 99, "tool_calls_by_class": { "cc_builtin": 89, "vendor_mcp": 10 }, "tool_result_tokens": 44411, "tool_time_ms": 46329, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.5602575, "cost_usd": 6.5602575, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 732692, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 91, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8617755, "tokens_cache_write": 120710, "tokens_input": 2111, "tokens_output": 41349, "tool_calls": 100, "tool_calls_by_class": { "cc_builtin": 99, "vendor_mcp": 1 }, "tool_result_tokens": 39882, "tool_time_ms": 47755, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.9239585, "cost_usd": 8.9239585, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 877653, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 118, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12713117, "tokens_cache_write": 141025, "tokens_input": 1835, "tokens_output": 45919, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 107, "vendor_mcp": 3 }, "tool_result_tokens": 48935, "tool_time_ms": 45467, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.3475114999999995, "cost_usd": 7.3475114999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 908592, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10092383, "tokens_cache_write": 123168, "tokens_input": 193, "tokens_output": 42747, "tool_calls": 95, "tool_calls_by_class": { "cc_builtin": 92, "vendor_mcp": 3 }, "tool_result_tokens": 40548, "tool_time_ms": 133311, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.0471895, "cost_usd": 7.0471895, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 828174, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9694399, "tokens_cache_write": 114612, "tokens_input": 194, "tokens_output": 42116, "tool_calls": 108, "tool_calls_by_class": { "cc_builtin": 107, "vendor_mcp": 1 }, "tool_result_tokens": 38935, "tool_time_ms": 67024, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.220095, "cost_usd": 7.220095, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 788830, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9861730, "tokens_cache_write": 126395, "tokens_input": 246, "tokens_output": 40962, "tool_calls": 95, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 7 }, "tool_result_tokens": 47886, "tool_time_ms": 79287, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.975775, "cost_usd": 6.975775, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 832745, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 98, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9540700, "tokens_cache_write": 114327, "tokens_input": 1901, "tokens_output": 42106, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 1 }, "tool_result_tokens": 39140, "tool_time_ms": 88293, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2301519999999995, "cost_usd": 6.2301519999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 743645, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8375194, "tokens_cache_write": 111040, "tokens_input": 176, "tokens_output": 37251, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 87, "vendor_mcp": 1 }, "tool_result_tokens": 36747, "tool_time_ms": 76491, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.0416935, "cost_usd": 6.0416935, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 698625, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7985667, "tokens_cache_write": 111313, "tokens_input": 436, "tokens_output": 37342, "tool_calls": 100, "tool_calls_by_class": { "cc_builtin": 99, "vendor_mcp": 1 }, "tool_result_tokens": 36167, "tool_time_ms": 60461, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.051433500000001, "cost_usd": 5.051433500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 501356, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6405177, "tokens_cache_write": 113738, "tokens_input": 178, "tokens_output": 28423, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 77 }, "tool_result_tokens": 25592, "tool_time_ms": 4134, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.1549749999999985, "cost_usd": 5.1549749999999985, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 569093, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6993590, "tokens_cache_write": 91229, "tokens_input": 183, "tokens_output": 29799, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 85 }, "tool_result_tokens": 28436, "tool_time_ms": 4025, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.9570775000000005, "cost_usd": 4.9570775000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 525846, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6860755, "tokens_cache_write": 83073, "tokens_input": 194, "tokens_output": 27800, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 73 }, "tool_result_tokens": 25652, "tool_time_ms": 4198, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.5956435, "cost_usd": 4.5956435, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 494769, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6292347, "tokens_cache_write": 83453, "tokens_input": 178, "tokens_output": 24562, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 73 }, "tool_result_tokens": 27690, "tool_time_ms": 3932, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.215436500000001, "cost_usd": 5.215436500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 571311, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7276493, "tokens_cache_write": 83063, "tokens_input": 202, "tokens_output": 29822, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 77 }, "tool_result_tokens": 24726, "tool_time_ms": 4668, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1733565, "cost_usd": 4.1733565, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 455323, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 90, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5701633, "tokens_cache_write": 73898, "tokens_input": 272, "tokens_output": 23288, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 66 }, "tool_result_tokens": 23066, "tool_time_ms": 4105, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.202161, "cost_usd": 5.202161, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 524400, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7293892, "tokens_cache_write": 86003, "tokens_input": 192, "tokens_output": 27769, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 78 }, "tool_result_tokens": 30166, "tool_time_ms": 3899, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.0283755, "cost_usd": 5.0283755, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 533343, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6935331, "tokens_cache_write": 86445, "tokens_input": 187, "tokens_output": 27813, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 84 }, "tool_result_tokens": 26697, "tool_time_ms": 4074, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.883838, "cost_usd": 4.883838, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 490152, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6821976, "tokens_cache_write": 78704, "tokens_input": 192, "tokens_output": 27394, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 71 }, "tool_result_tokens": 26698, "tool_time_ms": 3679, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7120615, "cost_usd": 4.7120615, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 481321, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6368743, "tokens_cache_write": 85997, "tokens_input": 174, "tokens_output": 26674, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 28084, "tool_time_ms": 3914, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.233690999999999, "cost_usd": 8.233690999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1135288, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 154, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6201222, "tokens_cache_write": 341264, "tokens_input": 588, "tokens_output": 68700, "tool_calls": 214, "tool_calls_by_class": { "cc_builtin": 194, "vendor_mcp": 20 }, "tool_result_tokens": 113250, "tool_time_ms": 63735, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.7964055000000005, "cost_usd": 7.7964055000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 946670, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 157, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6246201, "tokens_cache_write": 319347, "tokens_input": 297, "tokens_output": 59134, "tool_calls": 213, "tool_calls_by_class": { "cc_builtin": 193, "vendor_mcp": 20 }, "tool_result_tokens": 103889, "tool_time_ms": 11630, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.377435, "cost_usd": 7.377435, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1089591, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 154, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6101690, "tokens_cache_write": 280143, "tokens_input": 292, "tokens_output": 60948, "tool_calls": 206, "tool_calls_by_class": { "cc_builtin": 187, "vendor_mcp": 19 }, "tool_result_tokens": 99345, "tool_time_ms": 98814, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.110512499999999, "cost_usd": 8.110512499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1057519, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 168, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6786615, "tokens_cache_write": 310794, "tokens_input": 3413, "tokens_output": 63688, "tool_calls": 232, "tool_calls_by_class": { "cc_builtin": 209, "vendor_mcp": 23 }, "tool_result_tokens": 113343, "tool_time_ms": 41751, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.521809499999999, "cost_usd": 7.521809499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1000548, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 149, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5948679, "tokens_cache_write": 298535, "tokens_input": 2489, "tokens_output": 61987, "tool_calls": 210, "tool_calls_by_class": { "cc_builtin": 190, "vendor_mcp": 20 }, "tool_result_tokens": 107960, "tool_time_ms": 41395, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.242419500000001, "cost_usd": 7.242419500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 949469, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 148, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5875929, "tokens_cache_write": 284223, "tokens_input": 580, "tokens_output": 58373, "tool_calls": 206, "tool_calls_by_class": { "cc_builtin": 183, "vendor_mcp": 23 }, "tool_result_tokens": 101900, "tool_time_ms": 13940, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.366019999999999, "cost_usd": 7.366019999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 965952, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 151, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5996380, "tokens_cache_write": 287949, "tokens_input": 288, "tokens_output": 59476, "tool_calls": 201, "tool_calls_by_class": { "cc_builtin": 177, "vendor_mcp": 24 }, "tool_result_tokens": 105061, "tool_time_ms": 10700, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.491282999999999, "cost_usd": 7.491282999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1017402, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 157, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6189406, "tokens_cache_write": 284691, "tokens_input": 1949, "tokens_output": 61597, "tool_calls": 210, "tool_calls_by_class": { "cc_builtin": 186, "vendor_mcp": 24 }, "tool_result_tokens": 101932, "tool_time_ms": 40563, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.782228, "cost_usd": 7.782228, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1052284, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 159, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6455316, "tokens_cache_write": 302539, "tokens_input": 611, "tokens_output": 61045, "tool_calls": 219, "tool_calls_by_class": { "cc_builtin": 196, "vendor_mcp": 23 }, "tool_result_tokens": 110268, "tool_time_ms": 51519, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.508866499999999, "cost_usd": 7.508866499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 976255, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 151, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6008493, "tokens_cache_write": 301431, "tokens_input": 287, "tokens_output": 59555, "tool_calls": 211, "tool_calls_by_class": { "cc_builtin": 184, "vendor_mcp": 27 }, "tool_result_tokens": 111042, "tool_time_ms": 11835, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.501333, "cost_usd": 7.501333, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 800161, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 167, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6280556, "tokens_cache_write": 313283, "tokens_input": 325, "tokens_output": 49064, "tool_calls": 203, "tool_calls_by_class": { "cc_builtin": 191, "vendor_mcp": 12 }, "tool_result_tokens": 89821, "tool_time_ms": 9481, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.830122, "cost_usd": 6.830122, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 786659, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 159, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5927274, "tokens_cache_write": 271819, "tokens_input": 309, "tokens_output": 45870, "tool_calls": 192, "tool_calls_by_class": { "cc_builtin": 178, "vendor_mcp": 14 }, "tool_result_tokens": 84637, "tool_time_ms": 10231, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.0445795, "cost_usd": 7.0445795, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 780081, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5996999, "tokens_cache_write": 290769, "tokens_input": 533, "tokens_output": 45429, "tool_calls": 189, "tool_calls_by_class": { "cc_builtin": 177, "vendor_mcp": 12 }, "tool_result_tokens": 95522, "tool_time_ms": 9738, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.7655025, "cost_usd": 6.7655025, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 757353, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5935855, "tokens_cache_write": 267637, "tokens_input": 2091, "tokens_output": 44430, "tool_calls": 193, "tool_calls_by_class": { "cc_builtin": 179, "vendor_mcp": 14 }, "tool_result_tokens": 83241, "tool_time_ms": 10012, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.915904, "cost_usd": 6.915904, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 764381, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 162, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6076888, "tokens_cache_write": 275370, "tokens_input": 317, "tokens_output": 44887, "tool_calls": 194, "tool_calls_by_class": { "cc_builtin": 180, "vendor_mcp": 14 }, "tool_result_tokens": 87521, "tool_time_ms": 9552, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.715922, "cost_usd": 6.715922, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 776631, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 155, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5838044, "tokens_cache_write": 266704, "tokens_input": 302, "tokens_output": 45134, "tool_calls": 191, "tool_calls_by_class": { "cc_builtin": 178, "vendor_mcp": 13 }, "tool_result_tokens": 84162, "tool_time_ms": 9333, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.7232645, "cost_usd": 6.7232645, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 814036, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 157, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5919189, "tokens_cache_write": 266038, "tokens_input": 1233, "tokens_output": 43885, "tool_calls": 190, "tool_calls_by_class": { "cc_builtin": 177, "vendor_mcp": 13 }, "tool_result_tokens": 85898, "tool_time_ms": 10493, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.5382175, "cost_usd": 6.5382175, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 758606, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 152, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5688705, "tokens_cache_write": 254771, "tokens_input": 4431, "tokens_output": 44960, "tool_calls": 186, "tool_calls_by_class": { "cc_builtin": 172, "vendor_mcp": 14 }, "tool_result_tokens": 83212, "tool_time_ms": 8535, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.151932, "cost_usd": 7.151932, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 818270, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 162, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6228584, "tokens_cache_write": 286920, "tokens_input": 428, "tokens_output": 46652, "tool_calls": 199, "tool_calls_by_class": { "cc_builtin": 186, "vendor_mcp": 13 }, "tool_result_tokens": 99628, "tool_time_ms": 8865, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.333676, "cost_usd": 6.333676, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 762447, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 148, "model": "claude-opus-5", "path": "runs/cbm/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5538922, "tokens_cache_write": 249264, "tokens_input": 290, "tokens_output": 42805, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 167, "vendor_mcp": 15 }, "tool_result_tokens": 85650, "tool_time_ms": 8526, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7377848, "cost_usd": 4.7377848, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 560568, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 118, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11286366, "tokens_cache_write": 139044, "tokens_input": 827, "tokens_output": 34342, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 93, "vendor_mcp": 13 }, "tool_result_tokens": 30330, "tool_time_ms": 8663, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.0527574, "cost_usd": 6.0527574, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 630402, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 146, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15887578, "tokens_cache_write": 121733, "tokens_input": 292, "tokens_output": 37014, "tool_calls": 127, "tool_calls_by_class": { "cc_builtin": 112, "vendor_mcp": 15 }, "tool_result_tokens": 42331, "tool_time_ms": 18306, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.3518571999999995, "cost_usd": 5.3518571999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 614261, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 130, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13772104, "tokens_cache_write": 115236, "tokens_input": 260, "tokens_output": 35202, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 100, "vendor_mcp": 10 }, "tool_result_tokens": 38809, "tool_time_ms": 11199, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.200880399999999, "cost_usd": 5.200880399999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 630720, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 129, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13404208, "tokens_cache_write": 116374, "tokens_input": 258, "tokens_output": 32040, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 108, "vendor_mcp": 1 }, "tool_result_tokens": 40142, "tool_time_ms": 33021, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2649619, "cost_usd": 5.2649619, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 593359, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 112, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12672863, "tokens_cache_write": 165941, "tokens_input": 224, "tokens_output": 31119, "tool_calls": 107, "tool_calls_by_class": { "cc_builtin": 106, "vendor_mcp": 1 }, "tool_result_tokens": 47028, "tool_time_ms": 37475, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.4805816, "cost_usd": 4.4805816, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 622319, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 123, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11469002, "tokens_cache_write": 97818, "tokens_input": 246, "tokens_output": 30149, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 92, "vendor_mcp": 11 }, "tool_result_tokens": 32377, "tool_time_ms": 9314, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.9627536999999995, "cost_usd": 6.9627536999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 734445, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 149, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 18258479, "tokens_cache_write": 141776, "tokens_input": 298, "tokens_output": 42244, "tool_calls": 135, "tool_calls_by_class": { "cc_builtin": 116, "vendor_mcp": 19 }, "tool_result_tokens": 54587, "tool_time_ms": 27321, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2746555, "cost_usd": 5.2746555, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 552741, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 124, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13644045, "tokens_cache_write": 116758, "tokens_input": 248, "tokens_output": 32010, "tool_calls": 107, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 19 }, "tool_result_tokens": 41673, "tool_time_ms": 10234, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.816237799999999, "cost_usd": 4.816237799999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 452119, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 100, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12030476, "tokens_cache_write": 132185, "tokens_input": 200, "tokens_output": 27559, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 82, "vendor_mcp": 6 }, "tool_result_tokens": 54317, "tool_time_ms": 8505, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6338832000000005, "cost_usd": 3.6338832000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 420116, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 93, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8891554, "tokens_cache_write": 98442, "tokens_input": 240, "tokens_output": 25003, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 83, "vendor_mcp": 7 }, "tool_result_tokens": 36161, "tool_time_ms": 7995, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.571092699999999, "cost_usd": 2.571092699999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 383532, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5964889, "tokens_cache_write": 92342, "tokens_input": 178, "tokens_output": 15136, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 14417, "tool_time_ms": 3067, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3168577, "cost_usd": 2.3168577, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 303312, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 91, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5980639, "tokens_cache_write": 50645, "tokens_input": 182, "tokens_output": 14550, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 12607, "tool_time_ms": 3678, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4103041000000003, "cost_usd": 2.4103041000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 276287, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6180027, "tokens_cache_write": 58813, "tokens_input": 176, "tokens_output": 13526, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 17531, "tool_time_ms": 3151, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5948629, "cost_usd": 2.5948629, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 318871, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6689913, "tokens_cache_write": 60265, "tokens_input": 188, "tokens_output": 15049, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 17918, "tool_time_ms": 3543, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4763224000000004, "cost_usd": 2.4763224000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 285834, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6406948, "tokens_cache_write": 56606, "tokens_input": 184, "tokens_output": 14270, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 16003, "tool_time_ms": 3587, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2366505999999995, "cost_usd": 2.2366505999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 289816, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 81, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5852392, "tokens_cache_write": 47607, "tokens_input": 162, "tokens_output": 12987, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 16056, "tool_time_ms": 3248, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6033381999999996, "cost_usd": 2.6033381999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 310119, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6659044, "tokens_cache_write": 60751, "tokens_input": 188, "tokens_output": 16037, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 17471, "tool_time_ms": 3789, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1942645000000005, "cost_usd": 2.1942645000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 361650, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5629255, "tokens_cache_write": 53400, "tokens_input": 166, "tokens_output": 12306, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 15577, "tool_time_ms": 3544, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1031386, "cost_usd": 2.1031386, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 280181, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 86, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5545402, "tokens_cache_write": 38462, "tokens_input": 172, "tokens_output": 13882, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 11358, "tool_time_ms": 3329, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5332713999999994, "cost_usd": 2.5332713999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 322094, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 99, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6656458, "tokens_cache_write": 52570, "tokens_input": 198, "tokens_output": 14688, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 69 }, "tool_result_tokens": 13469, "tool_time_ms": 3443, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8704006999999985, "cost_usd": 4.8704006999999985, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 758622, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 174, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8715369, "tokens_cache_write": 270021, "tokens_input": 348, "tokens_output": 42308, "tool_calls": 179, "tool_calls_by_class": { "cc_builtin": 163, "vendor_mcp": 16 }, "tool_result_tokens": 72986, "tool_time_ms": 9772, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5924602, "cost_usd": 5.5924602, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 867910, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 183, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9576784, "tokens_cache_write": 326086, "tokens_input": 2018, "tokens_output": 50457, "tool_calls": 180, "tool_calls_by_class": { "cc_builtin": 145, "vendor_mcp": 35 }, "tool_result_tokens": 88118, "tool_time_ms": 9530, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7417508, "cost_usd": 4.7417508, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 777754, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 171, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8716466, "tokens_cache_write": 242550, "tokens_input": 342, "tokens_output": 44699, "tool_calls": 170, "tool_calls_by_class": { "cc_builtin": 140, "vendor_mcp": 30 }, "tool_result_tokens": 76671, "tool_time_ms": 9974, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.036382599999999, "cost_usd": 5.036382599999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 795656, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 181, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9354942, "tokens_cache_write": 250764, "tokens_input": 362, "tokens_output": 48282, "tool_calls": 175, "tool_calls_by_class": { "cc_builtin": 154, "vendor_mcp": 21 }, "tool_result_tokens": 86809, "tool_time_ms": 10158, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.696068199999999, "cost_usd": 5.696068199999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1001359, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 206, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10834694, "tokens_cache_write": 266454, "tokens_input": 412, "tokens_output": 56380, "tool_calls": 200, "tool_calls_by_class": { "cc_builtin": 152, "vendor_mcp": 48 }, "tool_result_tokens": 86063, "tool_time_ms": 10565, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.6768357499999995, "cost_usd": 4.6768357499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 780399, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 162, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7979145, "tokens_cache_write": 281643, "tokens_input": 436, "tokens_output": 42532, "tool_calls": 167, "tool_calls_by_class": { "cc_builtin": 131, "subagent": 2, "vendor_mcp": 34 }, "tool_result_tokens": 89023, "tool_time_ms": 10765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.830427499999999, "cost_usd": 4.830427499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 816539, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 171, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8747145, "tokens_cache_write": 246563, "tokens_input": 342, "tokens_output": 48392, "tool_calls": 173, "tool_calls_by_class": { "cc_builtin": 149, "vendor_mcp": 24 }, "tool_result_tokens": 78996, "tool_time_ms": 10285, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.6703892, "cost_usd": 4.6703892, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 787358, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 170, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8640864, "tokens_cache_write": 237750, "tokens_input": 340, "tokens_output": 43374, "tool_calls": 169, "tool_calls_by_class": { "cc_builtin": 140, "vendor_mcp": 29 }, "tool_result_tokens": 75926, "tool_time_ms": 10298, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7480376, "cost_usd": 4.7480376, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 816621, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 173, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8850712, "tokens_cache_write": 233066, "tokens_input": 346, "tokens_output": 46226, "tool_calls": 176, "tool_calls_by_class": { "cc_builtin": 146, "vendor_mcp": 30 }, "tool_result_tokens": 71819, "tool_time_ms": 9495, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.55244935, "cost_usd": 5.55244935, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 906760, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 188, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9704247, "tokens_cache_write": 320177, "tokens_input": 376, "tokens_output": 50132, "tool_calls": 183, "tool_calls_by_class": { "cc_builtin": 147, "subagent": 1, "vendor_mcp": 35 }, "tool_result_tokens": 104709, "tool_time_ms": 9905, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.23719855, "cost_usd": 5.23719855, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 696892, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 177, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8419716, "tokens_cache_write": 370092, "tokens_input": 354, "tokens_output": 36629, "tool_calls": 171, "tool_calls_by_class": { "cc_builtin": 157, "subagent": 2, "vendor_mcp": 12 }, "tool_result_tokens": 79991, "tool_time_ms": 9471, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.2589803, "cost_usd": 4.2589803, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 619749, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 167, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8127661, "tokens_cache_write": 226215, "tokens_input": 334, "tokens_output": 30826, "tool_calls": 150, "tool_calls_by_class": { "cc_builtin": 140, "vendor_mcp": 10 }, "tool_result_tokens": 60001, "tool_time_ms": 8336, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.3553103, "cost_usd": 4.3553103, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 568744, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 161, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7870711, "tokens_cache_write": 252741, "tokens_input": 322, "tokens_output": 31779, "tool_calls": 146, "tool_calls_by_class": { "cc_builtin": 136, "vendor_mcp": 10 }, "tool_result_tokens": 74184, "tool_time_ms": 9776, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.256438700000001, "cost_usd": 4.256438700000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 587326, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 165, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7984609, "tokens_cache_write": 234581, "tokens_input": 330, "tokens_output": 30172, "tool_calls": 144, "tool_calls_by_class": { "cc_builtin": 135, "vendor_mcp": 9 }, "tool_result_tokens": 65908, "tool_time_ms": 8851, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.6218341999999994, "cost_usd": 4.6218341999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 650477, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 174, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8591094, "tokens_cache_write": 254197, "tokens_input": 348, "tokens_output": 34552, "tool_calls": 160, "tool_calls_by_class": { "cc_builtin": 149, "vendor_mcp": 11 }, "tool_result_tokens": 74733, "tool_time_ms": 8637, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.250877, "cost_usd": 4.250877, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 597759, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 165, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8023170, "tokens_cache_write": 226506, "tokens_input": 330, "tokens_output": 32260, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 140, "vendor_mcp": 8 }, "tool_result_tokens": 58616, "tool_time_ms": 9132, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.5188261999999995, "cost_usd": 4.5188261999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 587105, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 174, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8309399, "tokens_cache_write": 268674, "tokens_input": 348, "tokens_output": 32339, "tool_calls": 159, "tool_calls_by_class": { "cc_builtin": 150, "subagent": 1, "vendor_mcp": 8 }, "tool_result_tokens": 74869, "tool_time_ms": 9348, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.168770900000001, "cost_usd": 4.168770900000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 596475, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 158, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7732183, "tokens_cache_write": 231458, "tokens_input": 316, "tokens_output": 30628, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 128, "vendor_mcp": 9 }, "tool_result_tokens": 69407, "tool_time_ms": 9001, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.0899867, "cost_usd": 4.0899867, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 601601, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 159, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7743339, "tokens_cache_write": 214443, "tokens_input": 939, "tokens_output": 31834, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 140, "vendor_mcp": 8 }, "tool_result_tokens": 63319, "tool_time_ms": 8570, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cbm", "arm_label": "CC+Codebase-Memory", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.6200342, "cost_usd": 4.6200342, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 641293, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 180, "model": "claude-sonnet-5", "path": "runs/cbm/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cbm|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8922894, "tokens_cache_write": 236096, "tokens_input": 360, "tokens_output": 35034, "tool_calls": 166, "tool_calls_by_class": { "cc_builtin": 145, "vendor_mcp": 21 }, "tool_result_tokens": 75885, "tool_time_ms": 10006, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.470609, "denominator": 30, "duration_basis": "run", "duration_ms": 415466, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/HumanExam/memos/rep01", "query_calls": 10, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1274389, "tokens_cache_write": 71616, "tokens_input": 57, "tokens_output": 26009, "tool_calls": 60, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 10 }, "tool_result_tokens": 29449, "tool_time_ms": 477, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.5262070000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 432292, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/HumanExam/memos/rep02", "query_calls": 14, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1189472, "tokens_cache_write": 70450, "tokens_input": 51, "tokens_output": 29112, "tool_calls": 72, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 14 }, "tool_result_tokens": 27571, "tool_time_ms": 529, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.1585065, "denominator": 30, "duration_basis": "run", "duration_ms": 357938, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/HumanExam/memos/rep03", "query_calls": 11, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1084229, "tokens_cache_write": 74967, "tokens_input": 49, "tokens_output": 22734, "tool_calls": 68, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 11 }, "tool_result_tokens": 30343, "tool_time_ms": 496, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8458179999999995, "denominator": 30, "duration_basis": "run", "duration_ms": 410925, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/HumanExam/memos/rep04", "query_calls": 7, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 655203, "tokens_cache_write": 55950, "tokens_input": 39, "tokens_output": 29817, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 7 }, "tool_result_tokens": 20741, "tool_time_ms": 688, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.3600189999999994, "denominator": 30, "duration_basis": "run", "duration_ms": 381139, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/HumanExam/memos/rep05", "query_calls": 9, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1422964, "tokens_cache_write": 62386, "tokens_input": 63, "tokens_output": 23132, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 9 }, "tool_result_tokens": 30076, "tool_time_ms": 259, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.2472785, "denominator": 30, "duration_basis": "run", "duration_ms": 444478, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/HumanExam/memos/rep06", "query_calls": 10, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1370551, "tokens_cache_write": 47511, "tokens_input": 69, "tokens_output": 25643, "tool_calls": 78, "tool_calls_by_class": { "product_builtin": 68, "product_semantic": 10 }, "tool_result_tokens": 23962, "tool_time_ms": 656, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.03194, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 527800, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 73, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/MultiTurnExam/memos/rep01", "query_calls": 226, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2935005, "tokens_cache_write": 102038, "tokens_input": 146, "tokens_output": 16400, "tool_calls": 58, "tool_calls_by_class": { "product_builtin": 48, "product_semantic": 10 }, "tool_result_tokens": 27004, "tool_time_ms": 514, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.5425094999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 514789, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/MultiTurnExam/memos/rep02", "query_calls": 264, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3165822, "tokens_cache_write": 124927, "tokens_input": 150, "tokens_output": 16272, "tool_calls": 68, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 13 }, "tool_result_tokens": 31014, "tool_time_ms": 412, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.4595065, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 454097, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 63, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/MultiTurnExam/memos/rep03", "query_calls": 325, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2486859, "tokens_cache_write": 100503, "tokens_input": 126, "tokens_output": 14302, "tool_calls": 51, "tool_calls_by_class": { "product_builtin": 33, "product_semantic": 18 }, "tool_result_tokens": 29622, "tool_time_ms": 364, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.3900935, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 434464, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 74, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/MultiTurnExam/memos/rep04", "query_calls": 196, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2555226, "tokens_cache_write": 80447, "tokens_input": 148, "tokens_output": 16556, "tool_calls": 62, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 9 }, "tool_result_tokens": 21492, "tool_time_ms": 721, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.212363, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 476975, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 81, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/MultiTurnExam/memos/rep05", "query_calls": 304, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3092643, "tokens_cache_write": 95896, "tokens_input": 162, "tokens_output": 18388, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 48, "product_semantic": 18 }, "tool_result_tokens": 25435, "tool_time_ms": 537, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.2534545, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 606619, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": 79, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos-hard/rep01", "query_calls": 41, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 821712, "tokens_cache_write": 325445, "tokens_input": 168, "tokens_output": 27240, "tool_calls": 102, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 41 }, "tool_result_tokens": 92101, "tool_time_ms": 2249, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.659804, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 59753, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": 8, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep01", "query_calls": 4, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 28244, "tokens_cache_write": 45492, "tokens_input": 16, "tokens_output": 1255, "tool_calls": 5, "tool_calls_by_class": { "product_builtin": 1, "product_semantic": 4 }, "tool_result_tokens": 8608, "tool_time_ms": 14, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.31282299999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 53597, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": 8, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep02", "query_calls": 4, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 58938, "tokens_cache_write": 14798, "tokens_input": 16, "tokens_output": 1375, "tool_calls": 5, "tool_calls_by_class": { "product_builtin": 1, "product_semantic": 4 }, "tool_result_tokens": 8608, "tool_time_ms": 20, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.288541, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 49592, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": 8, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep03", "query_calls": 4, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 56681, "tokens_cache_write": 13780, "tokens_input": 16, "tokens_output": 1189, "tool_calls": 5, "tool_calls_by_class": { "product_builtin": 1, "product_semantic": 4 }, "tool_result_tokens": 7122, "tool_time_ms": 22, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.21896349999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 57106, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": 8, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep04", "query_calls": 4, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 61816, "tokens_cache_write": 7467, "tokens_input": 16, "tokens_output": 1273, "tool_calls": 5, "tool_calls_by_class": { "product_builtin": 1, "product_semantic": 4 }, "tool_result_tokens": 6457, "tool_time_ms": 26, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2607005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 33025, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": 6, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep05", "query_calls": 3, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 42368, "tokens_cache_write": 14177, "tokens_input": 12, "tokens_output": 820, "tool_calls": 4, "tool_calls_by_class": { "product_builtin": 1, "product_semantic": 3 }, "tool_result_tokens": 7099, "tool_time_ms": 15, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.4739235, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 695007, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 114, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep06", "query_calls": 41, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 818306, "tokens_cache_write": 352275, "tokens_input": 228, "tokens_output": 24998, "tool_calls": 115, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 41 }, "tool_result_tokens": 68949, "tool_time_ms": 2155, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.393877, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 748321, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 113, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep07", "query_calls": 47, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 792767, "tokens_cache_write": 342176, "tokens_input": 226, "tokens_output": 26433, "tool_calls": 114, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 47 }, "tool_result_tokens": 62940, "tool_time_ms": 976, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.326273499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 773729, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 114, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep08", "query_calls": 42, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 988956, "tokens_cache_write": 232135, "tokens_input": 228, "tokens_output": 28667, "tool_calls": 122, "tool_calls_by_class": { "product_builtin": 80, "product_semantic": 42 }, "tool_result_tokens": 72062, "tool_time_ms": 1889, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.1308525, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 728481, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 110, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep09", "query_calls": 34, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 992215, "tokens_cache_write": 146803, "tokens_input": 220, "tokens_output": 26028, "tool_calls": 107, "tool_calls_by_class": { "product_builtin": 73, "product_semantic": 34 }, "tool_result_tokens": 67851, "tool_time_ms": 1811, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.3233619999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 750022, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 112, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep10", "query_calls": 45, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1051497, "tokens_cache_write": 154274, "tokens_input": 224, "tokens_output": 26824, "tool_calls": 113, "tool_calls_by_class": { "product_builtin": 68, "product_semantic": 45 }, "tool_result_tokens": 76584, "tool_time_ms": 1165, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.780576, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 751771, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 113, "model": "claude-fable-5", "path": "runs/cg-ask-fz-009/claude-fable-5/ZeroShotExam/memos/rep11", "query_calls": 41, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1044591, "tokens_cache_write": 113482, "tokens_input": 226, "tokens_output": 26304, "tool_calls": 110, "tool_calls_by_class": { "product_builtin": 69, "product_semantic": 41 }, "tool_result_tokens": 66806, "tool_time_ms": 1085, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5718838, "denominator": 20, "duration_basis": "run", "duration_ms": 262675, "exam": "HumanExam", "exam_epoch": 4, "fails": 7, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep01", "query_calls": 50, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 13, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3923333, "tokens_cache_write": 103826, "tokens_input": 388, "tokens_output": 9876, "tool_calls": 95, "tool_calls_by_class": { "product_builtin": 45, "product_semantic": 50 }, "tool_result_tokens": 61650, "tool_time_ms": 1060, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3272377, "denominator": 20, "duration_basis": "run", "duration_ms": 183975, "exam": "HumanExam", "exam_epoch": 4, "fails": 12, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep02", "query_calls": 24, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 8, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1964697, "tokens_cache_write": 75584, "tokens_input": 238, "tokens_output": 7210, "tool_calls": 79, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 24 }, "tool_result_tokens": 48705, "tool_time_ms": 1302, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.37750745, "denominator": 20, "duration_basis": "run", "duration_ms": 221840, "exam": "HumanExam", "exam_epoch": 4, "fails": 10, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep03", "query_calls": 35, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 10, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2304647, "tokens_cache_write": 85071, "tokens_input": 244, "tokens_output": 8092, "tool_calls": 77, "tool_calls_by_class": { "product_builtin": 42, "product_semantic": 35 }, "tool_result_tokens": 56445, "tool_time_ms": 284, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.36241575, "denominator": 20, "duration_basis": "run", "duration_ms": 195831, "exam": "HumanExam", "exam_epoch": 4, "fails": 13, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep04", "query_calls": 53, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 7, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1930775, "tokens_cache_write": 98181, "tokens_input": 217, "tokens_output": 9279, "tool_calls": 97, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 53 }, "tool_result_tokens": 64243, "tool_time_ms": 405, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3800401, "denominator": 20, "duration_basis": "run", "duration_ms": 210090, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep05", "query_calls": 37, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2146106, "tokens_cache_write": 91734, "tokens_input": 247, "tokens_output": 10103, "tool_calls": 97, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 37 }, "tool_result_tokens": 57052, "tool_time_ms": 948, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3931356, "denominator": 20, "duration_basis": "run", "duration_ms": 192857, "exam": "HumanExam", "exam_epoch": 4, "fails": 9, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep06", "query_calls": 34, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 11, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2428701, "tokens_cache_write": 90218, "tokens_input": 263, "tokens_output": 7446, "tool_calls": 76, "tool_calls_by_class": { "product_builtin": 42, "product_semantic": 34 }, "tool_result_tokens": 59257, "tool_time_ms": 425, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.27467300000000006, "denominator": 20, "duration_basis": "run", "duration_ms": 123833, "exam": "HumanExam", "exam_epoch": 4, "fails": 10, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep07", "query_calls": 14, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 10, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1125020, "tokens_cache_write": 104028, "tokens_input": 141, "tokens_output": 6399, "tool_calls": 51, "tool_calls_by_class": { "product_builtin": 37, "product_semantic": 14 }, "tool_result_tokens": 68316, "tool_time_ms": 1046, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.390156, "denominator": 20, "duration_basis": "run", "duration_ms": 188973, "exam": "HumanExam", "exam_epoch": 4, "fails": 9, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep08", "query_calls": 26, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 11, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2393795, "tokens_cache_write": 89430, "tokens_input": 264, "tokens_output": 7745, "tool_calls": 79, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 26 }, "tool_result_tokens": 58588, "tool_time_ms": 994, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.47211610000000004, "denominator": 20, "duration_basis": "run", "duration_ms": 227968, "exam": "HumanExam", "exam_epoch": 4, "fails": 10, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep09", "query_calls": 47, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 10, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3034741, "tokens_cache_write": 93544, "tokens_input": 312, "tokens_output": 10280, "tool_calls": 100, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 47 }, "tool_result_tokens": 59343, "tool_time_ms": 887, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.38263835, "denominator": 20, "duration_basis": "run", "duration_ms": 182450, "exam": "HumanExam", "exam_epoch": 4, "fails": 8, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep10", "query_calls": 32, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 12, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2391821, "tokens_cache_write": 80297, "tokens_input": 275, "tokens_output": 8562, "tool_calls": 82, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 32 }, "tool_result_tokens": 52011, "tool_time_ms": 1325, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.27793, "denominator": 20, "duration_basis": "run", "duration_ms": 134598, "exam": "HumanExam", "exam_epoch": 4, "fails": 12, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos-hard/rep11", "query_calls": 17, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|ffc82068051276c8ecade67885ff45af|11", "score": 8, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1353360, "tokens_cache_write": 92476, "tokens_input": 159, "tokens_output": 5368, "tool_calls": 50, "tool_calls_by_class": { "product_builtin": 33, "product_semantic": 17 }, "tool_result_tokens": 57122, "tool_time_ms": 931, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.41427559999999997, "denominator": 30, "duration_basis": "run", "duration_ms": 183986, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep01", "query_calls": 25, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2308856, "tokens_cache_write": 115324, "tokens_input": 230, "tokens_output": 7801, "tool_calls": 89, "tool_calls_by_class": { "product_builtin": 64, "product_semantic": 25 }, "tool_result_tokens": 65932, "tool_time_ms": 310, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.32979575, "denominator": 30, "duration_basis": "run", "duration_ms": 162431, "exam": "HumanExam", "exam_epoch": 4, "fails": 7, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep02", "query_calls": 6, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 23, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1748890, "tokens_cache_write": 96151, "tokens_input": 218, "tokens_output": 6900, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 6 }, "tool_result_tokens": 54942, "tool_time_ms": 137, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3582925, "denominator": 30, "duration_basis": "run", "duration_ms": 177871, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep03", "query_calls": 15, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 25, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2125065, "tokens_cache_write": 86880, "tokens_input": 236, "tokens_output": 7390, "tool_calls": 81, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 15 }, "tool_result_tokens": 56901, "tool_time_ms": 98, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2516258, "denominator": 30, "duration_basis": "run", "duration_ms": 155126, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep04", "query_calls": 32, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1280358, "tokens_cache_write": 70228, "tokens_input": 155, "tokens_output": 7130, "tool_calls": 64, "tool_calls_by_class": { "product_builtin": 32, "product_semantic": 32 }, "tool_result_tokens": 57270, "tool_time_ms": 221, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3893771, "denominator": 30, "duration_basis": "run", "duration_ms": 213029, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep05", "query_calls": 27, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 26, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2147751, "tokens_cache_write": 99532, "tokens_input": 232, "tokens_output": 9991, "tool_calls": 106, "tool_calls_by_class": { "product_builtin": 79, "product_semantic": 27 }, "tool_result_tokens": 64281, "tool_time_ms": 323, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.39585990000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 214217, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep06", "query_calls": 4, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2415284, "tokens_cache_write": 91110, "tokens_input": 254, "tokens_output": 8038, "tool_calls": 71, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 4 }, "tool_result_tokens": 60375, "tool_time_ms": 193, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.29786865, "denominator": 30, "duration_basis": "run", "duration_ms": 140743, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep07", "query_calls": 18, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 25, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1433949, "tokens_cache_write": 96543, "tokens_input": 170, "tokens_output": 6725, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 18 }, "tool_result_tokens": 60032, "tool_time_ms": 61, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.38302105000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 229337, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep08", "query_calls": 19, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2339103, "tokens_cache_write": 87895, "tokens_input": 262, "tokens_output": 7796, "tool_calls": 77, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 19 }, "tool_result_tokens": 57931, "tool_time_ms": 145, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.42011240000000005, "denominator": 30, "duration_basis": "run", "duration_ms": 249060, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep09", "query_calls": 8, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 25, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2653994, "tokens_cache_write": 92004, "tokens_input": 288, "tokens_output": 7884, "tool_calls": 79, "tool_calls_by_class": { "product_builtin": 71, "product_semantic": 8 }, "tool_result_tokens": 60998, "tool_time_ms": 228, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.38768325, "denominator": 30, "duration_basis": "run", "duration_ms": 210722, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep10", "query_calls": 10, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 25, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2259370, "tokens_cache_write": 101973, "tokens_input": 255, "tokens_output": 6805, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 56, "product_semantic": 10 }, "tool_result_tokens": 56826, "tool_time_ms": 97, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.33656030000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 177539, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep11", "query_calls": 34, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 25, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1800178, "tokens_cache_write": 95458, "tokens_input": 205, "tokens_output": 7403, "tool_calls": 80, "tool_calls_by_class": { "product_builtin": 46, "product_semantic": 34 }, "tool_result_tokens": 57700, "tool_time_ms": 431, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.28701145, "denominator": 30, "duration_basis": "run", "duration_ms": 164534, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep12", "query_calls": 11, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|12", "score": 26, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1667907, "tokens_cache_write": 70427, "tokens_input": 217, "tokens_output": 6394, "tool_calls": 67, "tool_calls_by_class": { "product_builtin": 56, "product_semantic": 11 }, "tool_result_tokens": 46309, "tool_time_ms": 295, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.36633955, "denominator": 30, "duration_basis": "run", "duration_ms": 181432, "exam": "HumanExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep13", "query_calls": 15, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|13", "score": 24, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2065253, "tokens_cache_write": 97721, "tokens_input": 218, "tokens_output": 7489, "tool_calls": 92, "tool_calls_by_class": { "product_builtin": 77, "product_semantic": 15 }, "tool_result_tokens": 64655, "tool_time_ms": 404, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3231444, "denominator": 30, "duration_basis": "run", "duration_ms": 178989, "exam": "HumanExam", "exam_epoch": 4, "fails": 8, "family": "product", "llm_calls": null, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/HumanExam/memos/rep14", "query_calls": 25, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|14", "score": 22, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1801524, "tokens_cache_write": 85400, "tokens_input": 222, "tokens_output": 7204, "tool_calls": 79, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 25 }, "tool_result_tokens": 56478, "tool_time_ms": 91, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.4212143, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2110777, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 217, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep02", "query_calls": 710, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 20322353, "tokens_cache_write": 194184, "tokens_input": 1259, "tokens_output": 28998, "tool_calls": 232, "tool_calls_by_class": { "product_builtin": 170, "product_semantic": 62 }, "tool_result_tokens": 108040, "tool_time_ms": 10237, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.46800465, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1485359, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 203, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 1057, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 20851614, "tokens_cache_write": 191273, "tokens_input": 1177, "tokens_output": 28515, "tool_calls": 215, "tool_calls_by_class": { "product_builtin": 129, "product_semantic": 86 }, "tool_result_tokens": 109168, "tool_time_ms": 2203, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3921724, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2080727, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 207, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 915, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 20053164, "tokens_cache_write": 183392, "tokens_input": 1221, "tokens_output": 31279, "tool_calls": 247, "tool_calls_by_class": { "product_builtin": 176, "product_semantic": 71 }, "tool_result_tokens": 103172, "tool_time_ms": 4419, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8408216000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1648812, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": 176, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 668, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 14520141, "tokens_cache_write": 194026, "tokens_input": 1055, "tokens_output": 29044, "tool_calls": 226, "tool_calls_by_class": { "product_builtin": 169, "product_semantic": 57 }, "tool_result_tokens": 109020, "tool_time_ms": 6343, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.0758225, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1608119, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 178, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 713, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 16979085, "tokens_cache_write": 185596, "tokens_input": 1069, "tokens_output": 28970, "tool_calls": 225, "tool_calls_by_class": { "product_builtin": 159, "product_semantic": 66 }, "tool_result_tokens": 107764, "tool_time_ms": 3658, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2134734000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1591594, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 187, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 668, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 18318544, "tokens_cache_write": 191384, "tokens_input": 1094, "tokens_output": 28259, "tool_calls": 215, "tool_calls_by_class": { "product_builtin": 157, "product_semantic": 58 }, "tool_result_tokens": 106030, "tool_time_ms": 5697, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7913677000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 702261, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 144, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep11", "query_calls": 726, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|11", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 14362962, "tokens_cache_write": 187614, "tokens_input": 879, "tokens_output": 23935, "tool_calls": 212, "tool_calls_by_class": { "product_builtin": 145, "product_semantic": 67 }, "tool_result_tokens": 104455, "tool_time_ms": 3395, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8559350000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 750261, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 153, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep12", "query_calls": 384, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|12", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 15114110, "tokens_cache_write": 178968, "tokens_input": 909, "tokens_output": 23981, "tool_calls": 193, "tool_calls_by_class": { "product_builtin": 160, "product_semantic": 33 }, "tool_result_tokens": 102311, "tool_time_ms": 3834, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.65565525, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 723951, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": 186, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep13", "query_calls": 213, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|13", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 22094970, "tokens_cache_write": 253093, "tokens_input": 1072, "tokens_output": 25744, "tool_calls": 180, "tool_calls_by_class": { "product_builtin": 138, "product_semantic": 42 }, "tool_result_tokens": 107771, "tool_time_ms": 2752, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.27843485, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 800042, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 197, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos-hard/rep14", "query_calls": 284, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|14", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 19097896, "tokens_cache_write": 185029, "tokens_input": 1134, "tokens_output": 27245, "tool_calls": 200, "tool_calls_by_class": { "product_builtin": 174, "product_semantic": 26 }, "tool_result_tokens": 111429, "tool_time_ms": 4563, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.4832883000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1038711, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 143, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 12000558, "tokens_cache_write": 148434, "tokens_input": 775, "tokens_output": 19383, "tool_calls": 124, "tool_calls_by_class": { "product_builtin": 124 }, "tool_result_tokens": 82536, "tool_time_ms": 1146, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.4732376500000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1016246, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 133, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep02", "query_calls": 36, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 11670004, "tokens_cache_write": 162541, "tokens_input": 741, "tokens_output": 20464, "tool_calls": 148, "tool_calls_by_class": { "product_builtin": 146, "product_semantic": 2 }, "tool_result_tokens": 91285, "tool_time_ms": 1454, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.6864059999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1123193, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 144, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 433, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 14104595, "tokens_cache_write": 149486, "tokens_input": 774, "tokens_output": 17663, "tool_calls": 124, "tool_calls_by_class": { "product_builtin": 100, "product_semantic": 24 }, "tool_result_tokens": 91254, "tool_time_ms": 695, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.3588970500000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 961225, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 133, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 10653003, "tokens_cache_write": 151631, "tokens_input": 748, "tokens_output": 20662, "tool_calls": 156, "tool_calls_by_class": { "product_builtin": 156 }, "tool_result_tokens": 83249, "tool_time_ms": 1466, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.99031755, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 653362, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 94, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 115, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7284043, "tokens_cache_write": 133421, "tokens_input": 517, "tokens_output": 18924, "tool_calls": 122, "tool_calls_by_class": { "product_builtin": 116, "product_semantic": 6 }, "tool_result_tokens": 74500, "tool_time_ms": 700, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.96182005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1125031, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 141, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 252, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7454698, "tokens_cache_write": 108305, "tokens_input": 764, "tokens_output": 16041, "tool_calls": 130, "tool_calls_by_class": { "product_builtin": 115, "product_semantic": 15 }, "tool_result_tokens": 59157, "tool_time_ms": 1114, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.16543065, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1232772, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 142, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 227, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 9293979, "tokens_cache_write": 116051, "tokens_input": 764, "tokens_output": 18041, "tool_calls": 117, "tool_calls_by_class": { "product_builtin": 106, "product_semantic": 11 }, "tool_result_tokens": 64258, "tool_time_ms": 854, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.0049938, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1073974, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 122, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 125, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7594128, "tokens_cache_write": 124700, "tokens_input": 676, "tokens_output": 17806, "tool_calls": 141, "tool_calls_by_class": { "product_builtin": 135, "product_semantic": 6 }, "tool_result_tokens": 71418, "tool_time_ms": 1347, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.13265495, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1261978, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 143, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 383, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 8974802, "tokens_cache_write": 110595, "tokens_input": 781, "tokens_output": 19230, "tool_calls": 135, "tool_calls_by_class": { "product_builtin": 113, "product_semantic": 22 }, "tool_result_tokens": 63853, "tool_time_ms": 707, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1927863500000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1188103, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 127, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 120, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 9090401, "tokens_cache_write": 147089, "tokens_input": 710, "tokens_output": 19835, "tool_calls": 156, "tool_calls_by_class": { "product_builtin": 150, "product_semantic": 6 }, "tool_result_tokens": 85162, "tool_time_ms": 1318, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.27682765, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 740084, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 247, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 127, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4253464, "tokens_cache_write": 533001, "tokens_input": 1500, "tokens_output": 36746, "tool_calls": 361, "tool_calls_by_class": { "product_builtin": 234, "product_semantic": 127 }, "tool_result_tokens": 262898, "tool_time_ms": 5824, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2331196, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 766559, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 275, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 106, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5026836, "tokens_cache_write": 425312, "tokens_input": 1641, "tokens_output": 39431, "tool_calls": 350, "tool_calls_by_class": { "product_builtin": 244, "product_semantic": 106 }, "tool_result_tokens": 257186, "tool_time_ms": 5051, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.06360485, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 615428, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 202, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 97, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3497901, "tokens_cache_write": 440351, "tokens_input": 1231, "tokens_output": 32429, "tool_calls": 311, "tool_calls_by_class": { "product_builtin": 214, "product_semantic": 97 }, "tool_result_tokens": 285088, "tool_time_ms": 5372, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.27084595, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 760915, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 256, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 114, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5021027, "tokens_cache_write": 463593, "tokens_input": 1542, "tokens_output": 37542, "tool_calls": 349, "tool_calls_by_class": { "product_builtin": 235, "product_semantic": 114 }, "tool_result_tokens": 290347, "tool_time_ms": 7950, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.20091155, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 805375, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 271, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 122, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4509293, "tokens_cache_write": 435909, "tokens_input": 1646, "tokens_output": 40690, "tool_calls": 373, "tool_calls_by_class": { "product_builtin": 251, "product_semantic": 122 }, "tool_result_tokens": 264606, "tool_time_ms": 8147, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1207651, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 682819, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 229, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 103, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4007446, "tokens_cache_write": 431202, "tokens_input": 1403, "tokens_output": 35923, "tool_calls": 350, "tool_calls_by_class": { "product_builtin": 247, "product_semantic": 103 }, "tool_result_tokens": 266423, "tool_time_ms": 7469, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.3246232, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 996441, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 7, "family": "product", "llm_calls": 292, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 133, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 13, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5302267, "tokens_cache_write": 454430, "tokens_input": 1769, "tokens_output": 44918, "tool_calls": 406, "tool_calls_by_class": { "product_builtin": 273, "product_semantic": 133 }, "tool_result_tokens": 275826, "tool_time_ms": 6250, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2639641, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 792896, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 264, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 125, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4702076, "tokens_cache_write": 465918, "tokens_input": 1639, "tokens_output": 41944, "tool_calls": 414, "tool_calls_by_class": { "product_builtin": 289, "product_semantic": 125 }, "tool_result_tokens": 274584, "tool_time_ms": 6289, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2990169, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 800670, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 274, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 147, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5259274, "tokens_cache_write": 458694, "tokens_input": 1667, "tokens_output": 39611, "tool_calls": 382, "tool_calls_by_class": { "product_builtin": 235, "product_semantic": 147 }, "tool_result_tokens": 272653, "tool_time_ms": 7429, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.4736593999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 888226, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 312, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 150, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6255064, "tokens_cache_write": 499552, "tokens_input": 1888, "tokens_output": 44365, "tool_calls": 418, "tool_calls_by_class": { "product_builtin": 268, "product_semantic": 150 }, "tool_result_tokens": 291820, "tool_time_ms": 6810, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.134428, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 597932, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 236, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 54, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2843225, "tokens_cache_write": 554238, "tokens_input": 1378, "tokens_output": 31186, "tool_calls": 305, "tool_calls_by_class": { "product_builtin": 251, "product_semantic": 54 }, "tool_result_tokens": 228353, "tool_time_ms": 3357, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.89218445, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 552511, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 223, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 62, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2572782, "tokens_cache_write": 392841, "tokens_input": 1300, "tokens_output": 28511, "tool_calls": 276, "tool_calls_by_class": { "product_builtin": 214, "product_semantic": 62 }, "tool_result_tokens": 225429, "tool_time_ms": 2999, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8885936000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 593642, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 236, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 49, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2920821, "tokens_cache_write": 353926, "tokens_input": 1379, "tokens_output": 30545, "tool_calls": 288, "tool_calls_by_class": { "product_builtin": 239, "product_semantic": 49 }, "tool_result_tokens": 219703, "tool_time_ms": 2944, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.81460825, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 546255, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 217, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 61, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2489755, "tokens_cache_write": 339675, "tokens_input": 1269, "tokens_output": 27954, "tool_calls": 274, "tool_calls_by_class": { "product_builtin": 213, "product_semantic": 61 }, "tool_result_tokens": 222096, "tool_time_ms": 3943, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.86471865, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 541203, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 198, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 53, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2326444, "tokens_cache_write": 395301, "tokens_input": 1158, "tokens_output": 27358, "tool_calls": 260, "tool_calls_by_class": { "product_builtin": 207, "product_semantic": 53 }, "tool_result_tokens": 239217, "tool_time_ms": 2548, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.99256045, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 613580, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 246, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 68, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3130882, "tokens_cache_write": 423601, "tokens_input": 1426, "tokens_output": 29709, "tool_calls": 281, "tool_calls_by_class": { "product_builtin": 213, "product_semantic": 68 }, "tool_result_tokens": 253899, "tool_time_ms": 2057, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.75889755, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 560246, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 209, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 55, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2421828, "tokens_cache_write": 301091, "tokens_input": 1226, "tokens_output": 27825, "tool_calls": 269, "tool_calls_by_class": { "product_builtin": 214, "product_semantic": 55 }, "tool_result_tokens": 190517, "tool_time_ms": 4200, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8404383, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 587597, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 223, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 47, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2531138, "tokens_cache_write": 347174, "tokens_input": 1312, "tokens_output": 30409, "tool_calls": 298, "tool_calls_by_class": { "product_builtin": 251, "product_semantic": 47 }, "tool_result_tokens": 216144, "tool_time_ms": 3577, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8609386, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 568382, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 220, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 63, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2574176, "tokens_cache_write": 364656, "tokens_input": 1286, "tokens_output": 29283, "tool_calls": 279, "tool_calls_by_class": { "product_builtin": 216, "product_semantic": 63 }, "tool_result_tokens": 221696, "tool_time_ms": 2255, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8387019, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 579568, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 218, "model": "claude-haiku-4-5", "path": "runs/cg-ask-fz-009/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 83, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2811869, "tokens_cache_write": 330320, "tokens_input": 1270, "tokens_output": 28669, "tool_calls": 271, "tool_calls_by_class": { "product_builtin": 188, "product_semantic": 83 }, "tool_result_tokens": 211064, "tool_time_ms": 2852, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.369025, "denominator": 20, "duration_basis": "run", "duration_ms": 395427, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep01", "query_calls": 61, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2983760, "tokens_cache_write": 97524, "tokens_input": 49, "tokens_output": 10695, "tool_calls": 111, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 61 }, "tool_result_tokens": 56898, "tool_time_ms": 997, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.303465, "denominator": 20, "duration_basis": "run", "duration_ms": 433515, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep02", "query_calls": 37, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3067460, "tokens_cache_write": 77296, "tokens_input": 52, "tokens_output": 11455, "tool_calls": 103, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 37 }, "tool_result_tokens": 50164, "tool_time_ms": 1113, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8331415, "denominator": 20, "duration_basis": "run", "duration_ms": 532713, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep03", "query_calls": 66, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4006773, "tokens_cache_write": 79424, "tokens_input": 71, "tokens_output": 13320, "tool_calls": 133, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 66 }, "tool_result_tokens": 49899, "tool_time_ms": 1313, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8251132500000002, "denominator": 20, "duration_basis": "run", "duration_ms": 434467, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep04", "query_calls": 41, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3986459, "tokens_cache_write": 88327, "tokens_input": 58, "tokens_output": 11182, "tool_calls": 102, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 41 }, "tool_result_tokens": 57805, "tool_time_ms": 942, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.063492, "denominator": 20, "duration_basis": "run", "duration_ms": 474126, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep05", "query_calls": 47, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4264784, "tokens_cache_write": 97760, "tokens_input": 60, "tokens_output": 12792, "tool_calls": 108, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 47 }, "tool_result_tokens": 64273, "tool_time_ms": 887, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.8283319999999996, "denominator": 20, "duration_basis": "run", "duration_ms": 654910, "exam": "HumanExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep06", "query_calls": 87, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5769564, "tokens_cache_write": 92420, "tokens_input": 85, "tokens_output": 14620, "tool_calls": 141, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 87 }, "tool_result_tokens": 55379, "tool_time_ms": 839, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.52247975, "denominator": 20, "duration_basis": "run", "duration_ms": 437925, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep07", "query_calls": 37, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3164247, "tokens_cache_write": 102797, "tokens_input": 45, "tokens_output": 11906, "tool_calls": 90, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 37 }, "tool_result_tokens": 64753, "tool_time_ms": 807, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.1262625, "denominator": 20, "duration_basis": "run", "duration_ms": 560721, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep08", "query_calls": 34, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4511000, "tokens_cache_write": 84258, "tokens_input": 70, "tokens_output": 13752, "tool_calls": 108, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 34 }, "tool_result_tokens": 53111, "tool_time_ms": 1106, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.06292425, "denominator": 20, "duration_basis": "run", "duration_ms": 350113, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep09", "query_calls": 41, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2559881, "tokens_cache_write": 84859, "tokens_input": 43, "tokens_output": 10096, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 41 }, "tool_result_tokens": 54279, "tool_time_ms": 1059, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.395704, "denominator": 20, "duration_basis": "run", "duration_ms": 535599, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos-hard/rep10", "query_calls": 60, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4779253, "tokens_cache_write": 110638, "tokens_input": 68, "tokens_output": 12570, "tool_calls": 118, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 60 }, "tool_result_tokens": 68863, "tool_time_ms": 1011, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8321619999999998, "denominator": 30, "duration_basis": "run", "duration_ms": 328466, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep01", "query_calls": 6, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1975884, "tokens_cache_write": 96060, "tokens_input": 34, "tokens_output": 9747, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 85, "product_semantic": 6 }, "tool_result_tokens": 57411, "tool_time_ms": 317, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.88472625, "denominator": 30, "duration_basis": "run", "duration_ms": 335638, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep02", "query_calls": 10, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2134700, "tokens_cache_write": 89729, "tokens_input": 39, "tokens_output": 10255, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 83, "product_semantic": 10 }, "tool_result_tokens": 50648, "tool_time_ms": 398, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.6068784999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 237833, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep03", "query_calls": 8, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1452292, "tokens_cache_write": 110154, "tokens_input": 24, "tokens_output": 7686, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 8 }, "tool_result_tokens": 68312, "tool_time_ms": 332, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.96672, "denominator": 30, "duration_basis": "run", "duration_ms": 376099, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep04", "query_calls": 7, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2135760, "tokens_cache_write": 95128, "tokens_input": 38, "tokens_output": 12164, "tool_calls": 95, "tool_calls_by_class": { "product_builtin": 88, "product_semantic": 7 }, "tool_result_tokens": 55654, "tool_time_ms": 382, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.343976, "denominator": 30, "duration_basis": "run", "duration_ms": 236810, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep05", "query_calls": 7, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1232927, "tokens_cache_write": 84070, "tokens_input": 25, "tokens_output": 8078, "tool_calls": 80, "tool_calls_by_class": { "product_builtin": 73, "product_semantic": 7 }, "tool_result_tokens": 49095, "tool_time_ms": 322, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.648591, "denominator": 30, "duration_basis": "run", "duration_ms": 275562, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep06", "query_calls": 11, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1757837, "tokens_cache_write": 87106, "tokens_input": 32, "tokens_output": 9004, "tool_calls": 85, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 11 }, "tool_result_tokens": 53075, "tool_time_ms": 255, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.60198275, "denominator": 30, "duration_basis": "run", "duration_ms": 323729, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep07", "query_calls": 9, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1814293, "tokens_cache_write": 71269, "tokens_input": 36, "tokens_output": 9969, "tool_calls": 90, "tool_calls_by_class": { "product_builtin": 81, "product_semantic": 9 }, "tool_result_tokens": 50709, "tool_time_ms": 412, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.53901925, "denominator": 30, "duration_basis": "run", "duration_ms": 288525, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep08", "query_calls": 6, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1480846, "tokens_cache_write": 91789, "tokens_input": 28, "tokens_output": 8991, "tool_calls": 82, "tool_calls_by_class": { "product_builtin": 76, "product_semantic": 6 }, "tool_result_tokens": 59761, "tool_time_ms": 238, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.81111125, "denominator": 30, "duration_basis": "run", "duration_ms": 321049, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep09", "query_calls": 31, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2088375, "tokens_cache_write": 88555, "tokens_input": 36, "tokens_output": 8531, "tool_calls": 83, "tool_calls_by_class": { "product_builtin": 52, "product_semantic": 31 }, "tool_result_tokens": 57998, "tool_time_ms": 166, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8852345000000001, "denominator": 30, "duration_basis": "run", "duration_ms": 305491, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/HumanExam/memos/rep10", "query_calls": 20, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2120099, "tokens_cache_write": 92452, "tokens_input": 37, "tokens_output": 9887, "tool_calls": 96, "tool_calls_by_class": { "product_builtin": 76, "product_semantic": 20 }, "tool_result_tokens": 52941, "tool_time_ms": 338, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.078557999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 665110, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 85, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep01", "query_calls": 608, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4522791, "tokens_cache_write": 392750, "tokens_input": 125, "tokens_output": 14474, "tool_calls": 112, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 53 }, "tool_result_tokens": 62513, "tool_time_ms": 1202, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.2974957499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 789783, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 96, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep02", "query_calls": 503, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5472919, "tokens_cache_write": 501597, "tokens_input": 136, "tokens_output": 17015, "tool_calls": 126, "tool_calls_by_class": { "product_builtin": 83, "product_semantic": 43 }, "tool_result_tokens": 55738, "tool_time_ms": 1503, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 7.216536, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 768947, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 90, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep03", "query_calls": 496, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7382122, "tokens_cache_write": 498264, "tokens_input": 130, "tokens_output": 16427, "tool_calls": 109, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 42 }, "tool_result_tokens": 96070, "tool_time_ms": 1675, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.982094, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 878100, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 105, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep04", "query_calls": 858, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6011013, "tokens_cache_write": 560914, "tokens_input": 145, "tokens_output": 18806, "tool_calls": 159, "tool_calls_by_class": { "product_builtin": 84, "product_semantic": 75 }, "tool_result_tokens": 60097, "tool_time_ms": 1621, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.449961249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1034009, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 130, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep05", "query_calls": 1278, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6969110, "tokens_cache_write": 387921, "tokens_input": 170, "tokens_output": 21602, "tool_calls": 201, "tool_calls_by_class": { "product_builtin": 84, "product_semantic": 117 }, "tool_result_tokens": 54756, "tool_time_ms": 1729, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.724591500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 751283, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 87, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep06", "query_calls": 511, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5908188, "tokens_cache_write": 381362, "tokens_input": 127, "tokens_output": 15454, "tool_calls": 110, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 43 }, "tool_result_tokens": 72844, "tool_time_ms": 1499, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.154848749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 782515, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 90, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep07", "query_calls": 623, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5347160, "tokens_cache_write": 487583, "tokens_input": 130, "tokens_output": 17329, "tool_calls": 136, "tool_calls_by_class": { "product_builtin": 82, "product_semantic": 54 }, "tool_result_tokens": 64564, "tool_time_ms": 1672, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.9800415000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 798123, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 96, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep08", "query_calls": 485, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4705673, "tokens_cache_write": 353704, "tokens_input": 136, "tokens_output": 16635, "tool_calls": 127, "tool_calls_by_class": { "product_builtin": 86, "product_semantic": 41 }, "tool_result_tokens": 42479, "tool_time_ms": 1350, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 7.27491625, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 967754, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep09", "query_calls": 285, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6516820, "tokens_cache_write": 562885, "tokens_input": 165, "tokens_output": 19906, "tool_calls": 154, "tool_calls_by_class": { "product_builtin": 129, "product_semantic": 25 }, "tool_result_tokens": 45417, "tool_time_ms": 2137, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.25452825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 765725, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 92, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos-hard/rep10", "query_calls": 503, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4316474, "tokens_cache_write": 269033, "tokens_input": 132, "tokens_output": 16567, "tool_calls": 125, "tool_calls_by_class": { "product_builtin": 83, "product_semantic": 42 }, "tool_result_tokens": 42957, "tool_time_ms": 1650, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.35499525, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 675870, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 89, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep01", "query_calls": 336, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2936588, "tokens_cache_write": 96513, "tokens_input": 149, "tokens_output": 11310, "tool_calls": 81, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 21 }, "tool_result_tokens": 30940, "tool_time_ms": 869, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.245676, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 607630, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 82, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep02", "query_calls": 123, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2246782, "tokens_cache_write": 136064, "tokens_input": 142, "tokens_output": 10847, "tool_calls": 81, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 7 }, "tool_result_tokens": 28142, "tool_time_ms": 1097, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.03513625, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 574708, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 77, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep03", "query_calls": 479, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2261640, "tokens_cache_write": 105861, "tokens_input": 137, "tokens_output": 9680, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 37, "product_semantic": 32 }, "tool_result_tokens": 34240, "tool_time_ms": 883, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.679386, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 581479, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 82, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep04", "query_calls": 405, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2530177, "tokens_cache_write": 184654, "tokens_input": 142, "tokens_output": 10380, "tool_calls": 80, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 27 }, "tool_result_tokens": 31518, "tool_time_ms": 988, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.12494775, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 596299, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 81, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep05", "query_calls": 97, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2453948, "tokens_cache_write": 103407, "tokens_input": 141, "tokens_output": 10039, "tool_calls": 68, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 5 }, "tool_result_tokens": 31560, "tool_time_ms": 856, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7885987499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 563707, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 77, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep06", "query_calls": 323, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2088265, "tokens_cache_write": 79809, "tokens_input": 137, "tokens_output": 9799, "tool_calls": 68, "tool_calls_by_class": { "product_builtin": 48, "product_semantic": 20 }, "tool_result_tokens": 26366, "tool_time_ms": 842, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.15035775, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 549738, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 76, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep07", "query_calls": 459, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2146693, "tokens_cache_write": 132473, "tokens_input": 136, "tokens_output": 9935, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 31 }, "tool_result_tokens": 28043, "tool_time_ms": 1107, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2887895, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 560580, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 76, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep08", "query_calls": 182, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2768544, "tokens_cache_write": 105726, "tokens_input": 136, "tokens_output": 9722, "tool_calls": 62, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 12 }, "tool_result_tokens": 34879, "tool_time_ms": 1076, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.391045, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 563956, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 79, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep09", "query_calls": 271, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2487775, "tokens_cache_write": 142994, "tokens_input": 139, "tokens_output": 10110, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 52, "product_semantic": 18 }, "tool_result_tokens": 34620, "tool_time_ms": 1011, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5433425, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 544362, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 76, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/MultiTurnExam/memos/rep10", "query_calls": 239, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2647700, "tokens_cache_write": 152930, "tokens_input": 136, "tokens_output": 10520, "tool_calls": 76, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 15 }, "tool_result_tokens": 33642, "tool_time_ms": 1414, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.4889977500000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 983921, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 131, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep01", "query_calls": 80, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1340323, "tokens_cache_write": 347421, "tokens_input": 171, "tokens_output": 25864, "tool_calls": 195, "tool_calls_by_class": { "product_builtin": 115, "product_semantic": 80 }, "tool_result_tokens": 132326, "tool_time_ms": 3449, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.38710675, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 884315, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 119, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep02", "query_calls": 78, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1156161, "tokens_cache_write": 191269, "tokens_input": 159, "tokens_output": 24512, "tool_calls": 186, "tool_calls_by_class": { "product_builtin": 108, "product_semantic": 78 }, "tool_result_tokens": 103072, "tool_time_ms": 2765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.31952925, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 838732, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 114, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep03", "query_calls": 78, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1124106, "tokens_cache_write": 190705, "tokens_input": 154, "tokens_output": 22592, "tool_calls": 167, "tool_calls_by_class": { "product_builtin": 89, "product_semantic": 78 }, "tool_result_tokens": 112561, "tool_time_ms": 2096, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3470587500000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1121790, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 124, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep04", "query_calls": 91, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1220890, "tokens_cache_write": 175359, "tokens_input": 164, "tokens_output": 25592, "tool_calls": 209, "tool_calls_by_class": { "product_builtin": 118, "product_semantic": 91 }, "tool_result_tokens": 105765, "tool_time_ms": 3302, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3265302500000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 939461, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 119, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep05", "query_calls": 90, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1243983, "tokens_cache_write": 174371, "tokens_input": 159, "tokens_output": 24557, "tool_calls": 193, "tool_calls_by_class": { "product_builtin": 103, "product_semantic": 90 }, "tool_result_tokens": 117272, "tool_time_ms": 2693, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6517305, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 995192, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 140, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep06", "query_calls": 130, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1677136, "tokens_cache_write": 185002, "tokens_input": 180, "tokens_output": 26240, "tool_calls": 232, "tool_calls_by_class": { "product_builtin": 102, "product_semantic": 130 }, "tool_result_tokens": 113241, "tool_time_ms": 3346, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.69601675, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1057438, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 132, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep07", "query_calls": 76, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1567726, "tokens_cache_write": 200227, "tokens_input": 172, "tokens_output": 26395, "tool_calls": 210, "tool_calls_by_class": { "product_builtin": 134, "product_semantic": 76 }, "tool_result_tokens": 123342, "tool_time_ms": 3943, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.1580369999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 884169, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 119, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep08", "query_calls": 87, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1142759, "tokens_cache_write": 162294, "tokens_input": 159, "tokens_output": 22861, "tool_calls": 178, "tool_calls_by_class": { "product_builtin": 91, "product_semantic": 87 }, "tool_result_tokens": 102020, "tool_time_ms": 3134, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2468855000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 894711, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 116, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep09", "query_calls": 84, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1202961, "tokens_cache_write": 169316, "tokens_input": 156, "tokens_output": 23456, "tool_calls": 191, "tool_calls_by_class": { "product_builtin": 107, "product_semantic": 84 }, "tool_result_tokens": 118802, "tool_time_ms": 2422, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.195955, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 892553, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 123, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos-hard/rep10", "query_calls": 96, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1283230, "tokens_cache_write": 153212, "tokens_input": 163, "tokens_output": 23838, "tool_calls": 195, "tool_calls_by_class": { "product_builtin": 99, "product_semantic": 96 }, "tool_result_tokens": 105461, "tool_time_ms": 2777, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.9850667499999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 925153, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 122, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep01", "query_calls": 62, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 785651, "tokens_cache_write": 334081, "tokens_input": 182, "tokens_output": 20133, "tool_calls": 162, "tool_calls_by_class": { "product_builtin": 100, "product_semantic": 62 }, "tool_result_tokens": 104105, "tool_time_ms": 1376, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6770875, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 952466, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 131, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep02", "query_calls": 69, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 954515, "tokens_cache_write": 267964, "tokens_input": 191, "tokens_output": 20964, "tool_calls": 174, "tool_calls_by_class": { "product_builtin": 105, "product_semantic": 69 }, "tool_result_tokens": 95887, "tool_time_ms": 1252, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2501397499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 821458, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 127, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep03", "query_calls": 71, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 991122, "tokens_cache_write": 198299, "tokens_input": 187, "tokens_output": 20571, "tool_calls": 171, "tool_calls_by_class": { "product_builtin": 100, "product_semantic": 71 }, "tool_result_tokens": 99015, "tool_time_ms": 1400, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3232852499999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 986905, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 147, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep04", "query_calls": 63, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1286238, "tokens_cache_write": 178357, "tokens_input": 207, "tokens_output": 22576, "tool_calls": 188, "tool_calls_by_class": { "product_builtin": 125, "product_semantic": 63 }, "tool_result_tokens": 99687, "tool_time_ms": 2281, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9371380000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 865916, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 128, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep05", "query_calls": 68, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1048421, "tokens_cache_write": 144526, "tokens_input": 188, "tokens_output": 20348, "tool_calls": 175, "tool_calls_by_class": { "product_builtin": 107, "product_semantic": 68 }, "tool_result_tokens": 106483, "tool_time_ms": 1823, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7980667499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 795552, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 128, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep06", "query_calls": 70, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1100741, "tokens_cache_write": 116885, "tokens_input": 188, "tokens_output": 20649, "tool_calls": 167, "tool_calls_by_class": { "product_builtin": 97, "product_semantic": 70 }, "tool_result_tokens": 103187, "tool_time_ms": 1258, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9321610000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 870704, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 133, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep07", "query_calls": 62, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1137892, "tokens_cache_write": 132276, "tokens_input": 193, "tokens_output": 21421, "tool_calls": 180, "tool_calls_by_class": { "product_builtin": 118, "product_semantic": 62 }, "tool_result_tokens": 101761, "tool_time_ms": 1919, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.6990232499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 820402, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 118, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep08", "query_calls": 62, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 979979, "tokens_cache_write": 113319, "tokens_input": 178, "tokens_output": 19996, "tool_calls": 161, "tool_calls_by_class": { "product_builtin": 99, "product_semantic": 62 }, "tool_result_tokens": 95021, "tool_time_ms": 1724, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.00657, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 978527, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 139, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep09", "query_calls": 71, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1215100, "tokens_cache_write": 133264, "tokens_input": 199, "tokens_output": 22605, "tool_calls": 186, "tool_calls_by_class": { "product_builtin": 115, "product_semantic": 71 }, "tool_result_tokens": 102568, "tool_time_ms": 1541, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.84743075, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 868488, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 120, "model": "claude-opus-4-6", "path": "runs/cg-ask-fz-009/claude-opus-4-6/ZeroShotExam/memos/rep10", "query_calls": 69, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 950599, "tokens_cache_write": 139009, "tokens_input": 180, "tokens_output": 20097, "tool_calls": 162, "tool_calls_by_class": { "product_builtin": 93, "product_semantic": 69 }, "tool_result_tokens": 100409, "tool_time_ms": 1351, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.29348175, "denominator": 20, "duration_basis": "run", "duration_ms": 313102, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep01", "query_calls": 27, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2267026, "tokens_cache_write": 119947, "tokens_input": 65, "tokens_output": 16399, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 42, "product_semantic": 27 }, "tool_result_tokens": 53725, "tool_time_ms": 672, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8091927500000002, "denominator": 20, "duration_basis": "run", "duration_ms": 265018, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep02", "query_calls": 27, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1938633, "tokens_cache_write": 82813, "tokens_input": 79, "tokens_output": 12876, "tool_calls": 67, "tool_calls_by_class": { "product_builtin": 40, "product_semantic": 27 }, "tool_result_tokens": 39601, "tool_time_ms": 1167, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.66492975, "denominator": 20, "duration_basis": "run", "duration_ms": 285542, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep03", "query_calls": 18, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1838497, "tokens_cache_write": 61877, "tokens_input": 85, "tokens_output": 14341, "tool_calls": 68, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 18 }, "tool_result_tokens": 28033, "tool_time_ms": 1767, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.82228475, "denominator": 20, "duration_basis": "run", "duration_ms": 247855, "exam": "HumanExam", "exam_epoch": 4, "fails": 9, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep04", "query_calls": 29, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 11, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1937072, "tokens_cache_write": 82851, "tokens_input": 71, "tokens_output": 13423, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 41, "product_semantic": 29 }, "tool_result_tokens": 39052, "tool_time_ms": 1233, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9003977499999998, "denominator": 20, "duration_basis": "run", "duration_ms": 318428, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep05", "query_calls": 45, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2103798, "tokens_cache_write": 66623, "tokens_input": 81, "tokens_output": 17268, "tool_calls": 92, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 45 }, "tool_result_tokens": 28130, "tool_time_ms": 1046, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.1371295000000003, "denominator": 20, "duration_basis": "run", "duration_ms": 282831, "exam": "HumanExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep06", "query_calls": 32, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2398909, "tokens_cache_write": 92956, "tokens_input": 75, "tokens_output": 14253, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 43, "product_semantic": 32 }, "tool_result_tokens": 43431, "tool_time_ms": 1240, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.63789975, "denominator": 20, "duration_basis": "run", "duration_ms": 302500, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep07", "query_calls": 33, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2975912, "tokens_cache_write": 121067, "tokens_input": 75, "tokens_output": 15716, "tool_calls": 80, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 33 }, "tool_result_tokens": 58916, "tool_time_ms": 1166, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.492541, "denominator": 20, "duration_basis": "run", "duration_ms": 224469, "exam": "HumanExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep08", "query_calls": 33, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1400857, "tokens_cache_write": 72474, "tokens_input": 55, "tokens_output": 13555, "tool_calls": 71, "tool_calls_by_class": { "product_builtin": 38, "product_semantic": 33 }, "tool_result_tokens": 33140, "tool_time_ms": 794, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8046615, "denominator": 20, "duration_basis": "run", "duration_ms": 331917, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep09", "query_calls": 27, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3330008, "tokens_cache_write": 114018, "tokens_input": 99, "tokens_output": 17062, "tool_calls": 77, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 27 }, "tool_result_tokens": 53744, "tool_time_ms": 883, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7897292500000002, "denominator": 20, "duration_basis": "run", "duration_ms": 283742, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep10", "query_calls": 28, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1934146, "tokens_cache_write": 67065, "tokens_input": 85, "tokens_output": 16123, "tool_calls": 85, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 28 }, "tool_result_tokens": 28736, "tool_time_ms": 792, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.427378, "denominator": 20, "duration_basis": "run", "duration_ms": 319380, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep11", "query_calls": 28, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|11", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2777396, "tokens_cache_write": 109764, "tokens_input": 71, "tokens_output": 14092, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 28 }, "tool_result_tokens": 45987, "tool_time_ms": 1600, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.81297075, "denominator": 20, "duration_basis": "run", "duration_ms": 310683, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep12", "query_calls": 24, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|12", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1835644, "tokens_cache_write": 84419, "tokens_input": 71, "tokens_output": 14687, "tool_calls": 71, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 24 }, "tool_result_tokens": 34200, "tool_time_ms": 1192, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.1129215, "denominator": 20, "duration_basis": "run", "duration_ms": 335597, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep13", "query_calls": 32, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|13", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2274468, "tokens_cache_write": 97638, "tokens_input": 85, "tokens_output": 14601, "tool_calls": 72, "tool_calls_by_class": { "product_builtin": 40, "product_semantic": 32 }, "tool_result_tokens": 41736, "tool_time_ms": 1134, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9637985, "denominator": 20, "duration_basis": "run", "duration_ms": 365665, "exam": "HumanExam", "exam_epoch": 4, "fails": 9, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep14", "query_calls": 29, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|14", "score": 11, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2075022, "tokens_cache_write": 78506, "tokens_input": 85, "tokens_output": 17408, "tool_calls": 80, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 29 }, "tool_result_tokens": 30181, "tool_time_ms": 1300, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.85317025, "denominator": 20, "duration_basis": "run", "duration_ms": 289093, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos-hard/rep15", "query_calls": 29, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|ffc82068051276c8ecade67885ff45af|15", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1811698, "tokens_cache_write": 91297, "tokens_input": 63, "tokens_output": 15056, "tool_calls": 81, "tool_calls_by_class": { "product_builtin": 52, "product_semantic": 29 }, "tool_result_tokens": 37572, "tool_time_ms": 1537, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.030979, "denominator": 30, "duration_basis": "run", "duration_ms": 155386, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep01", "query_calls": 6, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 633878, "tokens_cache_write": 73184, "tokens_input": 33, "tokens_output": 10259, "tool_calls": 57, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 6 }, "tool_result_tokens": 29061, "tool_time_ms": 261, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.97451525, "denominator": 30, "duration_basis": "run", "duration_ms": 172446, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 747233, "tokens_cache_write": 56035, "tokens_input": 41, "tokens_output": 10019, "tool_calls": 57, "tool_calls_by_class": { "product_builtin": 57 }, "tool_result_tokens": 25826, "tool_time_ms": 762, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9201849999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 159851, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep03", "query_calls": 7, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 680675, "tokens_cache_write": 51122, "tokens_input": 37, "tokens_output": 10406, "tool_calls": 55, "tool_calls_by_class": { "product_builtin": 48, "product_semantic": 7 }, "tool_result_tokens": 23587, "tool_time_ms": 462, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8544357499999999, "denominator": 30, "duration_basis": "run", "duration_ms": 145269, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep04", "query_calls": 7, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 539929, "tokens_cache_write": 55545, "tokens_input": 33, "tokens_output": 9486, "tool_calls": 56, "tool_calls_by_class": { "product_builtin": 49, "product_semantic": 7 }, "tool_result_tokens": 20713, "tool_time_ms": 372, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.7314015, "denominator": 30, "duration_basis": "run", "duration_ms": 135432, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep05", "query_calls": 15, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 459298, "tokens_cache_write": 41642, "tokens_input": 33, "tokens_output": 9653, "tool_calls": 53, "tool_calls_by_class": { "product_builtin": 38, "product_semantic": 15 }, "tool_result_tokens": 18674, "tool_time_ms": 370, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8497035, "denominator": 30, "duration_basis": "run", "duration_ms": 150025, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep06", "query_calls": 6, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 637837, "tokens_cache_write": 47204, "tokens_input": 37, "tokens_output": 9423, "tool_calls": 50, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 6 }, "tool_result_tokens": 21588, "tool_time_ms": 440, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.908427, "denominator": 30, "duration_basis": "run", "duration_ms": 164935, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep07", "query_calls": 5, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 770759, "tokens_cache_write": 42582, "tokens_input": 47, "tokens_output": 10267, "tool_calls": 59, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 5 }, "tool_result_tokens": 18123, "tool_time_ms": 396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.744453, "denominator": 30, "duration_basis": "run", "duration_ms": 136840, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep08", "query_calls": 9, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 501271, "tokens_cache_write": 43590, "tokens_input": 31, "tokens_output": 8849, "tool_calls": 52, "tool_calls_by_class": { "product_builtin": 43, "product_semantic": 9 }, "tool_result_tokens": 19888, "tool_time_ms": 545, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.6962545, "denominator": 30, "duration_basis": "run", "duration_ms": 115510, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep09", "query_calls": 5, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 394709, "tokens_cache_write": 47452, "tokens_input": 25, "tokens_output": 8088, "tool_calls": 38, "tool_calls_by_class": { "product_builtin": 33, "product_semantic": 5 }, "tool_result_tokens": 22326, "tool_time_ms": 452, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8057565, "denominator": 30, "duration_basis": "run", "duration_ms": 157114, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep10", "query_calls": 8, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 572618, "tokens_cache_write": 43686, "tokens_input": 37, "tokens_output": 9849, "tool_calls": 53, "tool_calls_by_class": { "product_builtin": 45, "product_semantic": 8 }, "tool_result_tokens": 19295, "tool_time_ms": 352, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.23589525, "denominator": 30, "duration_basis": "run", "duration_ms": 186399, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/HumanExam/memos/rep11", "query_calls": 4, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 997973, "tokens_cache_write": 80135, "tokens_input": 43, "tokens_output": 9434, "tool_calls": 51, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 4 }, "tool_result_tokens": 34679, "tool_time_ms": 308, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.403452000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 695685, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep01", "query_calls": 240, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6035974, "tokens_cache_write": 277304, "tokens_input": 178, "tokens_output": 26057, "tool_calls": 85, "tool_calls_by_class": { "product_builtin": 64, "product_semantic": 21 }, "tool_result_tokens": 60032, "tool_time_ms": 2346, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.96582175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 577880, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 73, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep02", "query_calls": 480, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5123421, "tokens_cache_write": 139101, "tokens_input": 146, "tokens_output": 21360, "tool_calls": 79, "tool_calls_by_class": { "product_builtin": 40, "product_semantic": 39 }, "tool_result_tokens": 56247, "tool_time_ms": 3066, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.960037499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 711320, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 86, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep03", "query_calls": 310, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7344355, "tokens_cache_write": 255752, "tokens_input": 172, "tokens_output": 27542, "tool_calls": 89, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 26 }, "tool_result_tokens": 71615, "tool_time_ms": 3716, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.54537525, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 791826, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 97, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep04", "query_calls": 375, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7654923, "tokens_cache_write": 159923, "tokens_input": 194, "tokens_output": 28697, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 31 }, "tool_result_tokens": 61210, "tool_time_ms": 2171, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.815977, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 722969, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 93, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep05", "query_calls": 460, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5285494, "tokens_cache_write": 238972, "tokens_input": 186, "tokens_output": 27149, "tool_calls": 102, "tool_calls_by_class": { "product_builtin": 64, "product_semantic": 38 }, "tool_result_tokens": 47307, "tool_time_ms": 2151, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.773446, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 804824, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 91, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep06", "query_calls": 260, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7012547, "tokens_cache_write": 245490, "tokens_input": 182, "tokens_output": 29278, "tool_calls": 108, "tool_calls_by_class": { "product_builtin": 83, "product_semantic": 25 }, "tool_result_tokens": 65449, "tool_time_ms": 2746, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.4236905, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 711764, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep07", "query_calls": 469, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5806131, "tokens_cache_write": 140616, "tokens_input": 160, "tokens_output": 25639, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 37 }, "tool_result_tokens": 57369, "tool_time_ms": 1636, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.0471415, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 826147, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep08", "query_calls": 729, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6787268, "tokens_cache_write": 143330, "tokens_input": 174, "tokens_output": 30273, "tool_calls": 116, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 59 }, "tool_result_tokens": 53715, "tool_time_ms": 2281, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.884062, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 641873, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 75, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep09", "query_calls": 228, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5086024, "tokens_cache_write": 125212, "tokens_input": 150, "tokens_output": 22309, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 56, "product_semantic": 19 }, "tool_result_tokens": 50280, "tool_time_ms": 2258, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.501529000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 736955, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos-hard/rep10", "query_calls": 618, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5330593, "tokens_cache_write": 191790, "tokens_input": 174, "tokens_output": 25467, "tool_calls": 90, "tool_calls_by_class": { "product_builtin": 39, "product_semantic": 51 }, "tool_result_tokens": 51602, "tool_time_ms": 1417, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7568322500000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 465456, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep01", "query_calls": 111, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3236772, "tokens_cache_write": 121041, "tokens_input": 158, "tokens_output": 15246, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 7 }, "tool_result_tokens": 39166, "tool_time_ms": 744, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.28878375, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 450463, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 78, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep02", "query_calls": 138, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2741570, "tokens_cache_write": 86723, "tokens_input": 156, "tokens_output": 15008, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 7 }, "tool_result_tokens": 24090, "tool_time_ms": 708, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.34129975, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 428124, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep03", "query_calls": 179, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2755737, "tokens_cache_write": 99589, "tokens_input": 150, "tokens_output": 13610, "tool_calls": 62, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 9 }, "tool_result_tokens": 32393, "tool_time_ms": 659, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.40831625, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 458870, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep04", "query_calls": 205, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2940315, "tokens_cache_write": 89323, "tokens_input": 158, "tokens_output": 15164, "tool_calls": 67, "tool_calls_by_class": { "product_builtin": 56, "product_semantic": 11 }, "tool_result_tokens": 30618, "tool_time_ms": 761, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.40507075, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 438174, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep05", "query_calls": 191, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3099154, "tokens_cache_write": 87967, "tokens_input": 150, "tokens_output": 12198, "tool_calls": 50, "tool_calls_by_class": { "product_builtin": 40, "product_semantic": 10 }, "tool_result_tokens": 31296, "tool_time_ms": 569, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.62467525, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 495003, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep06", "query_calls": 150, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3279133, "tokens_cache_write": 91279, "tokens_input": 158, "tokens_output": 16553, "tool_calls": 73, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 7 }, "tool_result_tokens": 32001, "tool_time_ms": 673, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.91635375, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 501845, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 83, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep07", "query_calls": 315, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3552360, "tokens_cache_write": 117351, "tokens_input": 166, "tokens_output": 16236, "tool_calls": 65, "tool_calls_by_class": { "product_builtin": 48, "product_semantic": 17 }, "tool_result_tokens": 36453, "tool_time_ms": 662, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6058155000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 470622, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 78, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep08", "query_calls": 225, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3353896, "tokens_cache_write": 89378, "tokens_input": 156, "tokens_output": 14779, "tool_calls": 61, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 11 }, "tool_result_tokens": 35808, "tool_time_ms": 705, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.253082, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 449297, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 76, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep09", "query_calls": 148, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2762344, "tokens_cache_write": 77780, "tokens_input": 152, "tokens_output": 15401, "tool_calls": 65, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 7 }, "tool_result_tokens": 24609, "tool_time_ms": 699, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.99408825, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 420592, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 71, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/MultiTurnExam/memos/rep10", "query_calls": 225, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2501869, "tokens_cache_write": 64103, "tokens_input": 142, "tokens_output": 13672, "tool_calls": 58, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 11 }, "tool_result_tokens": 22674, "tool_time_ms": 742, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.3538822500000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 620050, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 111, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep01", "query_calls": 72, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1007457, "tokens_cache_write": 342475, "tokens_input": 222, "tokens_output": 28343, "tool_calls": 139, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 72 }, "tool_result_tokens": 87388, "tool_time_ms": 1253, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7567725000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 594690, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 114, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep02", "query_calls": 75, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1183190, "tokens_cache_write": 230634, "tokens_input": 228, "tokens_output": 28903, "tool_calls": 149, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 75 }, "tool_result_tokens": 94529, "tool_time_ms": 2128, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.47030675, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 558349, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 107, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep03", "query_calls": 74, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1121911, "tokens_cache_write": 198529, "tokens_input": 214, "tokens_output": 26699, "tool_calls": 135, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 74 }, "tool_result_tokens": 93933, "tool_time_ms": 1986, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7868885, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 616184, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 117, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep04", "query_calls": 71, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1302987, "tokens_cache_write": 222944, "tokens_input": 234, "tokens_output": 29633, "tool_calls": 144, "tool_calls_by_class": { "product_builtin": 73, "product_semantic": 71 }, "tool_result_tokens": 99681, "tool_time_ms": 2172, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.75900775, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 644483, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 119, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep05", "query_calls": 92, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1299923, "tokens_cache_write": 213605, "tokens_input": 238, "tokens_output": 30913, "tool_calls": 158, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 92 }, "tool_result_tokens": 99322, "tool_time_ms": 1286, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6041835, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 632919, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 107, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep06", "query_calls": 58, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1218377, "tokens_cache_write": 208596, "tokens_input": 214, "tokens_output": 27608, "tool_calls": 133, "tool_calls_by_class": { "product_builtin": 75, "product_semantic": 58 }, "tool_result_tokens": 101235, "tool_time_ms": 1267, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.427511, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 587977, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 109, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep07", "query_calls": 84, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1176342, "tokens_cache_write": 179904, "tokens_input": 218, "tokens_output": 28554, "tool_calls": 150, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 84 }, "tool_result_tokens": 90394, "tool_time_ms": 1945, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3286007499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 591377, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 106, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep08", "query_calls": 78, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1162994, "tokens_cache_write": 163315, "tokens_input": 212, "tokens_output": 29013, "tool_calls": 148, "tool_calls_by_class": { "product_builtin": 70, "product_semantic": 78 }, "tool_result_tokens": 86948, "tool_time_ms": 2325, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.4845527499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 611357, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 112, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep09", "query_calls": 73, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1190853, "tokens_cache_write": 185205, "tokens_input": 224, "tokens_output": 29219, "tool_calls": 146, "tool_calls_by_class": { "product_builtin": 73, "product_semantic": 73 }, "tool_result_tokens": 90778, "tool_time_ms": 2042, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.37398475, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 599311, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 104, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos-hard/rep10", "query_calls": 61, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1078802, "tokens_cache_write": 182203, "tokens_input": 208, "tokens_output": 27791, "tool_calls": 135, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 61 }, "tool_result_tokens": 81064, "tool_time_ms": 2426, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.4161155, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 579796, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 117, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep01", "query_calls": 40, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 884141, "tokens_cache_write": 386372, "tokens_input": 234, "tokens_output": 22322, "tool_calls": 119, "tool_calls_by_class": { "product_builtin": 79, "product_semantic": 40 }, "tool_result_tokens": 80858, "tool_time_ms": 920, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.48363575, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 586251, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 114, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep02", "query_calls": 35, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1005679, "tokens_cache_write": 231041, "tokens_input": 228, "tokens_output": 21426, "tool_calls": 114, "tool_calls_by_class": { "product_builtin": 79, "product_semantic": 35 }, "tool_result_tokens": 90227, "tool_time_ms": 1199, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.196866, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 631339, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 121, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep03", "query_calls": 41, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1104037, "tokens_cache_write": 172286, "tokens_input": 242, "tokens_output": 22674, "tool_calls": 118, "tool_calls_by_class": { "product_builtin": 77, "product_semantic": 41 }, "tool_result_tokens": 83936, "tool_time_ms": 1008, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.324783, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 561022, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 126, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep04", "query_calls": 39, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1153396, "tokens_cache_write": 186284, "tokens_input": 252, "tokens_output": 23302, "tool_calls": 123, "tool_calls_by_class": { "product_builtin": 84, "product_semantic": 39 }, "tool_result_tokens": 91449, "tool_time_ms": 1740, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.11594075, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 550431, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 121, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep05", "query_calls": 38, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1071549, "tokens_cache_write": 162993, "tokens_input": 242, "tokens_output": 22410, "tool_calls": 119, "tool_calls_by_class": { "product_builtin": 81, "product_semantic": 38 }, "tool_result_tokens": 78085, "tool_time_ms": 1122, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.01029525, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 548636, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 116, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep06", "query_calls": 39, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1043383, "tokens_cache_write": 151299, "tokens_input": 232, "tokens_output": 21673, "tool_calls": 112, "tool_calls_by_class": { "product_builtin": 73, "product_semantic": 39 }, "tool_result_tokens": 78443, "tool_time_ms": 1100, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.0556632500000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 512900, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 114, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep07", "query_calls": 37, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1060184, "tokens_cache_write": 157573, "tokens_input": 228, "tokens_output": 21584, "tool_calls": 116, "tool_calls_by_class": { "product_builtin": 79, "product_semantic": 37 }, "tool_result_tokens": 84226, "tool_time_ms": 1200, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.14226625, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 553637, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 124, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep08", "query_calls": 49, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1150940, "tokens_cache_write": 158033, "tokens_input": 248, "tokens_output": 23114, "tool_calls": 127, "tool_calls_by_class": { "product_builtin": 78, "product_semantic": 49 }, "tool_result_tokens": 81854, "tool_time_ms": 1095, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.0534662499999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 577355, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 130, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep09", "query_calls": 48, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1243595, "tokens_cache_write": 130855, "tokens_input": 260, "tokens_output": 24501, "tool_calls": 137, "tool_calls_by_class": { "product_builtin": 89, "product_semantic": 48 }, "tool_result_tokens": 77490, "tool_time_ms": 1302, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.96649575, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 564183, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 127, "model": "claude-opus-4-8", "path": "runs/cg-ask-fz-009/claude-opus-4-8/ZeroShotExam/memos/rep10", "query_calls": 41, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1187889, "tokens_cache_write": 124101, "tokens_input": 254, "tokens_output": 23826, "tool_calls": 130, "tool_calls_by_class": { "product_builtin": 89, "product_semantic": 41 }, "tool_result_tokens": 69717, "tool_time_ms": 1191, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.88396325, "denominator": 20, "duration_basis": "run", "duration_ms": 557479, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep01", "query_calls": 38, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2472399, "tokens_cache_write": 120583, "tokens_input": 74, "tokens_output": 35750, "tool_calls": 89, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 38 }, "tool_result_tokens": 54005, "tool_time_ms": 909, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.20717625, "denominator": 20, "duration_basis": "run", "duration_ms": 608693, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep02", "query_calls": 39, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2794295, "tokens_cache_write": 131527, "tokens_input": 82, "tokens_output": 39503, "tool_calls": 97, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 39 }, "tool_result_tokens": 61484, "tool_time_ms": 1462, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.01585575, "denominator": 20, "duration_basis": "run", "duration_ms": 683487, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep03", "query_calls": 37, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4077549, "tokens_cache_write": 145033, "tokens_input": 100, "tokens_output": 42805, "tool_calls": 96, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 37 }, "tool_result_tokens": 66894, "tool_time_ms": 1052, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.387535, "denominator": 20, "duration_basis": "run", "duration_ms": 598729, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep04", "query_calls": 42, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5002100, "tokens_cache_write": 165128, "tokens_input": 102, "tokens_output": 34157, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 42 }, "tool_result_tokens": 76349, "tool_time_ms": 1208, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.2444982499999995, "denominator": 20, "duration_basis": "run", "duration_ms": 795169, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep05", "query_calls": 37, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3942029, "tokens_cache_write": 150059, "tokens_input": 88, "tokens_output": 53407, "tool_calls": 112, "tool_calls_by_class": { "product_builtin": 75, "product_semantic": 37 }, "tool_result_tokens": 68386, "tool_time_ms": 1538, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.62865125, "denominator": 20, "duration_basis": "run", "duration_ms": 674340, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep06", "query_calls": 38, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4795320, "tokens_cache_write": 188077, "tokens_input": 92, "tokens_output": 42202, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 38 }, "tool_result_tokens": 90151, "tool_time_ms": 1214, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.1893195, "denominator": 20, "duration_basis": "run", "duration_ms": 592301, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep07", "query_calls": 40, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3654689, "tokens_cache_write": 223556, "tokens_input": 70, "tokens_output": 38576, "tool_calls": 96, "tool_calls_by_class": { "product_builtin": 56, "product_semantic": 40 }, "tool_result_tokens": 106621, "tool_time_ms": 904, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.6382577499999997, "denominator": 20, "duration_basis": "run", "duration_ms": 559309, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep08", "query_calls": 34, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3652023, "tokens_cache_write": 141969, "tokens_input": 88, "tokens_output": 36980, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 34 }, "tool_result_tokens": 63541, "tool_time_ms": 1093, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.01427725, "denominator": 20, "duration_basis": "run", "duration_ms": 571071, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep09", "query_calls": 46, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3971247, "tokens_cache_write": 171023, "tokens_input": 82, "tokens_output": 38374, "tool_calls": 104, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 46 }, "tool_result_tokens": 83615, "tool_time_ms": 1360, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.2852742500000005, "denominator": 20, "duration_basis": "run", "duration_ms": 807017, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos-hard/rep10", "query_calls": 34, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5616421, "tokens_cache_write": 186447, "tokens_input": 104, "tokens_output": 52450, "tool_calls": 101, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 34 }, "tool_result_tokens": 91906, "tool_time_ms": 1543, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.20079725, "denominator": 30, "duration_basis": "run", "duration_ms": 481862, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep01", "query_calls": 17, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2013437, "tokens_cache_write": 84439, "tokens_input": 82, "tokens_output": 26637, "tool_calls": 80, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 17 }, "tool_result_tokens": 35573, "tool_time_ms": 507, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.85063425, "denominator": 30, "duration_basis": "run", "duration_ms": 514781, "exam": "HumanExam", "exam_epoch": 4, "fails": null, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep02", "query_calls": 14, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2801026, "tokens_cache_write": 113157, "tokens_input": 88, "tokens_output": 29698, "tool_calls": 80, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 14 }, "tool_result_tokens": 55853, "tool_time_ms": 485, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.68020725, "denominator": 30, "duration_basis": "run", "duration_ms": 413685, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep03", "query_calls": 13, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2602987, "tokens_cache_write": 126511, "tokens_input": 74, "tokens_output": 23506, "tool_calls": 72, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 13 }, "tool_result_tokens": 61686, "tool_time_ms": 421, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.06354075, "denominator": 30, "duration_basis": "run", "duration_ms": 625431, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep04", "query_calls": 11, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3054759, "tokens_cache_write": 96297, "tokens_input": 106, "tokens_output": 37351, "tool_calls": 98, "tool_calls_by_class": { "product_builtin": 87, "product_semantic": 11 }, "tool_result_tokens": 45023, "tool_time_ms": 1053, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6202232500000004, "denominator": 30, "duration_basis": "run", "duration_ms": 550614, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep05", "query_calls": 11, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2606869, "tokens_cache_write": 81399, "tokens_input": 94, "tokens_output": 32303, "tool_calls": 92, "tool_calls_by_class": { "product_builtin": 81, "product_semantic": 11 }, "tool_result_tokens": 41679, "tool_time_ms": 697, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3073235000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 375564, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep06", "query_calls": 15, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2123737, "tokens_cache_write": 97764, "tokens_input": 66, "tokens_output": 25364, "tool_calls": 78, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 15 }, "tool_result_tokens": 47258, "tool_time_ms": 482, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.0379435, "denominator": 30, "duration_basis": "run", "duration_ms": 387688, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep07", "query_calls": 12, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1788462, "tokens_cache_write": 80318, "tokens_input": 70, "tokens_output": 25655, "tool_calls": 76, "tool_calls_by_class": { "product_builtin": 64, "product_semantic": 12 }, "tool_result_tokens": 39173, "tool_time_ms": 661, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8876952500000002, "denominator": 30, "duration_basis": "run", "duration_ms": 352697, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep08", "query_calls": 13, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1574403, "tokens_cache_write": 79579, "tokens_input": 60, "tokens_output": 24113, "tool_calls": 73, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 13 }, "tool_result_tokens": 38078, "tool_time_ms": 575, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.37772, "denominator": 30, "duration_basis": "run", "duration_ms": 419732, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep09", "query_calls": 11, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2220745, "tokens_cache_write": 95398, "tokens_input": 72, "tokens_output": 26830, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 11 }, "tool_result_tokens": 47629, "tool_time_ms": 432, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2876475000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 456831, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep10", "query_calls": 12, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2241440, "tokens_cache_write": 72118, "tokens_input": 78, "tokens_output": 28632, "tool_calls": 74, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 12 }, "tool_result_tokens": 40306, "tool_time_ms": 486, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8978465, "denominator": 30, "duration_basis": "run", "duration_ms": 461855, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/HumanExam/memos/rep11", "query_calls": 13, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2880038, "tokens_cache_write": 131726, "tokens_input": 78, "tokens_output": 25366, "tool_calls": 76, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 13 }, "tool_result_tokens": 59554, "tool_time_ms": 566, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.167972499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 693404, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 67, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 248, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4747580, "tokens_cache_write": 161522, "tokens_input": 134, "tokens_output": 31360, "tool_calls": 94, "tool_calls_by_class": { "product_builtin": 71, "product_semantic": 23 }, "tool_result_tokens": 53323, "tool_time_ms": 5538, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.4877075, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 789351, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 74, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 414, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5237735, "tokens_cache_write": 150760, "tokens_input": 148, "tokens_output": 37034, "tool_calls": 113, "tool_calls_by_class": { "product_builtin": 79, "product_semantic": 34 }, "tool_result_tokens": 58894, "tool_time_ms": 4439, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.57999375, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 752533, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 73, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 299, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5369490, "tokens_cache_write": 166267, "tokens_input": 146, "tokens_output": 34214, "tool_calls": 99, "tool_calls_by_class": { "product_builtin": 72, "product_semantic": 27 }, "tool_result_tokens": 60897, "tool_time_ms": 4306, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.083304, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 682981, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 68, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 395, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5902448, "tokens_cache_write": 373068, "tokens_input": 136, "tokens_output": 31989, "tool_calls": 111, "tool_calls_by_class": { "product_builtin": 78, "product_semantic": 33 }, "tool_result_tokens": 75129, "tool_time_ms": 4166, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.47315, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 670227, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 66, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 411, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5017355, "tokens_cache_write": 191026, "tokens_input": 132, "tokens_output": 30796, "tool_calls": 102, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 36 }, "tool_result_tokens": 58020, "tool_time_ms": 3539, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.201691749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 657994, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 67, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 255, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4467006, "tokens_cache_write": 191891, "tokens_input": 134, "tokens_output": 30728, "tool_calls": 95, "tool_calls_by_class": { "product_builtin": 71, "product_semantic": 24 }, "tool_result_tokens": 57565, "tool_time_ms": 3795, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.8436779999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 591398, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 62, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 420, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3805691, "tokens_cache_write": 193846, "tokens_input": 124, "tokens_output": 29147, "tool_calls": 101, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 35 }, "tool_result_tokens": 50010, "tool_time_ms": 2979, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 6.068732750000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 631016, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 59, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 259, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5901448, "tokens_cache_write": 374311, "tokens_input": 118, "tokens_output": 31119, "tool_calls": 87, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 25 }, "tool_result_tokens": 98756, "tool_time_ms": 2798, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.703265, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 686608, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 67, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 377, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5471815, "tokens_cache_write": 183766, "tokens_input": 134, "tokens_output": 32726, "tool_calls": 94, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 34 }, "tool_result_tokens": 67553, "tool_time_ms": 3835, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.91576325, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 692798, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 65, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 328, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4466664, "tokens_cache_write": 138617, "tokens_input": 130, "tokens_output": 32617, "tool_calls": 82, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 29 }, "tool_result_tokens": 54331, "tool_time_ms": 3333, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.3463752500000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 690765, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 69, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 360, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3491833, "tokens_cache_write": 167891, "tokens_input": 138, "tokens_output": 22018, "tool_calls": 59, "tool_calls_by_class": { "product_builtin": 40, "product_semantic": 19 }, "tool_result_tokens": 51458, "tool_time_ms": 396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.83300075, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 734122, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 76, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 292, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4161119, "tokens_cache_write": 185381, "tokens_input": 152, "tokens_output": 23722, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 16 }, "tool_result_tokens": 53817, "tool_time_ms": 615, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.488739, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 723926, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 70, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 174, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4778653, "tokens_cache_write": 229922, "tokens_input": 140, "tokens_output": 26468, "tool_calls": 65, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 8 }, "tool_result_tokens": 70876, "tool_time_ms": 658, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.4292795, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 668108, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 69, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 193, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3862754, "tokens_cache_write": 144894, "tokens_input": 138, "tokens_output": 23665, "tool_calls": 63, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 9 }, "tool_result_tokens": 41809, "tool_time_ms": 710, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.7672637499999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 714786, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 71, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 199, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3998620, "tokens_cache_write": 170827, "tokens_input": 142, "tokens_output": 27983, "tool_calls": 71, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 11 }, "tool_result_tokens": 48822, "tool_time_ms": 677, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.2926902499999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 594714, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 70, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 152, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3595843, "tokens_cache_write": 143351, "tokens_input": 140, "tokens_output": 23925, "tool_calls": 64, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 9 }, "tool_result_tokens": 46508, "tool_time_ms": 715, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.604483, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 600968, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 72, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 323, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4052176, "tokens_cache_write": 161032, "tokens_input": 144, "tokens_output": 22849, "tool_calls": 63, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 19 }, "tool_result_tokens": 48704, "tool_time_ms": 360, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.16318425, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 575491, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 66, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 181, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3414761, "tokens_cache_write": 140071, "tokens_input": 132, "tokens_output": 23188, "tool_calls": 57, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 10 }, "tool_result_tokens": 49738, "tool_time_ms": 656, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.632736749999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 644942, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 74, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 196, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5024156, "tokens_cache_write": 237227, "tokens_input": 148, "tokens_output": 25490, "tool_calls": 68, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 10 }, "tool_result_tokens": 68849, "tool_time_ms": 1095, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.78894125, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 660009, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 80, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 309, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5593445, "tokens_cache_write": 219691, "tokens_input": 160, "tokens_output": 24734, "tool_calls": 82, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 16 }, "tool_result_tokens": 63692, "tool_time_ms": 1426, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.987705, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 942139, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 143, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 79, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1952800, "tokens_cache_write": 436840, "tokens_input": 286, "tokens_output": 51185, "tool_calls": 216, "tool_calls_by_class": { "product_builtin": 137, "product_semantic": 79 }, "tool_result_tokens": 136982, "tool_time_ms": 2929, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.33757075, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 916360, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 136, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 78, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1876259, "tokens_cache_write": 343785, "tokens_input": 272, "tokens_output": 49977, "tool_calls": 198, "tool_calls_by_class": { "product_builtin": 120, "product_semantic": 78 }, "tool_result_tokens": 139033, "tool_time_ms": 3428, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.41399175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 985495, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 154, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 76, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2380566, "tokens_cache_write": 307851, "tokens_input": 308, "tokens_output": 51924, "tool_calls": 220, "tool_calls_by_class": { "product_builtin": 144, "product_semantic": 76 }, "tool_result_tokens": 146219, "tool_time_ms": 3523, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.3636532500000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 939209, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 138, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 82, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2035409, "tokens_cache_write": 331055, "tokens_input": 276, "tokens_output": 51019, "tool_calls": 204, "tool_calls_by_class": { "product_builtin": 122, "product_semantic": 82 }, "tool_result_tokens": 161861, "tool_time_ms": 2816, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.38384575, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 968436, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 133, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 82, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2088069, "tokens_cache_write": 324845, "tokens_input": 266, "tokens_output": 52328, "tool_calls": 205, "tool_calls_by_class": { "product_builtin": 123, "product_semantic": 82 }, "tool_result_tokens": 157436, "tool_time_ms": 2760, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.9550772500000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 962032, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 140, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 77, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2056267, "tokens_cache_write": 257835, "tokens_input": 280, "tokens_output": 52563, "tool_calls": 208, "tool_calls_by_class": { "product_builtin": 131, "product_semantic": 77 }, "tool_result_tokens": 131586, "tool_time_ms": 2647, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.33674025, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1028607, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 79, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2339748, "tokens_cache_write": 276441, "tokens_input": 312, "tokens_output": 57502, "tool_calls": 227, "tool_calls_by_class": { "product_builtin": 148, "product_semantic": 79 }, "tool_result_tokens": 138918, "tool_time_ms": 3090, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.4225477500000006, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1024005, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 144, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 82, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2301353, "tokens_cache_write": 300485, "tokens_input": 288, "tokens_output": 55696, "tool_calls": 222, "tool_calls_by_class": { "product_builtin": 140, "product_semantic": 82 }, "tool_result_tokens": 150916, "tool_time_ms": 3243, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.182449, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1044870, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 147, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 74, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2183033, "tokens_cache_write": 262750, "tokens_input": 294, "tokens_output": 57891, "tool_calls": 217, "tool_calls_by_class": { "product_builtin": 143, "product_semantic": 74 }, "tool_result_tokens": 139350, "tool_time_ms": 2576, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.94182525, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 959857, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 147, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 72, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2164948, "tokens_cache_write": 253581, "tokens_input": 294, "tokens_output": 50920, "tool_calls": 207, "tool_calls_by_class": { "product_builtin": 135, "product_semantic": 72 }, "tool_result_tokens": 132528, "tool_time_ms": 2499, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.65871875, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 817482, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 146, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 57, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1499230, "tokens_cache_write": 460487, "tokens_input": 292, "tokens_output": 41184, "tool_calls": 186, "tool_calls_by_class": { "product_builtin": 129, "product_semantic": 57 }, "tool_result_tokens": 114480, "tool_time_ms": 1784, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.960433, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 765434, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 132, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 55, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1396101, "tokens_cache_write": 366158, "tokens_input": 264, "tokens_output": 38903, "tool_calls": 168, "tool_calls_by_class": { "product_builtin": 113, "product_semantic": 55 }, "tool_result_tokens": 132550, "tool_time_ms": 1938, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.52883475, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 732251, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 131, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 58, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1430587, "tokens_cache_write": 297861, "tokens_input": 262, "tokens_output": 38024, "tool_calls": 167, "tool_calls_by_class": { "product_builtin": 109, "product_semantic": 58 }, "tool_result_tokens": 123689, "tool_time_ms": 1421, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.31930375, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 830490, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 137, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 48, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1628530, "tokens_cache_write": 234867, "tokens_input": 274, "tokens_output": 41430, "tool_calls": 176, "tool_calls_by_class": { "product_builtin": 128, "product_semantic": 48 }, "tool_result_tokens": 113369, "tool_time_ms": 1719, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.3511105, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 785302, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 134, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 52, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1565616, "tokens_cache_write": 254710, "tokens_input": 268, "tokens_output": 39001, "tool_calls": 173, "tool_calls_by_class": { "product_builtin": 121, "product_semantic": 52 }, "tool_result_tokens": 121170, "tool_time_ms": 2242, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.39752525, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 882613, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 144, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 58, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1717533, "tokens_cache_write": 236427, "tokens_input": 288, "tokens_output": 42386, "tool_calls": 185, "tool_calls_by_class": { "product_builtin": 127, "product_semantic": 58 }, "tool_result_tokens": 128932, "tool_time_ms": 1658, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.6772565000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 880570, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 149, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 52, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1822483, "tokens_cache_write": 262560, "tokens_input": 298, "tokens_output": 44941, "tool_calls": 184, "tool_calls_by_class": { "product_builtin": 132, "product_semantic": 52 }, "tool_result_tokens": 130101, "tool_time_ms": 2612, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.4074005000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 813654, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 140, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 54, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1670751, "tokens_cache_write": 244600, "tokens_input": 280, "tokens_output": 41675, "tool_calls": 178, "tool_calls_by_class": { "product_builtin": 124, "product_semantic": 54 }, "tool_result_tokens": 125987, "tool_time_ms": 1625, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.1740255, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 734723, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 125, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 53, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1339476, "tokens_cache_write": 249714, "tokens_input": 250, "tokens_output": 37693, "tool_calls": 161, "tool_calls_by_class": { "product_builtin": 108, "product_semantic": 53 }, "tool_result_tokens": 126660, "tool_time_ms": 2013, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.6900495, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 868031, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 144, "model": "claude-opus-5", "path": "runs/cg-ask-fz-009/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 65, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1717144, "tokens_cache_write": 272226, "tokens_input": 288, "tokens_output": 45145, "tool_calls": 191, "tool_calls_by_class": { "product_builtin": 126, "product_semantic": 65 }, "tool_result_tokens": 133711, "tool_time_ms": 1932, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.8874828, "denominator": 20, "duration_basis": "run", "duration_ms": 842621, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep01", "query_calls": 83, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 8854431, "tokens_cache_write": 169754, "tokens_input": 207, "tokens_output": 39597, "tool_calls": 188, "tool_calls_by_class": { "product_builtin": 105, "product_semantic": 83 }, "tool_result_tokens": 77188, "tool_time_ms": 5394, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.74063035, "denominator": 20, "duration_basis": "run", "duration_ms": 499221, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep02", "query_calls": 17, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6078572, "tokens_cache_write": 137229, "tokens_input": 145, "tokens_output": 26801, "tool_calls": 121, "tool_calls_by_class": { "product_builtin": 104, "product_semantic": 17 }, "tool_result_tokens": 61449, "tool_time_ms": 3138, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8461323999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 677298, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep03", "query_calls": 62, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6185708, "tokens_cache_write": 131504, "tokens_input": 165, "tokens_output": 33119, "tool_calls": 157, "tool_calls_by_class": { "product_builtin": 95, "product_semantic": 62 }, "tool_result_tokens": 60278, "tool_time_ms": 3560, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8495403499999996, "denominator": 20, "duration_basis": "run", "duration_ms": 465503, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep04", "query_calls": 36, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3932327, "tokens_cache_write": 100011, "tokens_input": 137, "tokens_output": 19626, "tool_calls": 98, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 36 }, "tool_result_tokens": 44753, "tool_time_ms": 2375, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.2464494, "denominator": 20, "duration_basis": "run", "duration_ms": 723833, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep05", "query_calls": 41, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7709513, "tokens_cache_write": 128590, "tokens_input": 201, "tokens_output": 30052, "tool_calls": 167, "tool_calls_by_class": { "product_builtin": 126, "product_semantic": 41 }, "tool_result_tokens": 54777, "tool_time_ms": 4996, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.27858765, "denominator": 20, "duration_basis": "run", "duration_ms": 716104, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep06", "query_calls": 64, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7467538, "tokens_cache_write": 142099, "tokens_input": 205, "tokens_output": 33656, "tool_calls": 154, "tool_calls_by_class": { "product_builtin": 90, "product_semantic": 64 }, "tool_result_tokens": 62944, "tool_time_ms": 6015, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.45910605, "denominator": 20, "duration_basis": "run", "duration_ms": 633663, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep07", "query_calls": 60, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7983841, "tokens_cache_write": 149697, "tokens_input": 165, "tokens_output": 33473, "tool_calls": 164, "tool_calls_by_class": { "product_builtin": 104, "product_semantic": 60 }, "tool_result_tokens": 66102, "tool_time_ms": 2446, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.00713485, "denominator": 20, "duration_basis": "run", "duration_ms": 498934, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep08", "query_calls": 53, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4347587, "tokens_cache_write": 101405, "tokens_input": 155, "tokens_output": 21475, "tool_calls": 135, "tool_calls_by_class": { "product_builtin": 82, "product_semantic": 53 }, "tool_result_tokens": 43358, "tool_time_ms": 2455, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.60696625, "denominator": 20, "duration_basis": "run", "duration_ms": 583006, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep09", "query_calls": 55, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5841625, "tokens_cache_write": 106877, "tokens_input": 175, "tokens_output": 30211, "tool_calls": 139, "tool_calls_by_class": { "product_builtin": 84, "product_semantic": 55 }, "tool_result_tokens": 47449, "tool_time_ms": 1574, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5203065999999996, "denominator": 20, "duration_basis": "run", "duration_ms": 555061, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos-hard/rep10", "query_calls": 74, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5517237, "tokens_cache_write": 126086, "tokens_input": 161, "tokens_output": 26122, "tool_calls": 146, "tool_calls_by_class": { "product_builtin": 72, "product_semantic": 74 }, "tool_result_tokens": 55183, "tool_time_ms": 1725, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.80703795, "denominator": 30, "duration_basis": "run", "duration_ms": 179778, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep01", "query_calls": 11, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1117669, "tokens_cache_write": 69511, "tokens_input": 47, "tokens_output": 14062, "tool_calls": 72, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 11 }, "tool_result_tokens": 27540, "tool_time_ms": 301, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.25093685, "denominator": 30, "duration_basis": "run", "duration_ms": 239978, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep02", "query_calls": 4, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2188592, "tokens_cache_write": 96811, "tokens_input": 71, "tokens_output": 15407, "tool_calls": 77, "tool_calls_by_class": { "product_builtin": 73, "product_semantic": 4 }, "tool_result_tokens": 40573, "tool_time_ms": 740, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9057141, "denominator": 30, "duration_basis": "run", "duration_ms": 193395, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep03", "query_calls": 11, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1492742, "tokens_cache_write": 69610, "tokens_input": 63, "tokens_output": 13111, "tool_calls": 77, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 11 }, "tool_result_tokens": 26402, "tool_time_ms": 330, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8201691, "denominator": 30, "duration_basis": "run", "duration_ms": 176200, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep04", "query_calls": 12, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1245237, "tokens_cache_write": 68188, "tokens_input": 51, "tokens_output": 12716, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 12 }, "tool_result_tokens": 29248, "tool_time_ms": 669, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.77962365, "denominator": 30, "duration_basis": "run", "duration_ms": 177218, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep05", "query_calls": 11, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1117628, "tokens_cache_write": 68631, "tokens_input": 53, "tokens_output": 12454, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 11 }, "tool_result_tokens": 27443, "tool_time_ms": 694, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8304632999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 190002, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep06", "query_calls": 6, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1279221, "tokens_cache_write": 70940, "tokens_input": 49, "tokens_output": 12035, "tool_calls": 62, "tool_calls_by_class": { "product_builtin": 56, "product_semantic": 6 }, "tool_result_tokens": 34524, "tool_time_ms": 373, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.0488872999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 244851, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep07", "query_calls": 10, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1906496, "tokens_cache_write": 65854, "tokens_input": 77, "tokens_output": 15317, "tool_calls": 76, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 10 }, "tool_result_tokens": 30736, "tool_time_ms": 576, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.67136325, "denominator": 30, "duration_basis": "run", "duration_ms": 171046, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep08", "query_calls": 8, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1000350, "tokens_cache_write": 54159, "tokens_input": 49, "tokens_output": 11201, "tool_calls": 60, "tool_calls_by_class": { "product_builtin": 52, "product_semantic": 8 }, "tool_result_tokens": 25507, "tool_time_ms": 1391, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.87768045, "denominator": 30, "duration_basis": "run", "duration_ms": 207320, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep09", "query_calls": 14, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1407819, "tokens_cache_write": 68581, "tokens_input": 57, "tokens_output": 13199, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 14 }, "tool_result_tokens": 32616, "tool_time_ms": 772, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8372530499999999, "denominator": 30, "duration_basis": "run", "duration_ms": 175026, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/HumanExam/memos/rep10", "query_calls": 6, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1315916, "tokens_cache_write": 69631, "tokens_input": 49, "tokens_output": 12081, "tool_calls": 61, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 6 }, "tool_result_tokens": 39721, "tool_time_ms": 553, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6273814, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 688269, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 77, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 395, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4475498, "tokens_cache_write": 243976, "tokens_input": 154, "tokens_output": 24624, "tool_calls": 105, "tool_calls_by_class": { "product_builtin": 76, "product_semantic": 29 }, "tool_result_tokens": 42106, "tool_time_ms": 3307, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.7850528, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1163083, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 104, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 806, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7768296, "tokens_cache_write": 416776, "tokens_input": 208, "tokens_output": 59402, "tool_calls": 178, "tool_calls_by_class": { "product_builtin": 107, "product_semantic": 71 }, "tool_result_tokens": 59292, "tool_time_ms": 5116, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.9204077, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 773477, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 84, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 575, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7588354, "tokens_cache_write": 570086, "tokens_input": 168, "tokens_output": 33705, "tool_calls": 133, "tool_calls_by_class": { "product_builtin": 85, "product_semantic": 48 }, "tool_result_tokens": 79699, "tool_time_ms": 2765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.156424100000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 787575, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 93, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 392, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6667062, "tokens_cache_write": 433414, "tokens_input": 186, "tokens_output": 35363, "tool_calls": 135, "tool_calls_by_class": { "product_builtin": 104, "product_semantic": 31 }, "tool_result_tokens": 61126, "tool_time_ms": 5512, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.33269685, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 658176, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 83, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 294, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5070692, "tokens_cache_write": 369411, "tokens_input": 166, "tokens_output": 28380, "tool_calls": 111, "tool_calls_by_class": { "product_builtin": 88, "product_semantic": 23 }, "tool_result_tokens": 48090, "tool_time_ms": 4184, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.0493868, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 913934, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 91, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 936, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7216511, "tokens_cache_write": 334414, "tokens_input": 182, "tokens_output": 41989, "tool_calls": 148, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 81 }, "tool_result_tokens": 65542, "tool_time_ms": 2611, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.42984855, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 642786, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 81, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 677, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5817471, "tokens_cache_write": 339563, "tokens_input": 162, "tokens_output": 27384, "tool_calls": 118, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 56 }, "tool_result_tokens": 66176, "tool_time_ms": 4247, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.4153274999999996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 716731, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 86, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 571, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5646930, "tokens_cache_write": 336778, "tokens_input": 172, "tokens_output": 30521, "tool_calls": 120, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 46 }, "tool_result_tokens": 50494, "tool_time_ms": 3494, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.129294949999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 782407, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 90, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 677, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6914179, "tokens_cache_write": 413187, "tokens_input": 180, "tokens_output": 33670, "tool_calls": 130, "tool_calls_by_class": { "product_builtin": 75, "product_semantic": 55 }, "tool_result_tokens": 60275, "tool_time_ms": 3472, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.27522755, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 735786, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 87, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 483, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5873806, "tokens_cache_write": 536285, "tokens_input": 174, "tokens_output": 33433, "tool_calls": 137, "tool_calls_by_class": { "product_builtin": 97, "product_semantic": 40 }, "tool_result_tokens": 55086, "tool_time_ms": 3958, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.46896815, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 735340, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 80, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 316, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2879748, "tokens_cache_write": 103041, "tokens_input": 160, "tokens_output": 14544, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 17 }, "tool_result_tokens": 25095, "tool_time_ms": 1532, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.30201065, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 607022, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 72, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 347, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2402558, "tokens_cache_write": 92511, "tokens_input": 144, "tokens_output": 15593, "tool_calls": 65, "tool_calls_by_class": { "product_builtin": 48, "product_semantic": 17 }, "tool_result_tokens": 24839, "tool_time_ms": 670, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7729850000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 740466, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 81, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 188, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3105580, "tokens_cache_write": 165136, "tokens_input": 162, "tokens_output": 14771, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 9 }, "tool_result_tokens": 26212, "tool_time_ms": 1980, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.5335670000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 617441, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 272, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2533140, "tokens_cache_write": 155128, "tokens_input": 150, "tokens_output": 12763, "tool_calls": 67, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 14 }, "tool_result_tokens": 26530, "tool_time_ms": 1060, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9742582999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 813082, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 88, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 263, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3032276, "tokens_cache_write": 204666, "tokens_input": 176, "tokens_output": 19770, "tool_calls": 83, "tool_calls_by_class": { "product_builtin": 71, "product_semantic": 12 }, "tool_result_tokens": 24702, "tool_time_ms": 2158, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2730382999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 712279, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 399, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2542311, "tokens_cache_write": 78992, "tokens_input": 150, "tokens_output": 14245, "tool_calls": 73, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 20 }, "tool_result_tokens": 23676, "tool_time_ms": 613, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.67281155, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 748699, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 79, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 273, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2778721, "tokens_cache_write": 149751, "tokens_input": 158, "tokens_output": 18477, "tool_calls": 76, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 14 }, "tool_result_tokens": 25816, "tool_time_ms": 847, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.32575415, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 637635, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 70, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 299, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2547668, "tokens_cache_write": 96661, "tokens_input": 140, "tokens_output": 13237, "tool_calls": 60, "tool_calls_by_class": { "product_builtin": 45, "product_semantic": 15 }, "tool_result_tokens": 28579, "tool_time_ms": 992, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.69174275, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 623652, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 72, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 252, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2703115, "tokens_cache_write": 169403, "tokens_input": 144, "tokens_output": 16341, "tool_calls": 64, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 13 }, "tool_result_tokens": 28368, "tool_time_ms": 1648, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.01781555, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 510107, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 67, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 315, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1973116, "tokens_cache_write": 67285, "tokens_input": 134, "tokens_output": 11544, "tool_calls": 60, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 16 }, "tool_result_tokens": 21165, "tool_time_ms": 1242, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.8403649499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1340241, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 194, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 105, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3409899, "tokens_cache_write": 452467, "tokens_input": 388, "tokens_output": 74632, "tool_calls": 280, "tool_calls_by_class": { "product_builtin": 175, "product_semantic": 105 }, "tool_result_tokens": 144006, "tool_time_ms": 5461, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.09094215, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1116687, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 190, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 132, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2832053, "tokens_cache_write": 348487, "tokens_input": 380, "tokens_output": 62224, "tool_calls": 291, "tool_calls_by_class": { "product_builtin": 159, "product_semantic": 132 }, "tool_result_tokens": 150872, "tool_time_ms": 5585, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.74497765, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1150442, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 180, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 46, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2637963, "tokens_cache_write": 291217, "tokens_input": 360, "tokens_output": 57363, "tool_calls": 228, "tool_calls_by_class": { "product_builtin": 182, "product_semantic": 46 }, "tool_result_tokens": 135193, "tool_time_ms": 7902, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.9631924, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1358778, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 181, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 120, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3107563, "tokens_cache_write": 266974, "tokens_input": 362, "tokens_output": 68579, "tool_calls": 286, "tool_calls_by_class": { "product_builtin": 166, "product_semantic": 120 }, "tool_result_tokens": 138702, "tool_time_ms": 124850, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.90299785, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1288986, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 184, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 79, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2800692, "tokens_cache_write": 264795, "tokens_input": 368, "tokens_output": 71247, "tool_calls": 241, "tool_calls_by_class": { "product_builtin": 162, "product_semantic": 79 }, "tool_result_tokens": 138769, "tool_time_ms": 4808, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9358829, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 850059, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 132, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 34, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1541653, "tokens_cache_write": 214180, "tokens_input": 264, "tokens_output": 44628, "tool_calls": 186, "tool_calls_by_class": { "product_builtin": 152, "product_semantic": 34 }, "tool_result_tokens": 116641, "tool_time_ms": 6808, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.95544595, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1201493, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 200, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 87, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3328224, "tokens_cache_write": 251205, "tokens_input": 400, "tokens_output": 67584, "tool_calls": 269, "tool_calls_by_class": { "product_builtin": 182, "product_semantic": 87 }, "tool_result_tokens": 139382, "tool_time_ms": 8037, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.39689845, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1045401, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 171, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 72, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2543279, "tokens_cache_write": 195641, "tokens_input": 342, "tokens_output": 59949, "tool_calls": 240, "tool_calls_by_class": { "product_builtin": 168, "product_semantic": 72 }, "tool_result_tokens": 117652, "tool_time_ms": 7546, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.55927405, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 985656, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 157, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 56, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2235636, "tokens_cache_write": 264783, "tokens_input": 314, "tokens_output": 59647, "tool_calls": 220, "tool_calls_by_class": { "product_builtin": 164, "product_semantic": 56 }, "tool_result_tokens": 145613, "tool_time_ms": 5572, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.01320555, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1306686, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 207, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 61, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 20, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3298116, "tokens_cache_write": 223889, "tokens_input": 414, "tokens_output": 78863, "tool_calls": 280, "tool_calls_by_class": { "product_builtin": 219, "product_semantic": 61 }, "tool_result_tokens": 128711, "tool_time_ms": 7687, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.89301745, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 799701, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 158, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 48, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1715519, "tokens_cache_write": 483593, "tokens_input": 316, "tokens_output": 37596, "tool_calls": 190, "tool_calls_by_class": { "product_builtin": 142, "product_semantic": 48 }, "tool_result_tokens": 123169, "tool_time_ms": 3397, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.97283405, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 700013, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 143, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 42, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1537016, "tokens_cache_write": 264699, "tokens_input": 286, "tokens_output": 34550, "tool_calls": 153, "tool_calls_by_class": { "product_builtin": 111, "product_semantic": 42 }, "tool_result_tokens": 95712, "tool_time_ms": 3280, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.1401043, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 860380, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 158, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 42, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1888546, "tokens_cache_write": 241434, "tokens_input": 316, "tokens_output": 44481, "tool_calls": 182, "tool_calls_by_class": { "product_builtin": 140, "product_semantic": 42 }, "tool_result_tokens": 100610, "tool_time_ms": 2844, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.59355845, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 688276, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 137, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 41, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1542234, "tokens_cache_write": 175059, "tokens_input": 274, "tokens_output": 31573, "tool_calls": 152, "tool_calls_by_class": { "product_builtin": 111, "product_semantic": 41 }, "tool_result_tokens": 92596, "tool_time_ms": 2624, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.63823175, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 732011, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 146, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 37, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1654540, "tokens_cache_write": 159865, "tokens_input": 292, "tokens_output": 36100, "tool_calls": 169, "tool_calls_by_class": { "product_builtin": 132, "product_semantic": 37 }, "tool_result_tokens": 94080, "tool_time_ms": 2090, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.6194814499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 725731, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 141, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 42, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1573839, "tokens_cache_write": 168317, "tokens_input": 282, "tokens_output": 34353, "tool_calls": 164, "tool_calls_by_class": { "other": 1, "product_builtin": 121, "product_semantic": 42 }, "tool_result_tokens": 94363, "tool_time_ms": 2910, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.96622205, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 945312, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 166, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 41, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2102866, "tokens_cache_write": 166427, "tokens_input": 332, "tokens_output": 47351, "tool_calls": 196, "tool_calls_by_class": { "product_builtin": 155, "product_semantic": 41 }, "tool_result_tokens": 96189, "tool_time_ms": 4163, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7077957499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 759105, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 146, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 39, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1734370, "tokens_cache_write": 171261, "tokens_input": 292, "tokens_output": 36292, "tool_calls": 173, "tool_calls_by_class": { "product_builtin": 134, "product_semantic": 39 }, "tool_result_tokens": 101662, "tool_time_ms": 2167, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7617308, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 810553, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 147, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 41, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1779896, "tokens_cache_write": 169520, "tokens_input": 294, "tokens_output": 39412, "tool_calls": 163, "tool_calls_by_class": { "product_builtin": 122, "product_semantic": 41 }, "tool_result_tokens": 99827, "tool_time_ms": 3517, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.81567005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 821684, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 145, "model": "claude-sonnet-5", "path": "runs/cg-ask-fz-009/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 40, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1794321, "tokens_cache_write": 172381, "tokens_input": 290, "tokens_output": 42005, "tool_calls": 184, "tool_calls_by_class": { "product_builtin": 144, "product_semantic": 40 }, "tool_result_tokens": 99954, "tool_time_ms": 3021, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.06517416, "denominator": 20, "duration_basis": "run", "duration_ms": 87627, "exam": "HumanExam", "exam_epoch": 4, "fails": 8, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep01", "query_calls": 44, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 12, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 942538, "tokens_cache_write": 151384, "tokens_input": 33, "tokens_output": 7059, "tool_calls": 83, "tool_calls_by_class": { "product_builtin": 39, "product_semantic": 44 }, "tool_result_tokens": 131304, "tool_time_ms": 1454, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.08566693, "denominator": 20, "duration_basis": "run", "duration_ms": 102798, "exam": "HumanExam", "exam_epoch": 4, "fails": 8, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep02", "query_calls": 34, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 12, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1384524, "tokens_cache_write": 192577, "tokens_input": 39, "tokens_output": 8187, "tool_calls": 85, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 34 }, "tool_result_tokens": 168019, "tool_time_ms": 1300, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.06878983, "denominator": 20, "duration_basis": "run", "duration_ms": 81649, "exam": "HumanExam", "exam_epoch": 4, "fails": 7, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep03", "query_calls": 26, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1078784, "tokens_cache_write": 159279, "tokens_input": 36, "tokens_output": 6156, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 40, "product_semantic": 26 }, "tool_result_tokens": 136465, "tool_time_ms": 563, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.08624337, "denominator": 20, "duration_basis": "run", "duration_ms": 82171, "exam": "HumanExam", "exam_epoch": 4, "fails": 7, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep04", "query_calls": 43, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1058581, "tokens_cache_write": 228511, "tokens_input": 30, "tokens_output": 6615, "tool_calls": 90, "tool_calls_by_class": { "product_builtin": 47, "product_semantic": 43 }, "tool_result_tokens": 198186, "tool_time_ms": 1315, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.08728377000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 96372, "exam": "HumanExam", "exam_epoch": 4, "fails": 7, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep05", "query_calls": 34, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 13, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1686901, "tokens_cache_write": 182623, "tokens_input": 42, "tokens_output": 6568, "tool_calls": 75, "tool_calls_by_class": { "product_builtin": 41, "product_semantic": 34 }, "tool_result_tokens": 160381, "tool_time_ms": 1762, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.08114971000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 112760, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep06", "query_calls": 48, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1372568, "tokens_cache_write": 169083, "tokens_input": 42, "tokens_output": 9516, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 43, "product_semantic": 48 }, "tool_result_tokens": 144976, "tool_time_ms": 644, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.07194446, "denominator": 20, "duration_basis": "run", "duration_ms": 70384, "exam": "HumanExam", "exam_epoch": 4, "fails": 9, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep07", "query_calls": 39, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 11, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 503443, "tokens_cache_write": 216516, "tokens_input": 33, "tokens_output": 6450, "tool_calls": 82, "tool_calls_by_class": { "product_builtin": 43, "product_semantic": 39 }, "tool_result_tokens": 94118, "tool_time_ms": 1163, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.04602999000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 59672, "exam": "HumanExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep08", "query_calls": 27, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 466152, "tokens_cache_write": 120519, "tokens_input": 24, "tokens_output": 5477, "tool_calls": 58, "tool_calls_by_class": { "product_builtin": 31, "product_semantic": 27 }, "tool_result_tokens": 103108, "tool_time_ms": 717, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.07651539, "denominator": 20, "duration_basis": "run", "duration_ms": 78569, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep09", "query_calls": 25, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 996222, "tokens_cache_write": 194751, "tokens_input": 30, "tokens_output": 6581, "tool_calls": 76, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 25 }, "tool_result_tokens": 167824, "tool_time_ms": 1861, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.06945685, "denominator": 20, "duration_basis": "run", "duration_ms": 83198, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos-hard/rep10", "query_calls": 33, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 863355, "tokens_cache_write": 172951, "tokens_input": 30, "tokens_output": 7455, "tool_calls": 81, "tool_calls_by_class": { "product_builtin": 48, "product_semantic": 33 }, "tool_result_tokens": 154236, "tool_time_ms": 757, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.05488458, "denominator": 30, "duration_basis": "run", "duration_ms": 75330, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep01", "query_calls": 21, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 537244, "tokens_cache_write": 141430, "tokens_input": 27, "tokens_output": 7314, "tool_calls": 79, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 21 }, "tool_result_tokens": 121756, "tool_time_ms": 366, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.051738530000000005, "denominator": 30, "duration_basis": "run", "duration_ms": 55716, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep02", "query_calls": 24, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 471319, "tokens_cache_write": 143883, "tokens_input": 21, "tokens_output": 5281, "tool_calls": 60, "tool_calls_by_class": { "product_builtin": 36, "product_semantic": 24 }, "tool_result_tokens": 124584, "tool_time_ms": 707, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.043838749999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 68889, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep03", "query_calls": 9, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 443475, "tokens_cache_write": 112469, "tokens_input": 24, "tokens_output": 5706, "tool_calls": 59, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 9 }, "tool_result_tokens": 98189, "tool_time_ms": 506, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.05192268, "denominator": 30, "duration_basis": "run", "duration_ms": 56820, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep04", "query_calls": 22, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 331424, "tokens_cache_write": 142680, "tokens_input": 13393, "tokens_output": 5788, "tool_calls": 64, "tool_calls_by_class": { "product_builtin": 42, "product_semantic": 22 }, "tool_result_tokens": 126002, "tool_time_ms": 243, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.03683998, "denominator": 30, "duration_basis": "run", "duration_ms": 68317, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep05", "query_calls": 38, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 387304, "tokens_cache_write": 87554, "tokens_input": 33, "tokens_output": 5999, "tool_calls": 83, "tool_calls_by_class": { "product_builtin": 45, "product_semantic": 38 }, "tool_result_tokens": 74354, "tool_time_ms": 223, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.07217669, "denominator": 30, "duration_basis": "run", "duration_ms": 76879, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep06", "query_calls": 30, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 489067, "tokens_cache_write": 218895, "tokens_input": 36, "tokens_output": 6387, "tool_calls": 92, "tool_calls_by_class": { "product_builtin": 62, "product_semantic": 30 }, "tool_result_tokens": 121221, "tool_time_ms": 484, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.048407530000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 79864, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep07", "query_calls": 33, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 670759, "tokens_cache_write": 110579, "tokens_input": 36, "tokens_output": 6117, "tool_calls": 84, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 33 }, "tool_result_tokens": 95205, "tool_time_ms": 181, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.06705696, "denominator": 30, "duration_basis": "run", "duration_ms": 76890, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep08", "query_calls": 19, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 650918, "tokens_cache_write": 186476, "tokens_input": 30, "tokens_output": 6178, "tool_calls": 74, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 19 }, "tool_result_tokens": 163880, "tool_time_ms": 344, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.05989849, "denominator": 30, "duration_basis": "run", "duration_ms": 79929, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep09", "query_calls": 12, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 781487, "tokens_cache_write": 143415, "tokens_input": 33, "tokens_output": 7007, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 12 }, "tool_result_tokens": 120838, "tool_time_ms": 354, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.047538620000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 72202, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/HumanExam/memos/rep10", "query_calls": 21, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 572116, "tokens_cache_write": 113666, "tokens_input": 959, "tokens_output": 6240, "tool_calls": 87, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 21 }, "tool_result_tokens": 97350, "tool_time_ms": 439, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.38581538000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 470060, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 64, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep01", "query_calls": 786, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 11361519, "tokens_cache_write": 558596, "tokens_input": 192, "tokens_output": 15748, "tool_calls": 179, "tool_calls_by_class": { "product_builtin": 106, "product_semantic": 73 }, "tool_result_tokens": 306125, "tool_time_ms": 2304, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.31239233, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 480160, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 72, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep02", "query_calls": 699, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7644794, "tokens_cache_write": 564733, "tokens_input": 216, "tokens_output": 15225, "tool_calls": 187, "tool_calls_by_class": { "product_builtin": 125, "product_semantic": 62 }, "tool_result_tokens": 250450, "tool_time_ms": 2339, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.31089231, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 482531, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": 64, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep03", "query_calls": 611, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 9050968, "tokens_cache_write": 447511, "tokens_input": 192, "tokens_output": 14964, "tool_calls": 168, "tool_calls_by_class": { "product_builtin": 114, "product_semantic": 54 }, "tool_result_tokens": 264043, "tool_time_ms": 2974, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.59186088, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 544638, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 71, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep04", "query_calls": 899, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 10918479, "tokens_cache_write": 1409334, "tokens_input": 213, "tokens_output": 17596, "tool_calls": 216, "tool_calls_by_class": { "product_builtin": 131, "product_semantic": 85 }, "tool_result_tokens": 317499, "tool_time_ms": 3413, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.36976411, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 439875, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 58, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep05", "query_calls": 601, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 8780683, "tokens_cache_write": 692205, "tokens_input": 18046, "tokens_output": 14575, "tool_calls": 167, "tool_calls_by_class": { "product_builtin": 112, "product_semantic": 55 }, "tool_result_tokens": 290445, "tool_time_ms": 3579, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.33218926, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 371932, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 57, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep06", "query_calls": 812, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 8757408, "tokens_cache_write": 560314, "tokens_input": 171, "tokens_output": 14107, "tool_calls": 182, "tool_calls_by_class": { "product_builtin": 109, "product_semantic": 73 }, "tool_result_tokens": 307907, "tool_time_ms": 2933, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5496575100000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 454203, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 61, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep07", "query_calls": 794, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 12532918, "tokens_cache_write": 1116631, "tokens_input": 183, "tokens_output": 16504, "tool_calls": 189, "tool_calls_by_class": { "product_builtin": 116, "product_semantic": 73 }, "tool_result_tokens": 412893, "tool_time_ms": 3424, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.36062004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 400879, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 62, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep08", "query_calls": 845, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 9388862, "tokens_cache_write": 602600, "tokens_input": 17532, "tokens_output": 15572, "tool_calls": 196, "tool_calls_by_class": { "product_builtin": 120, "product_semantic": 76 }, "tool_result_tokens": 334489, "tool_time_ms": 2651, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.59750099, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 471172, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 64, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep09", "query_calls": 699, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 14240407, "tokens_cache_write": 1175229, "tokens_input": 192, "tokens_output": 15706, "tool_calls": 176, "tool_calls_by_class": { "product_builtin": 110, "product_semantic": 66 }, "tool_result_tokens": 379940, "tool_time_ms": 2943, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.41619545, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 556142, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 62, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos-hard/rep10", "query_calls": 862, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 11756735, "tokens_cache_write": 650059, "tokens_input": 186, "tokens_output": 15424, "tool_calls": 208, "tool_calls_by_class": { "product_builtin": 132, "product_semantic": 76 }, "tool_result_tokens": 363599, "tool_time_ms": 4024, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.17646592, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 331818, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 79, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep01", "query_calls": 536, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3676236, "tokens_cache_write": 368452, "tokens_input": 237, "tokens_output": 8984, "tool_calls": 98, "tool_calls_by_class": { "product_builtin": 64, "product_semantic": 34 }, "tool_result_tokens": 119257, "tool_time_ms": 896, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.1692264, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 355533, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 81, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep02", "query_calls": 857, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4792200, "tokens_cache_write": 243948, "tokens_input": 243, "tokens_output": 10289, "tool_calls": 130, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 67 }, "tool_result_tokens": 129756, "tool_time_ms": 695, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19394274, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 352725, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 84, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep03", "query_calls": 659, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4274292, "tokens_cache_write": 385986, "tokens_input": 252, "tokens_output": 9925, "tool_calls": 117, "tool_calls_by_class": { "product_builtin": 70, "product_semantic": 47 }, "tool_result_tokens": 158332, "tool_time_ms": 866, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.23566752000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 393757, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 86, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep04", "query_calls": 682, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5691516, "tokens_cache_write": 433240, "tokens_input": 6118, "tokens_output": 10253, "tool_calls": 131, "tool_calls_by_class": { "product_builtin": 85, "product_semantic": 46 }, "tool_result_tokens": 144493, "tool_time_ms": 668, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.18930674, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 368804, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 78, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep05", "query_calls": 706, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5164677, "tokens_cache_write": 291000, "tokens_input": 8434, "tokens_output": 9647, "tool_calls": 120, "tool_calls_by_class": { "product_builtin": 67, "product_semantic": 53 }, "tool_result_tokens": 160125, "tool_time_ms": 716, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19088300000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 370434, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 81, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep06", "query_calls": 663, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4104245, "tokens_cache_write": 386782, "tokens_input": 243, "tokens_output": 10045, "tool_calls": 109, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 46 }, "tool_result_tokens": 148057, "tool_time_ms": 836, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19866453, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 356862, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 83, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep07", "query_calls": 776, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3657334, "tokens_cache_write": 456733, "tokens_input": 249, "tokens_output": 9404, "tool_calls": 117, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 60 }, "tool_result_tokens": 122444, "tool_time_ms": 1057, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19966782, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 312125, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 76, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep08", "query_calls": 557, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3856321, "tokens_cache_write": 439852, "tokens_input": 228, "tokens_output": 10444, "tool_calls": 113, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 39 }, "tool_result_tokens": 140200, "tool_time_ms": 1760, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.21655502999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 301219, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 78, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep09", "query_calls": 729, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5171674, "tokens_cache_write": 402019, "tokens_input": 234, "tokens_output": 10475, "tool_calls": 129, "tool_calls_by_class": { "product_builtin": 75, "product_semantic": 54 }, "tool_result_tokens": 205264, "tool_time_ms": 636, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.16964629, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 291039, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 79, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/MultiTurnExam/memos/rep10", "query_calls": 581, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4331597, "tokens_cache_write": 288687, "tokens_input": 237, "tokens_output": 8996, "tool_calls": 92, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 41 }, "tool_result_tokens": 142136, "tool_time_ms": 531, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.14138188000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 506633, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 131, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep01", "query_calls": 105, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1120259, "tokens_cache_write": 370458, "tokens_input": 393, "tokens_output": 21903, "tool_calls": 231, "tool_calls_by_class": { "product_builtin": 126, "product_semantic": 105 }, "tool_result_tokens": 321595, "tool_time_ms": 4009, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.17632701, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 601362, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 153, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep02", "query_calls": 109, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1582338, "tokens_cache_write": 461541, "tokens_input": 459, "tokens_output": 24336, "tool_calls": 243, "tool_calls_by_class": { "product_builtin": 134, "product_semantic": 109 }, "tool_result_tokens": 357030, "tool_time_ms": 4122, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.17052338, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 519358, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 137, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep03", "query_calls": 101, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1700364, "tokens_cache_write": 445214, "tokens_input": 411, "tokens_output": 20942, "tool_calls": 222, "tool_calls_by_class": { "product_builtin": 121, "product_semantic": 101 }, "tool_result_tokens": 374315, "tool_time_ms": 4122, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.14444358, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 464668, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 124, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep04", "query_calls": 94, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1006939, "tokens_cache_write": 399088, "tokens_input": 372, "tokens_output": 20382, "tool_calls": 207, "tool_calls_by_class": { "product_builtin": 113, "product_semantic": 94 }, "tool_result_tokens": 315321, "tool_time_ms": 3925, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.15536169, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 519797, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 138, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep05", "query_calls": 117, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1213547, "tokens_cache_write": 428099, "tokens_input": 414, "tokens_output": 19986, "tool_calls": 226, "tool_calls_by_class": { "product_builtin": 109, "product_semantic": 117 }, "tool_result_tokens": 360340, "tool_time_ms": 3332, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.17803647, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 606906, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": 159, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep06", "query_calls": 116, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1526996, "tokens_cache_write": 463511, "tokens_input": 17172, "tokens_output": 23487, "tool_calls": 230, "tool_calls_by_class": { "product_builtin": 114, "product_semantic": 116 }, "tool_result_tokens": 336256, "tool_time_ms": 3321, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.13239172, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 560604, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 144, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep07", "query_calls": 102, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1314411, "tokens_cache_write": 319518, "tokens_input": 9032, "tokens_output": 20348, "tool_calls": 213, "tool_calls_by_class": { "product_builtin": 111, "product_semantic": 102 }, "tool_result_tokens": 268600, "tool_time_ms": 3077, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.15859441, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 559389, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 147, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep08", "query_calls": 118, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1452218, "tokens_cache_write": 411489, "tokens_input": 477, "tokens_output": 22152, "tool_calls": 256, "tool_calls_by_class": { "product_builtin": 138, "product_semantic": 118 }, "tool_result_tokens": 338060, "tool_time_ms": 4340, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.14418295, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 537622, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 147, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep09", "query_calls": 114, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1347340, "tokens_cache_write": 365891, "tokens_input": 441, "tokens_output": 21396, "tool_calls": 220, "tool_calls_by_class": { "product_builtin": 106, "product_semantic": 114 }, "tool_result_tokens": 305989, "tool_time_ms": 3300, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.15951679000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 566906, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 140, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos-hard/rep10", "query_calls": 101, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1398132, "tokens_cache_write": 416287, "tokens_input": 420, "tokens_output": 22832, "tool_calls": 234, "tool_calls_by_class": { "product_builtin": 133, "product_semantic": 101 }, "tool_result_tokens": 325033, "tool_time_ms": 3145, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.1282397, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 472957, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 137, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep01", "query_calls": 68, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 993155, "tokens_cache_write": 363616, "tokens_input": 411, "tokens_output": 14492, "tool_calls": 181, "tool_calls_by_class": { "product_builtin": 113, "product_semantic": 68 }, "tool_result_tokens": 280570, "tool_time_ms": 1243, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.12028915000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 506226, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 147, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep02", "query_calls": 80, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1170880, "tokens_cache_write": 312311, "tokens_input": 441, "tokens_output": 15588, "tool_calls": 197, "tool_calls_by_class": { "product_builtin": 117, "product_semantic": 80 }, "tool_result_tokens": 266991, "tool_time_ms": 1247, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.14089412, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 496808, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 145, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep03", "query_calls": 73, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1126216, "tokens_cache_write": 399412, "tokens_input": 472, "tokens_output": 15352, "tool_calls": 191, "tool_calls_by_class": { "product_builtin": 118, "product_semantic": 73 }, "tool_result_tokens": 299099, "tool_time_ms": 1631, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.09659843, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 434285, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 126, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep04", "query_calls": 68, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 27, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 773864, "tokens_cache_write": 253795, "tokens_input": 378, "tokens_output": 14664, "tool_calls": 184, "tool_calls_by_class": { "product_builtin": 116, "product_semantic": 68 }, "tool_result_tokens": 218357, "tool_time_ms": 1943, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.15078814000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 506718, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 161, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep05", "query_calls": 77, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1502362, "tokens_cache_write": 404414, "tokens_input": 657, "tokens_output": 16255, "tool_calls": 214, "tool_calls_by_class": { "product_builtin": 137, "product_semantic": 77 }, "tool_result_tokens": 313115, "tool_time_ms": 1680, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.13414326999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 475666, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 146, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep06", "query_calls": 74, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1175166, "tokens_cache_write": 364419, "tokens_input": 722, "tokens_output": 16159, "tool_calls": 211, "tool_calls_by_class": { "product_builtin": 137, "product_semantic": 74 }, "tool_result_tokens": 315775, "tool_time_ms": 2423, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.10297461, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 426086, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 130, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep07", "query_calls": 77, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 813688, "tokens_cache_write": 278485, "tokens_input": 390, "tokens_output": 14168, "tool_calls": 180, "tool_calls_by_class": { "product_builtin": 103, "product_semantic": 77 }, "tool_result_tokens": 248424, "tool_time_ms": 2257, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.12298813, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 484929, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 151, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep08", "query_calls": 82, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1191244, "tokens_cache_write": 318929, "tokens_input": 453, "tokens_output": 16117, "tool_calls": 206, "tool_calls_by_class": { "product_builtin": 124, "product_semantic": 82 }, "tool_result_tokens": 264670, "tool_time_ms": 3448, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.12644634, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 457249, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 147, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep09", "query_calls": 73, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1330577, "tokens_cache_write": 322172, "tokens_input": 441, "tokens_output": 16003, "tool_calls": 200, "tool_calls_by_class": { "product_builtin": 127, "product_semantic": 73 }, "tool_result_tokens": 269480, "tool_time_ms": 1521, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.11897268999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 463863, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 143, "model": "gpt-5.6-luna", "path": "runs/cg-ask-fz-009/gpt-5.6-luna/ZeroShotExam/memos/rep10", "query_calls": 68, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 891062, "tokens_cache_write": 333377, "tokens_input": 470, "tokens_output": 14761, "tool_calls": 175, "tool_calls_by_class": { "product_builtin": 107, "product_semantic": 68 }, "tool_result_tokens": 269915, "tool_time_ms": 1262, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.4762032, "denominator": 20, "duration_basis": "run", "duration_ms": 216427, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep01", "query_calls": 41, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1246363, "tokens_cache_write": 162050, "tokens_input": 42, "tokens_output": 8362, "tool_calls": 104, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 41 }, "tool_result_tokens": 134855, "tool_time_ms": 909, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9058652, "denominator": 20, "duration_basis": "run", "duration_ms": 127329, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep02", "query_calls": 39, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 534133, "tokens_cache_write": 110132, "tokens_input": 33, "tokens_output": 7071, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 39 }, "tool_result_tokens": 93194, "tool_time_ms": 735, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2199716, "denominator": 20, "duration_basis": "run", "duration_ms": 161194, "exam": "HumanExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep03", "query_calls": 40, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 901714, "tokens_cache_write": 142458, "tokens_input": 39, "tokens_output": 7342, "tool_calls": 103, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 40 }, "tool_result_tokens": 119892, "tool_time_ms": 1240, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.07073, "denominator": 20, "duration_basis": "run", "duration_ms": 173453, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep04", "query_calls": 51, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 822305, "tokens_cache_write": 116148, "tokens_input": 42, "tokens_output": 8045, "tool_calls": 122, "tool_calls_by_class": { "product_builtin": 71, "product_semantic": 51 }, "tool_result_tokens": 95868, "tool_time_ms": 526, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.3000132, "denominator": 20, "duration_basis": "run", "duration_ms": 175154, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep05", "query_calls": 51, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1109733, "tokens_cache_write": 137340, "tokens_input": 45, "tokens_output": 8462, "tool_calls": 110, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 51 }, "tool_result_tokens": 115704, "tool_time_ms": 722, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2128946, "denominator": 20, "duration_basis": "run", "duration_ms": 164124, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep06", "query_calls": 50, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 800874, "tokens_cache_write": 138625, "tokens_input": 30, "tokens_output": 9965, "tool_calls": 122, "tool_calls_by_class": { "product_builtin": 72, "product_semantic": 50 }, "tool_result_tokens": 115039, "tool_time_ms": 1160, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1622104000000002, "denominator": 20, "duration_basis": "run", "duration_ms": 162545, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep07", "query_calls": 56, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 789066, "tokens_cache_write": 133400, "tokens_input": 36, "tokens_output": 8972, "tool_calls": 117, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 56 }, "tool_result_tokens": 110485, "tool_time_ms": 746, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.7950576, "denominator": 20, "duration_basis": "run", "duration_ms": 124598, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep08", "query_calls": 48, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 500564, "tokens_cache_write": 89192, "tokens_input": 33, "tokens_output": 7437, "tool_calls": 98, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 48 }, "tool_result_tokens": 75552, "tool_time_ms": 1168, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2199056000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 135954, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep09", "query_calls": 50, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 751089, "tokens_cache_write": 151450, "tokens_input": 30, "tokens_output": 8105, "tool_calls": 114, "tool_calls_by_class": { "product_builtin": 64, "product_semantic": 50 }, "tool_result_tokens": 128007, "tool_time_ms": 684, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2502868, "denominator": 20, "duration_basis": "run", "duration_ms": 164988, "exam": "HumanExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos-hard/rep10", "query_calls": 50, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1062272, "tokens_cache_write": 127474, "tokens_input": 42, "tokens_output": 9392, "tool_calls": 108, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 50 }, "tool_result_tokens": 107814, "tool_time_ms": 492, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8589708, "denominator": 30, "duration_basis": "run", "duration_ms": 83948, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep01", "query_calls": 19, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 355942, "tokens_cache_write": 116446, "tokens_input": 21, "tokens_output": 6714, "tool_calls": 89, "tool_calls_by_class": { "product_builtin": 70, "product_semantic": 19 }, "tool_result_tokens": 99451, "tool_time_ms": 1313, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.949824, "denominator": 30, "duration_basis": "run", "duration_ms": 75187, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep02", "query_calls": 16, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 411045, "tokens_cache_write": 134646, "tokens_input": 24, "tokens_output": 5604, "tool_calls": 65, "tool_calls_by_class": { "product_builtin": 49, "product_semantic": 16 }, "tool_result_tokens": 114273, "tool_time_ms": 354, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8915776, "denominator": 30, "duration_basis": "run", "duration_ms": 103023, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep03", "query_calls": 11, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 531849, "tokens_cache_write": 111618, "tokens_input": 27, "tokens_output": 6032, "tool_calls": 55, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 11 }, "tool_result_tokens": 95639, "tool_time_ms": 582, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8580924, "denominator": 30, "duration_basis": "run", "duration_ms": 94740, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep04", "query_calls": 18, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 360616, "tokens_cache_write": 114946, "tokens_input": 24, "tokens_output": 6951, "tool_calls": 71, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 18 }, "tool_result_tokens": 85368, "tool_time_ms": 405, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.0545524, "denominator": 30, "duration_basis": "run", "duration_ms": 95028, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep05", "query_calls": 17, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 495836, "tokens_cache_write": 148790, "tokens_input": 27, "tokens_output": 5608, "tool_calls": 63, "tool_calls_by_class": { "product_builtin": 46, "product_semantic": 17 }, "tool_result_tokens": 112778, "tool_time_ms": 437, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.0143195999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 98085, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep06", "query_calls": 18, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 536929, "tokens_cache_write": 134256, "tokens_input": 27, "tokens_output": 6408, "tool_calls": 62, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 18 }, "tool_result_tokens": 117128, "tool_time_ms": 396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.133312, "denominator": 30, "duration_basis": "run", "duration_ms": 128904, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep07", "query_calls": 14, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 668935, "tokens_cache_write": 146594, "tokens_input": 27, "tokens_output": 6633, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 56, "product_semantic": 14 }, "tool_result_tokens": 127910, "tool_time_ms": 392, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2140698, "denominator": 30, "duration_basis": "run", "duration_ms": 120075, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep08", "query_calls": 16, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 795382, "tokens_cache_write": 156441, "tokens_input": 33, "tokens_output": 5679, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 16 }, "tool_result_tokens": 135312, "tool_time_ms": 435, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8593686, "denominator": 30, "duration_basis": "run", "duration_ms": 130542, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep09", "query_calls": 16, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 448539, "tokens_cache_write": 109369, "tokens_input": 27, "tokens_output": 6650, "tool_calls": 66, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 16 }, "tool_result_tokens": 94587, "tool_time_ms": 519, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9836146, "denominator": 30, "duration_basis": "run", "duration_ms": 140697, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/HumanExam/memos/rep10", "query_calls": 15, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 455984, "tokens_cache_write": 134021, "tokens_input": 24, "tokens_output": 6551, "tool_calls": 59, "tool_calls_by_class": { "product_builtin": 44, "product_semantic": 15 }, "tool_result_tokens": 117164, "tool_time_ms": 645, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.960821, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 575621, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 57, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep01", "query_calls": 699, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7911380, "tokens_cache_write": 514917, "tokens_input": 171, "tokens_output": 11050, "tool_calls": 155, "tool_calls_by_class": { "product_builtin": 89, "product_semantic": 66 }, "tool_result_tokens": 263220, "tool_time_ms": 2052, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.9581527999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 485466, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 58, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep02", "query_calls": 987, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4959842, "tokens_cache_write": 344676, "tokens_input": 174, "tokens_output": 12507, "tool_calls": 174, "tool_calls_by_class": { "product_builtin": 82, "product_semantic": 92 }, "tool_result_tokens": 189102, "tool_time_ms": 2280, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.506483, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 640704, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 65, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep03", "query_calls": 951, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7276370, "tokens_cache_write": 457835, "tokens_input": 195, "tokens_output": 15299, "tool_calls": 209, "tool_calls_by_class": { "product_builtin": 124, "product_semantic": 85 }, "tool_result_tokens": 227093, "tool_time_ms": 2774, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.1832058, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 482451, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 61, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep04", "query_calls": 1046, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6126072, "tokens_cache_write": 496705, "tokens_input": 183, "tokens_output": 12426, "tool_calls": 182, "tool_calls_by_class": { "product_builtin": 87, "product_semantic": 95 }, "tool_result_tokens": 190075, "tool_time_ms": 2016, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.2693864, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 518133, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 59, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep05", "query_calls": 942, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5077946, "tokens_cache_write": 401672, "tokens_input": 177, "tokens_output": 11457, "tool_calls": 173, "tool_calls_by_class": { "product_builtin": 87, "product_semantic": 86 }, "tool_result_tokens": 179872, "tool_time_ms": 1443, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.120855000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 556535, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 59, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep06", "query_calls": 841, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6236355, "tokens_cache_write": 472797, "tokens_input": 177, "tokens_output": 13081, "tool_calls": 183, "tool_calls_by_class": { "product_builtin": 103, "product_semantic": 80 }, "tool_result_tokens": 200084, "tool_time_ms": 1852, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.2061418, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 598135, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 62, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep07", "query_calls": 989, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4442732, "tokens_cache_write": 435457, "tokens_input": 186, "tokens_output": 12551, "tool_calls": 187, "tool_calls_by_class": { "product_builtin": 96, "product_semantic": 91 }, "tool_result_tokens": 168574, "tool_time_ms": 1728, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.2640444000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 583051, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 58, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep08", "query_calls": 887, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4023321, "tokens_cache_write": 277368, "tokens_input": 174, "tokens_output": 13359, "tool_calls": 182, "tool_calls_by_class": { "product_builtin": 102, "product_semantic": 80 }, "tool_result_tokens": 147611, "tool_time_ms": 1490, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.6570944, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 627157, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 60, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep09", "query_calls": 1194, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 4966176, "tokens_cache_write": 474468, "tokens_input": 3576, "tokens_output": 14199, "tool_calls": 196, "tool_calls_by_class": { "product_builtin": 92, "product_semantic": 104 }, "tool_result_tokens": 169861, "tool_time_ms": 1102, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 5.032206, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 637952, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 61, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos-hard/rep10", "query_calls": 846, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6103360, "tokens_cache_write": 465270, "tokens_input": 183, "tokens_output": 13189, "tool_calls": 166, "tool_calls_by_class": { "product_builtin": 88, "product_semantic": 78 }, "tool_result_tokens": 193439, "tool_time_ms": 2142, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2573556000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 355753, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 76, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep01", "query_calls": 733, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2713479, "tokens_cache_write": 200572, "tokens_input": 596, "tokens_output": 8336, "tool_calls": 118, "tool_calls_by_class": { "product_builtin": 61, "product_semantic": 57 }, "tool_result_tokens": 114655, "tool_time_ms": 444, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.9893586, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 407182, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep02", "query_calls": 607, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3485859, "tokens_cache_write": 283171, "tokens_input": 225, "tokens_output": 8913, "tool_calls": 107, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 47 }, "tool_result_tokens": 172213, "tool_time_ms": 1087, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5385838, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 355825, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 75, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep03", "query_calls": 729, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2295497, "tokens_cache_write": 289965, "tokens_input": 225, "tokens_output": 8483, "tool_calls": 107, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 53 }, "tool_result_tokens": 97026, "tool_time_ms": 448, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6744329999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 370712, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 78, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep04", "query_calls": 677, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3301880, "tokens_cache_write": 236361, "tokens_input": 234, "tokens_output": 8547, "tool_calls": 104, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 54 }, "tool_result_tokens": 126769, "tool_time_ms": 1795, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5663142, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 391019, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep05", "query_calls": 596, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2687448, "tokens_cache_write": 261855, "tokens_input": 225, "tokens_output": 9058, "tool_calls": 116, "tool_calls_by_class": { "product_builtin": 72, "product_semantic": 44 }, "tool_result_tokens": 131773, "tool_time_ms": 572, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8171228, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 403748, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 74, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep06", "query_calls": 631, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3111912, "tokens_cache_write": 280586, "tokens_input": 222, "tokens_output": 8427, "tool_calls": 101, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 46 }, "tool_result_tokens": 126407, "tool_time_ms": 922, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2978216000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 500540, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 78, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep07", "query_calls": 762, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2677589, "tokens_cache_write": 209398, "tokens_input": 234, "tokens_output": 8943, "tool_calls": 116, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 59 }, "tool_result_tokens": 102507, "tool_time_ms": 1385, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3727844, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 488588, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 74, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep08", "query_calls": 683, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2546016, "tokens_cache_write": 236510, "tokens_input": 222, "tokens_output": 8547, "tool_calls": 103, "tool_calls_by_class": { "product_builtin": 55, "product_semantic": 48 }, "tool_result_tokens": 119006, "tool_time_ms": 910, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3411018000000006, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 510541, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 75, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep09", "query_calls": 619, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2897067, "tokens_cache_write": 201067, "tokens_input": 225, "tokens_output": 8802, "tool_calls": 105, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 46 }, "tool_result_tokens": 123735, "tool_time_ms": 1069, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.4349994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 480118, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 72, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/MultiTurnExam/memos/rep10", "query_calls": 666, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2297676, "tokens_cache_write": 271481, "tokens_input": 216, "tokens_output": 7883, "tool_calls": 104, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 53 }, "tool_result_tokens": 109141, "tool_time_ms": 1213, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3160088, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 551466, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 118, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep01", "query_calls": 85, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 849767, "tokens_cache_write": 328414, "tokens_input": 3843, "tokens_output": 15933, "tool_calls": 166, "tool_calls_by_class": { "product_builtin": 81, "product_semantic": 85 }, "tool_result_tokens": 228750, "tool_time_ms": 2029, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.065371, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 545219, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 113, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep02", "query_calls": 88, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 916575, "tokens_cache_write": 271093, "tokens_input": 339, "tokens_output": 17096, "tool_calls": 190, "tool_calls_by_class": { "product_builtin": 102, "product_semantic": 88 }, "tool_result_tokens": 225224, "tool_time_ms": 1960, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5659618, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 594318, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 129, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep03", "query_calls": 97, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1149047, "tokens_cache_write": 347687, "tokens_input": 387, "tokens_output": 18318, "tool_calls": 194, "tool_calls_by_class": { "product_builtin": 97, "product_semantic": 97 }, "tool_result_tokens": 248808, "tool_time_ms": 2696, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9491214, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 558939, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 118, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep04", "query_calls": 102, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 806151, "tokens_cache_write": 254749, "tokens_input": 354, "tokens_output": 17575, "tool_calls": 183, "tool_calls_by_class": { "product_builtin": 81, "product_semantic": 102 }, "tool_result_tokens": 183204, "tool_time_ms": 2212, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.4813634, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 534335, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": 112, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep05", "query_calls": 82, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 960926, "tokens_cache_write": 353077, "tokens_input": 1547, "tokens_output": 16271, "tool_calls": 181, "tool_calls_by_class": { "product_builtin": 99, "product_semantic": 82 }, "tool_result_tokens": 275820, "tool_time_ms": 2628, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2796272, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 587002, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 120, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep06", "query_calls": 81, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1045118, "tokens_cache_write": 301876, "tokens_input": 360, "tokens_output": 17538, "tool_calls": 180, "tool_calls_by_class": { "product_builtin": 99, "product_semantic": 81 }, "tool_result_tokens": 244968, "tool_time_ms": 2750, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.9310374, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 578503, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 124, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep07", "query_calls": 79, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1226711, "tokens_cache_write": 419785, "tokens_input": 372, "tokens_output": 16997, "tool_calls": 176, "tool_calls_by_class": { "product_builtin": 97, "product_semantic": 79 }, "tool_result_tokens": 264888, "tool_time_ms": 2829, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2512537999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 572945, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 123, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep08", "query_calls": 90, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 987007, "tokens_cache_write": 301319, "tokens_input": 369, "tokens_output": 17419, "tool_calls": 183, "tool_calls_by_class": { "product_builtin": 93, "product_semantic": 90 }, "tool_result_tokens": 233401, "tool_time_ms": 2396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.0377974, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 551836, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 125, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep09", "query_calls": 85, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 900556, "tokens_cache_write": 271295, "tokens_input": 375, "tokens_output": 15980, "tool_calls": 175, "tool_calls_by_class": { "product_builtin": 90, "product_semantic": 85 }, "tool_result_tokens": 208224, "tool_time_ms": 3455, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9040956000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 557141, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 122, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos-hard/rep10", "query_calls": 94, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 933104, "tokens_cache_write": 239866, "tokens_input": 366, "tokens_output": 16503, "tool_calls": 189, "tool_calls_by_class": { "product_builtin": 95, "product_semantic": 94 }, "tool_result_tokens": 212876, "tool_time_ms": 2523, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.647553, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 528155, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 137, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep01", "query_calls": 71, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 759925, "tokens_cache_write": 216127, "tokens_input": 452, "tokens_output": 13057, "tool_calls": 160, "tool_calls_by_class": { "product_builtin": 89, "product_semantic": 71 }, "tool_result_tokens": 169343, "tool_time_ms": 843, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.3795015999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 481254, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 128, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep02", "query_calls": 63, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 663139, "tokens_cache_write": 172514, "tokens_input": 384, "tokens_output": 12507, "tool_calls": 143, "tool_calls_by_class": { "product_builtin": 80, "product_semantic": 63 }, "tool_result_tokens": 144524, "tool_time_ms": 1710, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.766523, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 537501, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 142, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep03", "query_calls": 69, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 838850, "tokens_cache_write": 232023, "tokens_input": 662, "tokens_output": 13411, "tool_calls": 159, "tool_calls_by_class": { "product_builtin": 90, "product_semantic": 69 }, "tool_result_tokens": 169285, "tool_time_ms": 3124, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.5057994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 504792, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 137, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep04", "query_calls": 75, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 783701, "tokens_cache_write": 181423, "tokens_input": 411, "tokens_output": 14178, "tool_calls": 176, "tool_calls_by_class": { "product_builtin": 101, "product_semantic": 75 }, "tool_result_tokens": 147318, "tool_time_ms": 1367, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.5193106, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 492394, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 133, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep05", "query_calls": 74, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 762024, "tokens_cache_write": 191229, "tokens_input": 399, "tokens_output": 12838, "tool_calls": 156, "tool_calls_by_class": { "product_builtin": 82, "product_semantic": 74 }, "tool_result_tokens": 155602, "tool_time_ms": 994, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.5935542, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 515276, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 137, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep06", "query_calls": 66, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 797903, "tokens_cache_write": 197953, "tokens_input": 5147, "tokens_output": 13202, "tool_calls": 162, "tool_calls_by_class": { "product_builtin": 96, "product_semantic": 66 }, "tool_result_tokens": 154969, "tool_time_ms": 2559, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.5809506, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 536014, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 142, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep07", "query_calls": 64, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 900744, "tokens_cache_write": 185901, "tokens_input": 547, "tokens_output": 14448, "tool_calls": 167, "tool_calls_by_class": { "product_builtin": 103, "product_semantic": 64 }, "tool_result_tokens": 169167, "tool_time_ms": 2800, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9334220000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 517997, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 132, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep08", "query_calls": 64, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 978140, "tokens_cache_write": 252674, "tokens_input": 554, "tokens_output": 13829, "tool_calls": 167, "tool_calls_by_class": { "product_builtin": 103, "product_semantic": 64 }, "tool_result_tokens": 217890, "tool_time_ms": 1964, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.4576378, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 503603, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 135, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep09", "query_calls": 66, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 781272, "tokens_cache_write": 178581, "tokens_input": 446, "tokens_output": 12522, "tool_calls": 149, "tool_calls_by_class": { "product_builtin": 83, "product_semantic": 66 }, "tool_result_tokens": 164818, "tool_time_ms": 2037, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.3762772, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 489103, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 131, "model": "gpt-5.6-sol", "path": "runs/cg-ask-fz-009/gpt-5.6-sol/ZeroShotExam/memos/rep10", "query_calls": 68, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 798713, "tokens_cache_write": 157340, "tokens_input": 393, "tokens_output": 13426, "tool_calls": 156, "tool_calls_by_class": { "product_builtin": 88, "product_semantic": 68 }, "tool_result_tokens": 145961, "tool_time_ms": 1565, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.6947842, "denominator": 20, "duration_basis": "run", "duration_ms": 81714, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep01", "query_calls": 45, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 457616, "tokens_cache_write": 207402, "tokens_input": 30, "tokens_output": 7058, "tool_calls": 94, "tool_calls_by_class": { "product_builtin": 49, "product_semantic": 45 }, "tool_result_tokens": 103910, "tool_time_ms": 936, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5693498, "denominator": 20, "duration_basis": "run", "duration_ms": 118687, "exam": "HumanExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep02", "query_calls": 52, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 735389, "tokens_cache_write": 122848, "tokens_input": 36, "tokens_output": 9590, "tool_calls": 117, "tool_calls_by_class": { "product_builtin": 65, "product_semantic": 52 }, "tool_result_tokens": 100978, "tool_time_ms": 1174, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.45170619999999995, "denominator": 20, "duration_basis": "run", "duration_ms": 65445, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep03", "query_calls": 33, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 367456, "tokens_cache_write": 126314, "tokens_input": 21, "tokens_output": 5199, "tool_calls": 63, "tool_calls_by_class": { "product_builtin": 30, "product_semantic": 33 }, "tool_result_tokens": 109562, "tool_time_ms": 532, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5541421, "denominator": 20, "duration_basis": "run", "duration_ms": 87272, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep04", "query_calls": 34, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 632618, "tokens_cache_write": 134901, "tokens_input": 27, "tokens_output": 7526, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 34 }, "tool_result_tokens": 115634, "tool_time_ms": 1387, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5925713, "denominator": 20, "duration_basis": "run", "duration_ms": 114679, "exam": "HumanExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep05", "query_calls": 50, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 718639, "tokens_cache_write": 130971, "tokens_input": 30, "tokens_output": 10113, "tool_calls": 114, "tool_calls_by_class": { "product_builtin": 64, "product_semantic": 50 }, "tool_result_tokens": 112738, "tool_time_ms": 848, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.6302387, "denominator": 20, "duration_basis": "run", "duration_ms": 115161, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep06", "query_calls": 63, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 871691, "tokens_cache_write": 133441, "tokens_input": 33, "tokens_output": 10186, "tool_calls": 132, "tool_calls_by_class": { "product_builtin": 69, "product_semantic": 63 }, "tool_result_tokens": 112848, "tool_time_ms": 802, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5255398, "denominator": 20, "duration_basis": "run", "duration_ms": 48326, "exam": "HumanExam", "exam_epoch": 4, "fails": 8, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep07", "query_calls": 21, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 12, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 336169, "tokens_cache_write": 162836, "tokens_input": 18, "tokens_output": 4265, "tool_calls": 48, "tool_calls_by_class": { "product_builtin": 27, "product_semantic": 21 }, "tool_result_tokens": 144726, "tool_time_ms": 1144, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5466653, "denominator": 20, "duration_basis": "run", "duration_ms": 105222, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep08", "query_calls": 56, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 792849, "tokens_cache_write": 112523, "tokens_input": 36, "tokens_output": 8893, "tool_calls": 122, "tool_calls_by_class": { "product_builtin": 66, "product_semantic": 56 }, "tool_result_tokens": 92536, "tool_time_ms": 1248, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.7422178, "denominator": 20, "duration_basis": "run", "duration_ms": 79471, "exam": "HumanExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep09", "query_calls": 31, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 386169, "tokens_cache_write": 232120, "tokens_input": 24, "tokens_output": 7053, "tool_calls": 83, "tool_calls_by_class": { "product_builtin": 52, "product_semantic": 31 }, "tool_result_tokens": 117252, "tool_time_ms": 999, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.6269929000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 102291, "exam": "HumanExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos-hard/rep10", "query_calls": 52, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 748457, "tokens_cache_write": 148743, "tokens_input": 30, "tokens_output": 8782, "tool_calls": 115, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 52 }, "tool_result_tokens": 124498, "tool_time_ms": 1180, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.335814, "denominator": 30, "duration_basis": "run", "duration_ms": 64293, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep01", "query_calls": 26, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 246670, "tokens_cache_write": 82948, "tokens_input": 21, "tokens_output": 6589, "tool_calls": 71, "tool_calls_by_class": { "product_builtin": 45, "product_semantic": 26 }, "tool_result_tokens": 68765, "tool_time_ms": 350, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.42586660000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 64827, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep02", "query_calls": 19, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 369428, "tokens_cache_write": 110850, "tokens_input": 24, "tokens_output": 6234, "tool_calls": 69, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 19 }, "tool_result_tokens": 94372, "tool_time_ms": 352, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5880004, "denominator": 30, "duration_basis": "run", "duration_ms": 72517, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep03", "query_calls": 21, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 527682, "tokens_cache_write": 158632, "tokens_input": 24, "tokens_output": 7153, "tool_calls": 81, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 21 }, "tool_result_tokens": 131928, "tool_time_ms": 329, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3322003, "denominator": 30, "duration_basis": "run", "duration_ms": 127535, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep04", "query_calls": 19, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 346504, "tokens_cache_write": 74243, "tokens_input": 24, "tokens_output": 6437, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 51, "product_semantic": 19 }, "tool_result_tokens": 76365, "tool_time_ms": 372, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3930022, "denominator": 30, "duration_basis": "run", "duration_ms": 79600, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep05", "query_calls": 35, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 287016, "tokens_cache_write": 103594, "tokens_input": 21, "tokens_output": 6381, "tool_calls": 78, "tool_calls_by_class": { "product_builtin": 43, "product_semantic": 35 }, "tool_result_tokens": 87682, "tool_time_ms": 197, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3530941, "denominator": 30, "duration_basis": "run", "duration_ms": 63756, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep06", "query_calls": 13, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 202268, "tokens_cache_write": 95301, "tokens_input": 18, "tokens_output": 6196, "tool_calls": 59, "tool_calls_by_class": { "product_builtin": 46, "product_semantic": 13 }, "tool_result_tokens": 81987, "tool_time_ms": 361, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.46044399999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 82100, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep07", "query_calls": 27, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 439290, "tokens_cache_write": 111496, "tokens_input": 27, "tokens_output": 7816, "tool_calls": 86, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 27 }, "tool_result_tokens": 91877, "tool_time_ms": 361, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.4145031, "denominator": 30, "duration_basis": "run", "duration_ms": 64561, "exam": "HumanExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep08", "query_calls": 18, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 389463, "tokens_cache_write": 106017, "tokens_input": 24, "tokens_output": 5960, "tool_calls": 70, "tool_calls_by_class": { "product_builtin": 52, "product_semantic": 18 }, "tool_result_tokens": 90090, "tool_time_ms": 286, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.4458254, "denominator": 30, "duration_basis": "run", "duration_ms": 81986, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep09", "query_calls": 22, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 443502, "tokens_cache_write": 106550, "tokens_input": 27, "tokens_output": 7558, "tool_calls": 81, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 22 }, "tool_result_tokens": 86634, "tool_time_ms": 272, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5485324, "denominator": 30, "duration_basis": "run", "duration_ms": 58748, "exam": "HumanExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": null, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/HumanExam/memos/rep10", "query_calls": 15, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 188362, "tokens_cache_write": 176324, "tokens_input": 21, "tokens_output": 5834, "tool_calls": 68, "tool_calls_by_class": { "product_builtin": 53, "product_semantic": 15 }, "tool_result_tokens": 76487, "tool_time_ms": 428, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9791075, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 370612, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 59, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep01", "query_calls": 987, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 5384130, "tokens_cache_write": 305691, "tokens_input": 177, "tokens_output": 11475, "tool_calls": 164, "tool_calls_by_class": { "product_builtin": 77, "product_semantic": 87 }, "tool_result_tokens": 166784, "tool_time_ms": 1789, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.5991305000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1201322, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 56, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep02", "query_calls": 900, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6303295, "tokens_cache_write": 871683, "tokens_input": 168, "tokens_output": 13244, "tool_calls": 180, "tool_calls_by_class": { "product_builtin": 98, "product_semantic": 82 }, "tool_result_tokens": 240321, "tool_time_ms": 2563, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.924767, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 405016, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 56, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep03", "query_calls": 611, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7651300, "tokens_cache_write": 498806, "tokens_input": 168, "tokens_output": 12263, "tool_calls": 181, "tool_calls_by_class": { "product_builtin": 123, "product_semantic": 58 }, "tool_result_tokens": 269134, "tool_time_ms": 2255, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5131752, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1220101, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 55, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep04", "query_calls": 863, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 7028566, "tokens_cache_write": 381528, "tokens_input": 165, "tokens_output": 12776, "tool_calls": 176, "tool_calls_by_class": { "product_builtin": 97, "product_semantic": 79 }, "tool_result_tokens": 244326, "tool_time_ms": 2202, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5525077000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1192414, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 59, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep05", "query_calls": 792, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6804736, "tokens_cache_write": 418105, "tokens_input": 177, "tokens_output": 12162, "tool_calls": 182, "tool_calls_by_class": { "product_builtin": 114, "product_semantic": 68 }, "tool_result_tokens": 229520, "tool_time_ms": 2164, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6152947, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 372879, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 55, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep06", "query_calls": 768, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6328081, "tokens_cache_write": 487573, "tokens_input": 165, "tokens_output": 10868, "tool_calls": 153, "tool_calls_by_class": { "product_builtin": 88, "product_semantic": 65 }, "tool_result_tokens": 235677, "tool_time_ms": 2107, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7094810000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1211550, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 57, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep07", "query_calls": 829, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6578365, "tokens_cache_write": 497012, "tokens_input": 171, "tokens_output": 12578, "tool_calls": 173, "tool_calls_by_class": { "product_builtin": 99, "product_semantic": 74 }, "tool_result_tokens": 233814, "tool_time_ms": 2472, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8564109, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1197212, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 57, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep08", "query_calls": 819, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6633572, "tokens_cache_write": 554401, "tokens_input": 171, "tokens_output": 11946, "tool_calls": 197, "tool_calls_by_class": { "product_builtin": 121, "product_semantic": 76 }, "tool_result_tokens": 224061, "tool_time_ms": 2418, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.9530739, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1256545, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 59, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep09", "query_calls": 533, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 6840422, "tokens_cache_write": 578611, "tokens_input": 177, "tokens_output": 11509, "tool_calls": 160, "tool_calls_by_class": { "product_builtin": 112, "product_semantic": 48 }, "tool_result_tokens": 237279, "tool_time_ms": 2262, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.0221906, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1402558, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 60, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos-hard/rep10", "query_calls": 800, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 8125383, "tokens_cache_write": 486028, "tokens_input": 16658, "tokens_output": 12394, "tool_calls": 181, "tool_calls_by_class": { "product_builtin": 108, "product_semantic": 73 }, "tool_result_tokens": 244598, "tool_time_ms": 1848, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.0375485, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 620813, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 69, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep01", "query_calls": 347, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2478160, "tokens_cache_write": 183385, "tokens_input": 207, "tokens_output": 6920, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 69, "product_semantic": 22 }, "tool_result_tokens": 94341, "tool_time_ms": 690, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2895313000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 613917, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 73, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep02", "query_calls": 477, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2454844, "tokens_cache_write": 287221, "tokens_input": 259, "tokens_output": 6666, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 59, "product_semantic": 32 }, "tool_result_tokens": 77020, "tool_time_ms": 548, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1145154, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 709464, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 72, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep03", "query_calls": 483, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2839792, "tokens_cache_write": 182882, "tokens_input": 216, "tokens_output": 7410, "tool_calls": 94, "tool_calls_by_class": { "product_builtin": 65, "product_semantic": 29 }, "tool_result_tokens": 103463, "tool_time_ms": 562, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7428203999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 752254, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 73, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep04", "query_calls": 485, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3044337, "tokens_cache_write": 419422, "tokens_input": 219, "tokens_output": 7080, "tool_calls": 90, "tool_calls_by_class": { "product_builtin": 58, "product_semantic": 32 }, "tool_result_tokens": 142131, "tool_time_ms": 452, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2291605, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 790856, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 74, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep05", "query_calls": 716, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 3312665, "tokens_cache_write": 191083, "tokens_input": 222, "tokens_output": 7373, "tool_calls": 102, "tool_calls_by_class": { "product_builtin": 57, "product_semantic": 45 }, "tool_result_tokens": 125427, "tool_time_ms": 688, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2053888000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 678209, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 70, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep06", "query_calls": 656, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2852484, "tokens_cache_write": 220640, "tokens_input": 210, "tokens_output": 6906, "tool_calls": 97, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 43 }, "tool_result_tokens": 110296, "tool_time_ms": 501, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1701223, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 642881, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 70, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep07", "query_calls": 609, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2169014, "tokens_cache_write": 258715, "tokens_input": 210, "tokens_output": 7426, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 54, "product_semantic": 39 }, "tool_result_tokens": 87687, "tool_time_ms": 681, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9524642000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 266447, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 71, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep08", "query_calls": 629, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2516301, "tokens_cache_write": 141404, "tokens_input": 213, "tokens_output": 7939, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 50, "product_semantic": 41 }, "tool_result_tokens": 84873, "tool_time_ms": 545, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.2482292, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 970706, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 69, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep09", "query_calls": 484, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2884006, "tokens_cache_write": 235348, "tokens_input": 207, "tokens_output": 6887, "tool_calls": 91, "tool_calls_by_class": { "product_builtin": 63, "product_semantic": 28 }, "tool_result_tokens": 122637, "tool_time_ms": 1859, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1089503, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 937904, "exam": "MultiTurnExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 72, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/MultiTurnExam/memos/rep10", "query_calls": 542, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 2542464, "tokens_cache_write": 205215, "tokens_input": 216, "tokens_output": 7249, "tool_calls": 93, "tool_calls_by_class": { "product_builtin": 60, "product_semantic": 33 }, "tool_result_tokens": 113936, "tool_time_ms": 1336, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.366845, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 426353, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 118, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep01", "query_calls": 87, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 970450, "tokens_cache_write": 399622, "tokens_input": 354, "tokens_output": 14416, "tool_calls": 190, "tool_calls_by_class": { "product_builtin": 103, "product_semantic": 87 }, "tool_result_tokens": 264515, "tool_time_ms": 3509, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9756403, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 441657, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 122, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep02", "query_calls": 104, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 835484, "tokens_cache_write": 253731, "tokens_input": 366, "tokens_output": 14457, "tool_calls": 188, "tool_calls_by_class": { "product_builtin": 84, "product_semantic": 104 }, "tool_result_tokens": 203288, "tool_time_ms": 3552, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.3240862, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 471593, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 126, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep03", "query_calls": 92, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1269896, "tokens_cache_write": 354166, "tokens_input": 414, "tokens_output": 15322, "tool_calls": 190, "tool_calls_by_class": { "product_builtin": 98, "product_semantic": 92 }, "tool_result_tokens": 230618, "tool_time_ms": 3402, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9969760000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 431826, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 121, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep04", "query_calls": 103, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 969455, "tokens_cache_write": 250470, "tokens_input": 363, "tokens_output": 14682, "tool_calls": 189, "tool_calls_by_class": { "product_builtin": 86, "product_semantic": 103 }, "tool_result_tokens": 199465, "tool_time_ms": 3025, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.090311, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 426559, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 3, "family": "product", "llm_calls": 122, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep05", "query_calls": 90, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1048315, "tokens_cache_write": 282632, "tokens_input": 1354, "tokens_output": 14280, "tool_calls": 193, "tool_calls_by_class": { "product_builtin": 103, "product_semantic": 90 }, "tool_result_tokens": 236291, "tool_time_ms": 2742, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9819856, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 448904, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 128, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep06", "query_calls": 114, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1077933, "tokens_cache_write": 236446, "tokens_input": 384, "tokens_output": 14543, "tool_calls": 199, "tool_calls_by_class": { "product_builtin": 85, "product_semantic": 114 }, "tool_result_tokens": 206750, "tool_time_ms": 3336, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8530381, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 377035, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 4, "family": "product", "llm_calls": 115, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep07", "query_calls": 87, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 815668, "tokens_cache_write": 213809, "tokens_input": 345, "tokens_output": 12891, "tool_calls": 180, "tool_calls_by_class": { "product_builtin": 93, "product_semantic": 87 }, "tool_result_tokens": 189156, "tool_time_ms": 4046, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1368804, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 406249, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 116, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep08", "query_calls": 91, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1066827, "tokens_cache_write": 298822, "tokens_input": 348, "tokens_output": 14647, "tool_calls": 198, "tool_calls_by_class": { "product_builtin": 107, "product_semantic": 91 }, "tool_result_tokens": 250739, "tool_time_ms": 3019, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9764969000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 393009, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 6, "family": "product", "llm_calls": 121, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep09", "query_calls": 104, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1197147, "tokens_cache_write": 227591, "tokens_input": 363, "tokens_output": 13947, "tool_calls": 178, "tool_calls_by_class": { "product_builtin": 74, "product_semantic": 104 }, "tool_result_tokens": 198710, "tool_time_ms": 2955, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.1528521999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 429810, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 5, "family": "product", "llm_calls": 124, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos-hard/rep10", "query_calls": 102, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 15, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1167871, "tokens_cache_write": 285732, "tokens_input": 372, "tokens_output": 17017, "tool_calls": 208, "tool_calls_by_class": { "product_builtin": 106, "product_semantic": 102 }, "tool_result_tokens": 234229, "tool_time_ms": 2780, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.0520686000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 370148, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 134, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep01", "query_calls": 72, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1049078, "tokens_cache_write": 285018, "tokens_input": 402, "tokens_output": 10742, "tool_calls": 166, "tool_calls_by_class": { "product_builtin": 94, "product_semantic": 72 }, "tool_result_tokens": 219792, "tool_time_ms": 1133, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.729055, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 344294, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 132, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep02", "query_calls": 72, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 726365, "tokens_cache_write": 186108, "tokens_input": 396, "tokens_output": 9810, "tool_calls": 151, "tool_calls_by_class": { "product_builtin": 79, "product_semantic": 72 }, "tool_result_tokens": 144850, "tool_time_ms": 1193, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.6886777, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 332875, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 122, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep03", "query_calls": 63, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 602861, "tokens_cache_write": 181267, "tokens_input": 409, "tokens_output": 9510, "tool_calls": 140, "tool_calls_by_class": { "product_builtin": 77, "product_semantic": 63 }, "tool_result_tokens": 137574, "tool_time_ms": 1860, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8031848, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 387101, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 0, "family": "product", "llm_calls": 145, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep04", "query_calls": 70, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 898059, "tokens_cache_write": 198734, "tokens_input": 435, "tokens_output": 10489, "tool_calls": 162, "tool_calls_by_class": { "product_builtin": 92, "product_semantic": 70 }, "tool_result_tokens": 171613, "tool_time_ms": 3011, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9029739000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 412117, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 152, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep05", "query_calls": 67, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1012772, "tokens_cache_write": 231875, "tokens_input": 456, "tokens_output": 9985, "tool_calls": 155, "tool_calls_by_class": { "product_builtin": 88, "product_semantic": 67 }, "tool_result_tokens": 190137, "tool_time_ms": 2916, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.0019632, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 389705, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 134, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep06", "query_calls": 72, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 917216, "tokens_cache_write": 275400, "tokens_input": 402, "tokens_output": 10768, "tool_calls": 161, "tool_calls_by_class": { "product_builtin": 89, "product_semantic": 72 }, "tool_result_tokens": 236557, "tool_time_ms": 1256, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.6532557000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 378175, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 135, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep07", "query_calls": 76, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 750156, "tokens_cache_write": 153201, "tokens_input": 405, "tokens_output": 9951, "tool_calls": 158, "tool_calls_by_class": { "product_builtin": 82, "product_semantic": 76 }, "tool_result_tokens": 138125, "tool_time_ms": 1006, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.7554062, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 377533, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 129, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep08", "query_calls": 65, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 813021, "tokens_cache_write": 190232, "tokens_input": 387, "tokens_output": 9704, "tool_calls": 150, "tool_calls_by_class": { "product_builtin": 85, "product_semantic": 65 }, "tool_result_tokens": 167424, "tool_time_ms": 3101, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.8732827999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 389984, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 2, "family": "product", "llm_calls": 137, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep09", "query_calls": 72, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 862484, "tokens_cache_write": 229480, "tokens_input": 411, "tokens_output": 10522, "tool_calls": 170, "tool_calls_by_class": { "product_builtin": 98, "product_semantic": 72 }, "tool_result_tokens": 201184, "tool_time_ms": 1140, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cg-ask-fz-009", "arm_label": "Corograph Ask (Ask) @ fz-009", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.9351564, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 406631, "exam": "ZeroShotExam", "exam_epoch": 4, "fails": 1, "family": "product", "llm_calls": 134, "model": "gpt-5.6-terra", "path": "runs/cg-ask-fz-009/gpt-5.6-terra/ZeroShotExam/memos/rep10", "query_calls": 64, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cg-ask-fz-009|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "corograph", "token_source": "ledger_all_calls", "tokens_cache_read": 1008447, "tokens_cache_write": 245022, "tokens_input": 402, "tokens_output": 10009, "tool_calls": 147, "tool_calls_by_class": { "product_builtin": 83, "product_semantic": 64 }, "tool_result_tokens": 214068, "tool_time_ms": 1570, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.19611, "cost_usd": 4.19611, "denominator": 20, "duration_basis": "run", "duration_ms": 305861, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 26, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1906860, "tokens_cache_write": 61998, "tokens_input": 49, "tokens_output": 20976, "tool_calls": 31, "tool_calls_by_class": { "cc_builtin": 29, "vendor_mcp": 2 }, "tool_result_tokens": 22507, "tool_time_ms": 3694, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.147133999999999, "cost_usd": 5.147133999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 275899, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 17, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1883444, "tokens_cache_write": 114866, "tokens_input": 32, "tokens_output": 19321, "tool_calls": 23, "tool_calls_by_class": { "cc_builtin": 15, "vendor_mcp": 8 }, "tool_result_tokens": 56313, "tool_time_ms": 2543, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.082481, "cost_usd": 4.082481, "denominator": 20, "duration_basis": "run", "duration_ms": 332881, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 20, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1501791, "tokens_cache_write": 69126, "tokens_input": 37, "tokens_output": 23956, "tool_calls": 36, "tool_calls_by_class": { "cc_builtin": 34, "vendor_mcp": 2 }, "tool_result_tokens": 24520, "tool_time_ms": 2003, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.349524000000001, "cost_usd": 5.349524000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 364066, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 34, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2628024, "tokens_cache_write": 73793, "tokens_input": 64, "tokens_output": 24900, "tool_calls": 42, "tool_calls_by_class": { "cc_builtin": 40, "vendor_mcp": 2 }, "tool_result_tokens": 26748, "tool_time_ms": 2061, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.903625000000001, "cost_usd": 5.903625000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 308920, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 37, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3080295, "tokens_cache_write": 82773, "tokens_input": 9937, "tokens_output": 21370, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 42, "vendor_mcp": 3 }, "tool_result_tokens": 34132, "tool_time_ms": 2851, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.494047000000004, "cost_usd": 13.494047000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 511336, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 79, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9319047, "tokens_cache_write": 140876, "tokens_input": 153, "tokens_output": 27119, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 53, "vendor_mcp": 6 }, "tool_result_tokens": 52623, "tool_time_ms": 4704, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.202424999999998, "cost_usd": 10.202424999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 527584, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 74, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6747225, "tokens_cache_write": 104968, "tokens_input": 144, "tokens_output": 27088, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 51, "vendor_mcp": 3 }, "tool_result_tokens": 31142, "tool_time_ms": 8033, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.928716999999999, "cost_usd": 10.928716999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 754959, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 79, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6485733, "tokens_cache_write": 299046, "tokens_input": 153, "tokens_output": 27386, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 57, "vendor_mcp": 2 }, "tool_result_tokens": 25981, "tool_time_ms": 13004, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.626445499999997, "cost_usd": 12.626445499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 517110, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 73, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7899058, "tokens_cache_write": 266201, "tokens_input": 139, "tokens_output": 27759, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 49, "vendor_mcp": 5 }, "tool_result_tokens": 47480, "tool_time_ms": 4851, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.236346000000001, "cost_usd": 10.236346000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 461330, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 74, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6967132, "tokens_cache_write": 113782, "tokens_input": 143, "tokens_output": 24537, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 50, "vendor_mcp": 4 }, "tool_result_tokens": 37359, "tool_time_ms": 3954, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.652519, "cost_usd": 7.652519, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 355172, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5184249, "tokens_cache_write": 77690, "tokens_input": 157, "tokens_output": 18258, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 13174, "tool_time_ms": 1917, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.232122, "cost_usd": 8.232122, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 521865, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5665872, "tokens_cache_write": 81705, "tokens_input": 165, "tokens_output": 18610, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 15076, "tool_time_ms": 2175, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.059508, "cost_usd": 7.059508, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 329021, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 74, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4720258, "tokens_cache_write": 73162, "tokens_input": 146, "tokens_output": 17491, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 11088, "tool_time_ms": 2181, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.5669309999999985, "cost_usd": 7.5669309999999985, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 382694, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5187351, "tokens_cache_write": 75632, "tokens_input": 154, "tokens_output": 17308, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 12390, "tool_time_ms": 2471, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.334676, "cost_usd": 7.334676, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 360408, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 76, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4896196, "tokens_cache_write": 76907, "tokens_input": 149, "tokens_output": 17977, "tool_calls": 46, "tool_calls_by_class": { "cc_builtin": 46 }, "tool_result_tokens": 12638, "tool_time_ms": 2248, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.21083125, "cost_usd": 10.21083125, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 579423, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3017895, "tokens_cache_write": 310660, "tokens_input": 6300, "tokens_output": 35120, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 55, "vendor_mcp": 17 }, "tool_result_tokens": 124902, "tool_time_ms": 5454, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.880367500000002, "cost_usd": 10.880367500000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 569186, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 81, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2970779, "tokens_cache_write": 330520, "tokens_input": 159, "tokens_output": 34182, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 52, "vendor_mcp": 19 }, "tool_result_tokens": 134503, "tool_time_ms": 6046, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.0649425, "cost_usd": 10.0649425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 595140, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 85, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3039036, "tokens_cache_write": 305214, "tokens_input": 169, "tokens_output": 34601, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 17 }, "tool_result_tokens": 126395, "tool_time_ms": 5033, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.155518499999998, "cost_usd": 11.155518499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 650623, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 95, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3486973, "tokens_cache_write": 311348, "tokens_input": 318, "tokens_output": 38186, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 67, "vendor_mcp": 17 }, "tool_result_tokens": 126102, "tool_time_ms": 5678, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.141658499999997, "cost_usd": 11.141658499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 638599, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 91, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3318944, "tokens_cache_write": 342128, "tokens_input": 215, "tokens_output": 38786, "tool_calls": 80, "tool_calls_by_class": { "cc_builtin": 62, "vendor_mcp": 18 }, "tool_result_tokens": 142505, "tool_time_ms": 5939, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3009830000000004, "cost_usd": 2.3009830000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 65045, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 17, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 500093, "tokens_cache_write": 83230, "tokens_input": 34, "tokens_output": 2719, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 8, "vendor_mcp": 2 }, "tool_result_tokens": 12907, "tool_time_ms": 591, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6969640000000001, "cost_usd": 1.6969640000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 85808, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 19, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 590984, "tokens_cache_write": 47710, "tokens_input": 38, "tokens_output": 3028, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 11, "vendor_mcp": 1 }, "tool_result_tokens": 6714, "tool_time_ms": 512, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.686691, "cost_usd": 1.686691, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 70594, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 18, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 557561, "tokens_cache_write": 49431, "tokens_input": 36, "tokens_output": 2803, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 10, "vendor_mcp": 1 }, "tool_result_tokens": 8034, "tool_time_ms": 526, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.80266, "cost_usd": 1.80266, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 85503, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 18, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 560620, "tokens_cache_write": 54879, "tokens_input": 36, "tokens_output": 2882, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 2 }, "tool_result_tokens": 12655, "tool_time_ms": 571, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.639622, "cost_usd": 1.639622, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 56669, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 17, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 524612, "tokens_cache_write": 49111, "tokens_input": 34, "tokens_output": 2649, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 9, "vendor_mcp": 1 }, "tool_result_tokens": 7593, "tool_time_ms": 511, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.736726, "cost_usd": 11.736726, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 478894, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2910486, "tokens_cache_write": 380960, "tokens_input": 174, "tokens_output": 24106, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 37, "vendor_mcp": 20 }, "tool_result_tokens": 136725, "tool_time_ms": 5073, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.996055, "cost_usd": 11.996055, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 523748, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3021255, "tokens_cache_write": 379052, "tokens_input": 176, "tokens_output": 27840, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 39, "vendor_mcp": 20 }, "tool_result_tokens": 134579, "tool_time_ms": 5130, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.695269000000001, "cost_usd": 11.695269000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 498808, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 83, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2814659, "tokens_cache_write": 382240, "tokens_input": 166, "tokens_output": 24683, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 33, "vendor_mcp": 21 }, "tool_result_tokens": 141394, "tool_time_ms": 5220, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 12.285183, "cost_usd": 12.285183, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 480732, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2896493, "tokens_cache_write": 403500, "tokens_input": 174, "tokens_output": 26339, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 37, "vendor_mcp": 22 }, "tool_result_tokens": 150783, "tool_time_ms": 5328, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 12.903524000000003, "cost_usd": 12.903524000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 551273, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 89, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3151754, "tokens_cache_write": 415240, "tokens_input": 177, "tokens_output": 28904, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 38, "vendor_mcp": 23 }, "tool_result_tokens": 158843, "tool_time_ms": 6523, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 14.362696999999999, "cost_usd": 14.362696999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 489451, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3193967, "tokens_cache_write": 486897, "tokens_input": 184, "tokens_output": 28579, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 38, "vendor_mcp": 26 }, "tool_result_tokens": 175825, "tool_time_ms": 7518, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 12.948776999999998, "cost_usd": 12.948776999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 481602, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-fable-5", "path": "runs/codegraph/claude-fable-5/ZeroShotExam/memos/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|12", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2925257, "tokens_cache_write": 433414, "tokens_input": 174, "tokens_output": 27070, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 34, "vendor_mcp": 23 }, "tool_result_tokens": 152621, "tool_time_ms": 9801, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9603465500000001, "cost_usd": 1.9603465500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 344216, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 12, "family": "cc_competitor", "llm_calls": 55, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 8, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5810875, "tokens_cache_write": 627520, "tokens_input": 400, "tokens_output": 14771, "tool_calls": 35, "tool_calls_by_class": { "vendor_mcp": 35 }, "tool_result_tokens": 215397, "tool_time_ms": 5083, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.844664, "cost_usd": 4.844664, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 696595, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 201, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 22522890, "tokens_cache_write": 1229371, "tokens_input": 1203, "tokens_output": 26486, "tool_calls": 181, "tool_calls_by_class": { "cc_builtin": 177, "vendor_mcp": 4 }, "tool_result_tokens": 76130, "tool_time_ms": 9601, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9511691000000002, "cost_usd": 1.9511691000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 344172, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 10, "family": "cc_competitor", "llm_calls": 56, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5974095, "tokens_cache_write": 614594, "tokens_input": 415, "tokens_output": 15724, "tool_calls": 36, "tool_calls_by_class": { "cc_builtin": 5, "vendor_mcp": 31 }, "tool_result_tokens": 213064, "tool_time_ms": 5232, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.7449848, "cost_usd": 5.7449848, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 697029, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 172, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 22584950, "tokens_cache_write": 1663970, "tokens_input": 1046, "tokens_output": 24759, "tool_calls": 152, "tool_calls_by_class": { "cc_builtin": 141, "vendor_mcp": 11 }, "tool_result_tokens": 103386, "tool_time_ms": 7271, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.21466455, "cost_usd": 6.21466455, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 792880, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 210, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 25091681, "tokens_cache_write": 1673555, "tokens_input": 1261, "tokens_output": 27578, "tool_calls": 190, "tool_calls_by_class": { "cc_builtin": 182, "harness_builtin": 1, "vendor_mcp": 7 }, "tool_result_tokens": 91932, "tool_time_ms": 7998, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8125747500000005, "cost_usd": 2.8125747500000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 481183, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 99, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10879122, "tokens_cache_write": 792527, "tokens_input": 636, "tokens_output": 17939, "tool_calls": 79, "tool_calls_by_class": { "cc_builtin": 53, "vendor_mcp": 26 }, "tool_result_tokens": 190279, "tool_time_ms": 5421, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.981760700000001, "cost_usd": 4.981760700000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 641264, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 160, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19831711, "tokens_cache_write": 1413697, "tokens_input": 979, "tokens_output": 25590, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 118, "harness_builtin": 2, "vendor_mcp": 20 }, "tool_result_tokens": 160068, "tool_time_ms": 6792, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.379028, "cost_usd": 6.379028, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 917555, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 246, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 29621647, "tokens_cache_write": 1609693, "tokens_input": 1476, "tokens_output": 31327, "tool_calls": 226, "tool_calls_by_class": { "cc_builtin": 217, "harness_builtin": 1, "vendor_mcp": 8 }, "tool_result_tokens": 107789, "tool_time_ms": 8279, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 1.1628289, "cost_usd": 1.1628289, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 223211, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 8, "family": "cc_competitor", "llm_calls": 38, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4696729, "tokens_cache_write": 315108, "tokens_input": 310, "tokens_output": 12526, "tool_calls": 18, "tool_calls_by_class": { "vendor_mcp": 18 }, "tool_result_tokens": 111380, "tool_time_ms": 2626, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7781711, "cost_usd": 2.7781711, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 533370, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 166, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15810421, "tokens_cache_write": 541108, "tokens_input": 1083, "tokens_output": 22766, "tool_calls": 136, "tool_calls_by_class": { "cc_builtin": 133, "vendor_mcp": 3 }, "tool_result_tokens": 52120, "tool_time_ms": 3643, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6629661999999996, "cost_usd": 1.6629661999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 335762, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 79, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10192542, "tokens_cache_write": 284116, "tokens_input": 570, "tokens_output": 14982, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 38, "vendor_mcp": 11 }, "tool_result_tokens": 80675, "tool_time_ms": 2528, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1122144, "cost_usd": 3.1122144, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 393627, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 112, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15958984, "tokens_cache_write": 717400, "tokens_input": 771, "tokens_output": 16149, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 69, "vendor_mcp": 13 }, "tool_result_tokens": 92570, "tool_time_ms": 3320, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1217329000000005, "cost_usd": 4.1217329000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 529441, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 141, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 18502779, "tokens_cache_write": 1081351, "tokens_input": 918, "tokens_output": 21567, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 101, "vendor_mcp": 10 }, "tool_result_tokens": 81448, "tool_time_ms": 3657, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2798253, "cost_usd": 2.2798253, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 409419, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 136, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13483143, "tokens_cache_write": 423686, "tokens_input": 894, "tokens_output": 16649, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 103, "vendor_mcp": 3 }, "tool_result_tokens": 52896, "tool_time_ms": 3757, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4405742999999998, "cost_usd": 2.4405742999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 345652, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 99, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12835553, "tokens_cache_write": 541129, "tokens_input": 691, "tokens_output": 14814, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 59, "vendor_mcp": 10 }, "tool_result_tokens": 81447, "tool_time_ms": 2776, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2235259000000003, "cost_usd": 2.2235259000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 423163, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 132, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14587029, "tokens_cache_write": 337248, "tokens_input": 872, "tokens_output": 17891, "tool_calls": 102, "tool_calls_by_class": { "cc_builtin": 95, "vendor_mcp": 7 }, "tool_result_tokens": 59949, "tool_time_ms": 3346, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9034762999999995, "cost_usd": 2.9034762999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 432676, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 107, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10945979, "tokens_cache_write": 838267, "tokens_input": 736, "tokens_output": 18651, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 47, "harness_builtin": 1, "vendor_mcp": 29 }, "tool_result_tokens": 187614, "tool_time_ms": 5211, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3721297, "cost_usd": 2.3721297, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 369875, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 112, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/MultiTurnExam/memos/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11924487, "tokens_cache_write": 548089, "tokens_input": 758, "tokens_output": 16549, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 76, "vendor_mcp": 6 }, "tool_result_tokens": 62545, "tool_time_ms": 2888, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4078468, "cost_usd": 2.4078468, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 482289, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 12, "family": "cc_competitor", "llm_calls": 155, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 8, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6659668, "tokens_cache_write": 816391, "tokens_input": 957, "tokens_output": 26019, "tool_calls": 133, "tool_calls_by_class": { "cc_builtin": 74, "harness_builtin": 1, "subagent": 1, "vendor_mcp": 57 }, "tool_result_tokens": 398528, "tool_time_ms": 7364, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5775896, "cost_usd": 2.5775896, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 535248, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "cc_competitor", "llm_calls": 157, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7666136, "tokens_cache_write": 836466, "tokens_input": 939, "tokens_output": 27421, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 77, "vendor_mcp": 60 }, "tool_result_tokens": 414079, "tool_time_ms": 12376, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3949449, "cost_usd": 2.3949449, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 459965, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 141, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7235649, "tokens_cache_write": 777784, "tokens_input": 852, "tokens_output": 22992, "tool_calls": 121, "tool_calls_by_class": { "cc_builtin": 56, "harness_builtin": 2, "vendor_mcp": 63 }, "tool_result_tokens": 409658, "tool_time_ms": 7844, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2047465, "cost_usd": 2.2047465, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 466452, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 138, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6693245, "tokens_cache_write": 710412, "tokens_input": 838, "tokens_output": 22752, "tool_calls": 118, "tool_calls_by_class": { "cc_builtin": 64, "vendor_mcp": 54 }, "tool_result_tokens": 363053, "tool_time_ms": 7521, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0873286, "cost_usd": 2.0873286, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 438264, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 122, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5573826, "tokens_cache_write": 708912, "tokens_input": 747, "tokens_output": 22275, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 48, "vendor_mcp": 55 }, "tool_result_tokens": 366871, "tool_time_ms": 7116, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1588817, "cost_usd": 2.1588817, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 530451, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 10, "family": "cc_competitor", "llm_calls": 151, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7203317, "tokens_cache_write": 655115, "tokens_input": 920, "tokens_output": 25480, "tool_calls": 131, "tool_calls_by_class": { "cc_builtin": 79, "harness_builtin": 1, "vendor_mcp": 51 }, "tool_result_tokens": 360067, "tool_time_ms": 8268, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.167772, "cost_usd": 2.167772, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 427828, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 10, "family": "cc_competitor", "llm_calls": 120, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5403140, "tokens_cache_write": 754973, "tokens_input": 742, "tokens_output": 23354, "tool_calls": 100, "tool_calls_by_class": { "cc_builtin": 43, "vendor_mcp": 57 }, "tool_result_tokens": 375599, "tool_time_ms": 7504, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2197125, "cost_usd": 2.2197125, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 465678, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 10, "family": "cc_competitor", "llm_calls": 130, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5941705, "tokens_cache_write": 752820, "tokens_input": 797, "tokens_output": 23821, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 53, "vendor_mcp": 57 }, "tool_result_tokens": 377594, "tool_time_ms": 7656, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4196507, "cost_usd": 2.4196507, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 549221, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 161, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8000317, "tokens_cache_write": 741366, "tokens_input": 977, "tokens_output": 27182, "tool_calls": 141, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 53 }, "tool_result_tokens": 366416, "tool_time_ms": 7694, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0391475, "cost_usd": 2.0391475, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 405182, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 11, "family": "cc_competitor", "llm_calls": 114, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 9, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5219195, "tokens_cache_write": 708383, "tokens_input": 707, "tokens_output": 19951, "tool_calls": 94, "tool_calls_by_class": { "cc_builtin": 41, "vendor_mcp": 53 }, "tool_result_tokens": 346821, "tool_time_ms": 7375, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1286206, "cost_usd": 2.1286206, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 383657, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 112, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4331416, "tokens_cache_write": 797243, "tokens_input": 733, "tokens_output": 20052, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 21, "vendor_mcp": 61 }, "tool_result_tokens": 388945, "tool_time_ms": 7376, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1910782, "cost_usd": 2.1910782, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 452257, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 132, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5276867, "tokens_cache_write": 766173, "tokens_input": 845, "tokens_output": 24573, "tool_calls": 102, "tool_calls_by_class": { "cc_builtin": 45, "vendor_mcp": 57 }, "tool_result_tokens": 384016, "tool_time_ms": 7357, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1137761, "cost_usd": 2.1137761, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 405544, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 114, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4246061, "tokens_cache_write": 791929, "tokens_input": 747, "tokens_output": 20913, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 23, "vendor_mcp": 61 }, "tool_result_tokens": 379377, "tool_time_ms": 7360, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0722482, "cost_usd": 2.0722482, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 403335, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 120, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4692342, "tokens_cache_write": 748101, "tokens_input": 777, "tokens_output": 21207, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 33, "vendor_mcp": 57 }, "tool_result_tokens": 372096, "tool_time_ms": 7048, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.911948, "cost_usd": 1.911948, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 380331, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 111, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4191030, "tokens_cache_write": 696789, "tokens_input": 722, "tokens_output": 19709, "tool_calls": 81, "tool_calls_by_class": { "cc_builtin": 25, "vendor_mcp": 56 }, "tool_result_tokens": 366566, "tool_time_ms": 7434, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9757476000000003, "cost_usd": 1.9757476000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 405985, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 108, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 24, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4000566, "tokens_cache_write": 735407, "tokens_input": 707, "tokens_output": 20834, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 18, "vendor_mcp": 60 }, "tool_result_tokens": 367217, "tool_time_ms": 8227, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0213752, "cost_usd": 2.0213752, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 406116, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 120, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4776762, "tokens_cache_write": 719886, "tokens_input": 767, "tokens_output": 20632, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 30, "vendor_mcp": 60 }, "tool_result_tokens": 380409, "tool_time_ms": 7140, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2315581, "cost_usd": 2.2315581, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 421359, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 120, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4728401, "tokens_cache_write": 821045, "tokens_input": 778, "tokens_output": 23170, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 28, "vendor_mcp": 62 }, "tool_result_tokens": 405040, "tool_time_ms": 7992, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0410809000000003, "cost_usd": 2.0410809000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 400969, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 115, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4364739, "tokens_cache_write": 749739, "tokens_input": 749, "tokens_output": 20876, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 25, "vendor_mcp": 60 }, "tool_result_tokens": 377195, "tool_time_ms": 8050, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9351848999999997, "cost_usd": 1.9351848999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 373333, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 111, "model": "claude-haiku-4-5", "path": "runs/codegraph/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4152519, "tokens_cache_write": 708952, "tokens_input": 719, "tokens_output": 20262, "tool_calls": 81, "tool_calls_by_class": { "cc_builtin": 22, "vendor_mcp": 59 }, "tool_result_tokens": 372723, "tool_time_ms": 8032, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.112612249999999, "cost_usd": 5.112612249999999, "denominator": 20, "duration_basis": "run", "duration_ms": 571654, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 111, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3689272, "tokens_cache_write": 283582, "tokens_input": 147, "tokens_output": 13641, "tool_calls": 223, "tool_calls_by_class": { "cc_builtin": 215, "subagent": 8 }, "tool_result_tokens": 115083, "tool_time_ms": 8379, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.620010000000004, "cost_usd": 6.620010000000004, "denominator": 20, "duration_basis": "run", "duration_ms": 443259, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 137, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4570855, "tokens_cache_write": 414013, "tokens_input": 183, "tokens_output": 22387, "tool_calls": 251, "tool_calls_by_class": { "cc_builtin": 225, "subagent": 13, "vendor_mcp": 13 }, "tool_result_tokens": 207483, "tool_time_ms": 8596, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.245480750000002, "cost_usd": 5.245480750000002, "denominator": 20, "duration_basis": "run", "duration_ms": 506241, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 110, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3976454, "tokens_cache_write": 306200, "tokens_input": 148, "tokens_output": 20720, "tool_calls": 198, "tool_calls_by_class": { "cc_builtin": 176, "subagent": 8, "vendor_mcp": 14 }, "tool_result_tokens": 156095, "tool_time_ms": 7520, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.9124372500000035, "cost_usd": 4.9124372500000035, "denominator": 20, "duration_basis": "run", "duration_ms": 592283, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 121, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4708942, "tokens_cache_write": 216505, "tokens_input": 143, "tokens_output": 27400, "tool_calls": 222, "tool_calls_by_class": { "cc_builtin": 217, "subagent": 5 }, "tool_result_tokens": 96059, "tool_time_ms": 8946, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.899549999999998, "cost_usd": 5.899549999999998, "denominator": 20, "duration_basis": "run", "duration_ms": 588738, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 116, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4364750, "tokens_cache_write": 370960, "tokens_input": 8301, "tokens_output": 28697, "tool_calls": 209, "tool_calls_by_class": { "cc_builtin": 179, "subagent": 8, "vendor_mcp": 22 }, "tool_result_tokens": 206494, "tool_time_ms": 8698, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.284678500000007, "cost_usd": 9.284678500000007, "denominator": 20, "duration_basis": "run", "duration_ms": 612267, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 151, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7987212, "tokens_cache_write": 443992, "tokens_input": 2965, "tokens_output": 38702, "tool_calls": 314, "tool_calls_by_class": { "cc_builtin": 282, "subagent": 11, "vendor_mcp": 21 }, "tool_result_tokens": 227512, "tool_time_ms": 10781, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.8511505, "cost_usd": 6.8511505, "denominator": 20, "duration_basis": "run", "duration_ms": 381493, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 160, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6703286, "tokens_cache_write": 329144, "tokens_input": 269, "tokens_output": 23904, "tool_calls": 292, "tool_calls_by_class": { "cc_builtin": 268, "subagent": 8, "vendor_mcp": 16 }, "tool_result_tokens": 187131, "tool_time_ms": 16055, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.151886249999999, "cost_usd": 5.151886249999999, "denominator": 20, "duration_basis": "run", "duration_ms": 396948, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3044135, "tokens_cache_write": 367138, "tokens_input": 136, "tokens_output": 20826, "tool_calls": 198, "tool_calls_by_class": { "cc_builtin": 173, "subagent": 10, "vendor_mcp": 15 }, "tool_result_tokens": 189642, "tool_time_ms": 6362, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.56722125, "cost_usd": 4.56722125, "denominator": 20, "duration_basis": "run", "duration_ms": 513980, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2805490, "tokens_cache_write": 290275, "tokens_input": 120, "tokens_output": 16060, "tool_calls": 209, "tool_calls_by_class": { "cc_builtin": 193, "subagent": 7, "vendor_mcp": 9 }, "tool_result_tokens": 155060, "tool_time_ms": 9057, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.763764000000002, "cost_usd": 4.763764000000002, "denominator": 20, "duration_basis": "run", "duration_ms": 542469, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3093728, "tokens_cache_write": 286786, "tokens_input": 128, "tokens_output": 16626, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 161, "subagent": 8, "vendor_mcp": 13 }, "tool_result_tokens": 138298, "tool_time_ms": 7196, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.89704625, "cost_usd": 4.89704625, "denominator": 30, "duration_basis": "run", "duration_ms": 357739, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4572670, "tokens_cache_write": 239756, "tokens_input": 92, "tokens_output": 18138, "tool_calls": 251, "tool_calls_by_class": { "cc_builtin": 247, "subagent": 3, "vendor_mcp": 1 }, "tool_result_tokens": 107444, "tool_time_ms": 5254, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.785880999999999, "cost_usd": 4.785880999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 229421, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 108, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3568837, "tokens_cache_write": 271510, "tokens_input": 134, "tokens_output": 17514, "tool_calls": 287, "tool_calls_by_class": { "cc_builtin": 272, "subagent": 11, "vendor_mcp": 4 }, "tool_result_tokens": 128782, "tool_time_ms": 5217, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.5165215, "cost_usd": 4.5165215, "denominator": 30, "duration_basis": "run", "duration_ms": 286240, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 71, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3484428, "tokens_cache_write": 273667, "tokens_input": 1541, "tokens_output": 22941, "tool_calls": 217, "tool_calls_by_class": { "cc_builtin": 212, "subagent": 4, "vendor_mcp": 1 }, "tool_result_tokens": 156937, "tool_time_ms": 3873, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.756841749999997, "cost_usd": 4.756841749999997, "denominator": 30, "duration_basis": "run", "duration_ms": 249577, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2440501, "tokens_cache_write": 364952, "tokens_input": 134, "tokens_output": 21446, "tool_calls": 195, "tool_calls_by_class": { "cc_builtin": 164, "subagent": 12, "vendor_mcp": 19 }, "tool_result_tokens": 194246, "tool_time_ms": 5426, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8980465, "cost_usd": 4.8980465, "denominator": 30, "duration_basis": "run", "duration_ms": 311159, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 100, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4484338, "tokens_cache_write": 252512, "tokens_input": 116, "tokens_output": 19280, "tool_calls": 262, "tool_calls_by_class": { "cc_builtin": 250, "subagent": 6, "vendor_mcp": 6 }, "tool_result_tokens": 131223, "tool_time_ms": 4615, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.118764999999998, "cost_usd": 4.118764999999998, "denominator": 30, "duration_basis": "run", "duration_ms": 279166, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2410605, "tokens_cache_write": 284325, "tokens_input": 1666, "tokens_output": 19167, "tool_calls": 228, "tool_calls_by_class": { "cc_builtin": 207, "subagent": 12, "vendor_mcp": 9 }, "tool_result_tokens": 134657, "tool_time_ms": 5255, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5639857500000005, "cost_usd": 3.5639857500000005, "denominator": 30, "duration_basis": "run", "duration_ms": 280576, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 59, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1779839, "tokens_cache_write": 265921, "tokens_input": 93, "tokens_output": 19104, "tool_calls": 161, "tool_calls_by_class": { "cc_builtin": 148, "subagent": 8, "vendor_mcp": 5 }, "tool_result_tokens": 118450, "tool_time_ms": 3637, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7852749999999986, "cost_usd": 3.7852749999999986, "denominator": 30, "duration_basis": "run", "duration_ms": 335366, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 58, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2382875, "tokens_cache_write": 256799, "tokens_input": 86, "tokens_output": 14309, "tool_calls": 177, "tool_calls_by_class": { "cc_builtin": 163, "subagent": 6, "vendor_mcp": 8 }, "tool_result_tokens": 136163, "tool_time_ms": 4709, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8443015, "cost_usd": 2.8443015, "denominator": 30, "duration_basis": "run", "duration_ms": 248193, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 59, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1818773, "tokens_cache_write": 193110, "tokens_input": 91, "tokens_output": 10450, "tool_calls": 138, "tool_calls_by_class": { "cc_builtin": 122, "subagent": 7, "vendor_mcp": 9 }, "tool_result_tokens": 92122, "tool_time_ms": 4078, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7990919999999986, "cost_usd": 3.7990919999999986, "denominator": 30, "duration_basis": "run", "duration_ms": 383012, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 70, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2335324, "tokens_cache_write": 268037, "tokens_input": 102, "tokens_output": 16331, "tool_calls": 164, "tool_calls_by_class": { "cc_builtin": 146, "subagent": 8, "vendor_mcp": 10 }, "tool_result_tokens": 129206, "tool_time_ms": 4251, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.286026000000002, "cost_usd": 7.286026000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 790175, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 85, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8919381, "tokens_cache_write": 211364, "tokens_input": 127, "tokens_output": 19202, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 68, "vendor_mcp": 14 }, "tool_result_tokens": 103824, "tool_time_ms": 19139, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.319784499999998, "cost_usd": 5.319784499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 779219, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 105, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8036349, "tokens_cache_write": 85996, "tokens_input": 145, "tokens_output": 17637, "tool_calls": 93, "tool_calls_by_class": { "cc_builtin": 91, "vendor_mcp": 2 }, "tool_result_tokens": 33511, "tool_time_ms": 6719, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.737322999999999, "cost_usd": 4.737322999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 653375, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7056086, "tokens_cache_write": 82249, "tokens_input": 133, "tokens_output": 15445, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 81, "vendor_mcp": 2 }, "tool_result_tokens": 32235, "tool_time_ms": 5216, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8114585000000005, "cost_usd": 5.8114585000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 611242, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8455847, "tokens_cache_write": 116200, "tokens_input": 132, "tokens_output": 16835, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 80, "vendor_mcp": 6 }, "tool_result_tokens": 58388, "tool_time_ms": 6081, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.985579499999999, "cost_usd": 4.985579499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 558961, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7089149, "tokens_cache_write": 105710, "tokens_input": 121, "tokens_output": 15332, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 67, "vendor_mcp": 4 }, "tool_result_tokens": 51533, "tool_time_ms": 7869, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.450551, "cost_usd": 6.450551, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 755194, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8365920, "tokens_cache_write": 155939, "tokens_input": 121, "tokens_output": 19799, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 63, "vendor_mcp": 11 }, "tool_result_tokens": 83796, "tool_time_ms": 10036, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.995182, "cost_usd": 5.995182, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 634732, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 83, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8589294, "tokens_cache_write": 130846, "tokens_input": 125, "tokens_output": 15658, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 65, "vendor_mcp": 8 }, "tool_result_tokens": 71274, "tool_time_ms": 6503, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.050910500000001, "cost_usd": 5.050910500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 524690, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 74, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7245301, "tokens_cache_write": 112178, "tokens_input": 116, "tokens_output": 12236, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 47, "vendor_mcp": 9 }, "tool_result_tokens": 64153, "tool_time_ms": 7227, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.417743499999999, "cost_usd": 5.417743499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 650980, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7743757, "tokens_cache_write": 109465, "tokens_input": 128, "tokens_output": 18023, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 73, "vendor_mcp": 4 }, "tool_result_tokens": 51604, "tool_time_ms": 16534, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.242826, "cost_usd": 8.242826, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 808554, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 119, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12643752, "tokens_cache_write": 137388, "tokens_input": 159, "tokens_output": 21851, "tool_calls": 108, "tool_calls_by_class": { "cc_builtin": 102, "vendor_mcp": 6 }, "tool_result_tokens": 69786, "tool_time_ms": 17459, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2976285000000005, "cost_usd": 3.2976285000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 441139, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4477157, "tokens_cache_write": 81737, "tokens_input": 141, "tokens_output": 9639, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 10289, "tool_time_ms": 2344, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3645365, "cost_usd": 3.3645365, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 401012, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5078263, "tokens_cache_write": 58205, "tokens_input": 146, "tokens_output": 9705, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 11391, "tool_time_ms": 2194, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.764377, "cost_usd": 3.764377, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 571334, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 96, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5742974, "tokens_cache_write": 61871, "tokens_input": 156, "tokens_output": 10936, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 14135, "tool_time_ms": 2543, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.62942, "cost_usd": 2.62942, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 421827, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 72, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3806340, "tokens_cache_write": 53109, "tokens_input": 132, "tokens_output": 7780, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 48 }, "tool_result_tokens": 9925, "tool_time_ms": 2128, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7399719999999994, "cost_usd": 2.7399719999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 390528, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 75, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4024564, "tokens_cache_write": 53339, "tokens_input": 135, "tokens_output": 7745, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 9858, "tool_time_ms": 1969, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6354875, "cost_usd": 2.6354875, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 355492, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 71, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3816025, "tokens_cache_write": 53567, "tokens_input": 131, "tokens_output": 7646, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 10152, "tool_time_ms": 2043, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.85563, "cost_usd": 2.85563, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 417995, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 78, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4198690, "tokens_cache_write": 52922, "tokens_input": 138, "tokens_output": 9055, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 8493, "tool_time_ms": 2017, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0316294999999998, "cost_usd": 3.0316294999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 427119, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 85, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4651719, "tokens_cache_write": 48127, "tokens_input": 145, "tokens_output": 8951, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 8160, "tool_time_ms": 2347, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3846535, "cost_usd": 2.3846535, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 329046, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 66, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3460907, "tokens_cache_write": 45967, "tokens_input": 126, "tokens_output": 7756, "tool_calls": 44, "tool_calls_by_class": { "cc_builtin": 44 }, "tool_result_tokens": 7385, "tool_time_ms": 2114, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1937990000000003, "cost_usd": 3.1937990000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 442631, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4798078, "tokens_cache_write": 56438, "tokens_input": 146, "tokens_output": 9186, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 11317, "tool_time_ms": 2196, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.649190000000002, "cost_usd": 5.649190000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 778770, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3760820, "tokens_cache_write": 313906, "tokens_input": 139, "tokens_output": 25161, "tool_calls": 120, "tool_calls_by_class": { "cc_builtin": 101, "vendor_mcp": 19 }, "tool_result_tokens": 144575, "tool_time_ms": 6457, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.134931999999999, "cost_usd": 6.134931999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 726989, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3926844, "tokens_cache_write": 359315, "tokens_input": 142, "tokens_output": 23106, "tool_calls": 119, "tool_calls_by_class": { "cc_builtin": 96, "vendor_mcp": 23 }, "tool_result_tokens": 165728, "tool_time_ms": 6611, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2180005000000005, "cost_usd": 5.2180005000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 687769, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3392271, "tokens_cache_write": 301998, "tokens_input": 127, "tokens_output": 20050, "tool_calls": 93, "tool_calls_by_class": { "cc_builtin": 69, "vendor_mcp": 24 }, "tool_result_tokens": 164909, "tool_time_ms": 6293, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.615843000000002, "cost_usd": 6.615843000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1027664, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 137, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5675666, "tokens_cache_write": 301479, "tokens_input": 264, "tokens_output": 30476, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 127, "vendor_mcp": 21 }, "tool_result_tokens": 155135, "tool_time_ms": 8817, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.4887025000000005, "cost_usd": 6.4887025000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 857560, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 114, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4727865, "tokens_cache_write": 342685, "tokens_input": 154, "tokens_output": 27886, "tool_calls": 136, "tool_calls_by_class": { "cc_builtin": 111, "vendor_mcp": 25 }, "tool_result_tokens": 188981, "tool_time_ms": 6335, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.025289999999999, "cost_usd": 6.025289999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 875579, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4153890, "tokens_cache_write": 331569, "tokens_input": 231, "tokens_output": 25260, "tool_calls": 127, "tool_calls_by_class": { "cc_builtin": 100, "vendor_mcp": 27 }, "tool_result_tokens": 183361, "tool_time_ms": 6722, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.50516875, "cost_usd": 6.50516875, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1008929, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 118, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4471590, "tokens_cache_write": 361965, "tokens_input": 162, "tokens_output": 22610, "tool_calls": 151, "tool_calls_by_class": { "cc_builtin": 126, "subagent": 1, "vendor_mcp": 24 }, "tool_result_tokens": 183352, "tool_time_ms": 8330, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.911307, "cost_usd": 5.911307, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 793454, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 111, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4429724, "tokens_cache_write": 308478, "tokens_input": 153, "tokens_output": 24436, "tool_calls": 127, "tool_calls_by_class": { "cc_builtin": 104, "vendor_mcp": 23 }, "tool_result_tokens": 165519, "tool_time_ms": 5772, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.712148499999999, "cost_usd": 5.712148499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 750031, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 108, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4435277, "tokens_cache_write": 293067, "tokens_input": 148, "tokens_output": 22524, "tool_calls": 126, "tool_calls_by_class": { "cc_builtin": 107, "vendor_mcp": 19 }, "tool_result_tokens": 153078, "tool_time_ms": 5013, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.756267500000001, "cost_usd": 6.756267500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1035064, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 127, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5470605, "tokens_cache_write": 321927, "tokens_input": 254, "tokens_output": 32017, "tool_calls": 146, "tool_calls_by_class": { "cc_builtin": 123, "vendor_mcp": 23 }, "tool_result_tokens": 169612, "tool_time_ms": 7325, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.187393999999999, "cost_usd": 5.187393999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 546126, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 83, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2814768, "tokens_cache_write": 341297, "tokens_input": 143, "tokens_output": 14653, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 51, "vendor_mcp": 23 }, "tool_result_tokens": 152398, "tool_time_ms": 5938, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2894, "cost_usd": 5.2894, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 566484, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3082580, "tokens_cache_write": 337932, "tokens_input": 148, "tokens_output": 14722, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 51, "vendor_mcp": 25 }, "tool_result_tokens": 169751, "tool_time_ms": 5989, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.168705, "cost_usd": 5.168705, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 584743, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3260930, "tokens_cache_write": 314525, "tokens_input": 153, "tokens_output": 15689, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 62, "vendor_mcp": 22 }, "tool_result_tokens": 146744, "tool_time_ms": 6350, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.048942500000001, "cost_usd": 5.048942500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 532669, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2944075, "tokens_cache_write": 321621, "tokens_input": 144, "tokens_output": 14399, "tool_calls": 75, "tool_calls_by_class": { "cc_builtin": 52, "vendor_mcp": 23 }, "tool_result_tokens": 156823, "tool_time_ms": 5518, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.514245000000002, "cost_usd": 4.514245000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 548002, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2748110, "tokens_cache_write": 278606, "tokens_input": 141, "tokens_output": 14137, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 18 }, "tool_result_tokens": 121937, "tool_time_ms": 5146, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.1113504999999995, "cost_usd": 5.1113504999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 596172, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3004201, "tokens_cache_write": 323039, "tokens_input": 147, "tokens_output": 15125, "tool_calls": 81, "tool_calls_by_class": { "cc_builtin": 57, "vendor_mcp": 24 }, "tool_result_tokens": 158027, "tool_time_ms": 6413, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.141259499999999, "cost_usd": 5.141259499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 582208, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 85, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2968359, "tokens_cache_write": 328083, "tokens_input": 145, "tokens_output": 15021, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 53, "vendor_mcp": 24 }, "tool_result_tokens": 160864, "tool_time_ms": 6844, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.9222435, "cost_usd": 4.9222435, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 551083, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2957637, "tokens_cache_write": 309252, "tokens_input": 146, "tokens_output": 14007, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 51, "vendor_mcp": 22 }, "tool_result_tokens": 147664, "tool_time_ms": 6592, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.256348, "cost_usd": 5.256348, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 648073, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 91, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3172096, "tokens_cache_write": 328542, "tokens_input": 151, "tokens_output": 15365, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 59, "vendor_mcp": 24 }, "tool_result_tokens": 160484, "tool_time_ms": 10265, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.091195499999999, "cost_usd": 5.091195499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 558775, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-4-6", "path": "runs/codegraph/claude-opus-4-6/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2810461, "tokens_cache_write": 333011, "tokens_input": 141, "tokens_output": 14206, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 42, "vendor_mcp": 25 }, "tool_result_tokens": 165577, "tool_time_ms": 11131, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.785746999999998, "cost_usd": 5.785746999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 581386, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 72, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6870794, "tokens_cache_write": 148269, "tokens_input": 137, "tokens_output": 34679, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 51, "vendor_mcp": 3 }, "tool_result_tokens": 36184, "tool_time_ms": 4019, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.336210499999999, "cost_usd": 6.336210499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 531016, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 55, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7571741, "tokens_cache_write": 173878, "tokens_input": 107, "tokens_output": 32441, "tool_calls": 35, "tool_calls_by_class": { "cc_builtin": 26, "vendor_mcp": 9 }, "tool_result_tokens": 68144, "tool_time_ms": 4474, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.780407000000001, "cost_usd": 7.780407000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 631232, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 74, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10150074, "tokens_cache_write": 176582, "tokens_input": 145, "tokens_output": 37553, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 45, "vendor_mcp": 9 }, "tool_result_tokens": 67852, "tool_time_ms": 6558, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.530139499999998, "cost_usd": 7.530139499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 591438, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 69, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9664699, "tokens_cache_write": 183086, "tokens_input": 131, "tokens_output": 34651, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 39, "vendor_mcp": 10 }, "tool_result_tokens": 72742, "tool_time_ms": 5129, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.875195499999999, "cost_usd": 7.875195499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 672372, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 68, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9814821, "tokens_cache_write": 193503, "tokens_input": 131, "tokens_output": 41284, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 36, "vendor_mcp": 12 }, "tool_result_tokens": 75793, "tool_time_ms": 11158, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.768764, "cost_usd": 6.768764, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 615259, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 71, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8786038, "tokens_cache_write": 160094, "tokens_input": 136, "tokens_output": 30965, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 45, "vendor_mcp": 6 }, "tool_result_tokens": 60066, "tool_time_ms": 6052, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.776688, "cost_usd": 5.776688, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 472080, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 62, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7380976, "tokens_cache_write": 143590, "tokens_input": 120, "tokens_output": 25988, "tool_calls": 42, "tool_calls_by_class": { "cc_builtin": 36, "vendor_mcp": 6 }, "tool_result_tokens": 55059, "tool_time_ms": 4125, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.033252499999998, "cost_usd": 8.033252499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 731616, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 68, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9943615, "tokens_cache_write": 196044, "tokens_input": 131, "tokens_output": 44014, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 38, "vendor_mcp": 10 }, "tool_result_tokens": 74609, "tool_time_ms": 11879, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.106570999999999, "cost_usd": 5.106570999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 494823, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 63, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6323852, "tokens_cache_write": 121703, "tokens_input": 123, "tokens_output": 29080, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 43, "vendor_mcp": 4 }, "tool_result_tokens": 39979, "tool_time_ms": 3760, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.6476645, "cost_usd": 6.6476645, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 563233, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8839679, "tokens_cache_write": 147737, "tokens_input": 151, "tokens_output": 29988, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 61, "vendor_mcp": 4 }, "tool_result_tokens": 54345, "tool_time_ms": 7486, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9178455, "cost_usd": 3.9178455, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 354757, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4996841, "tokens_cache_write": 100995, "tokens_input": 155, "tokens_output": 16348, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 11834, "tool_time_ms": 2088, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.052692, "cost_usd": 4.052692, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 386104, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5599634, "tokens_cache_write": 82615, "tokens_input": 155, "tokens_output": 17038, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 49, "vendor_mcp": 1 }, "tool_result_tokens": 15431, "tool_time_ms": 2182, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6738835000000005, "cost_usd": 3.6738835000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 381055, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4971517, "tokens_cache_write": 73318, "tokens_input": 154, "tokens_output": 18167, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 9185, "tool_time_ms": 2258, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.476232499999999, "cost_usd": 3.476232499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 450146, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 77, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4716775, "tokens_cache_write": 71612, "tokens_input": 150, "tokens_output": 16039, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 47 }, "tool_result_tokens": 9889, "tool_time_ms": 2212, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4028519999999993, "cost_usd": 3.4028519999999993, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 332890, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 75, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4599754, "tokens_cache_write": 71424, "tokens_input": 147, "tokens_output": 15520, "tool_calls": 46, "tool_calls_by_class": { "cc_builtin": 46 }, "tool_result_tokens": 9777, "tool_time_ms": 2027, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1240835, "cost_usd": 3.1240835, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 306049, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 71, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4209357, "tokens_cache_write": 66200, "tokens_input": 136, "tokens_output": 14269, "tool_calls": 41, "tool_calls_by_class": { "cc_builtin": 41 }, "tool_result_tokens": 7813, "tool_time_ms": 2092, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.014057500000001, "cost_usd": 4.014057500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 401107, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5435535, "tokens_cache_write": 80565, "tokens_input": 158, "tokens_output": 19594, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 12457, "tool_time_ms": 2329, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8087934999999997, "cost_usd": 3.8087934999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 349021, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5301797, "tokens_cache_write": 74340, "tokens_input": 164, "tokens_output": 16547, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 10932, "tool_time_ms": 2277, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1984785, "cost_usd": 4.1984785, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 348635, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5877587, "tokens_cache_write": 85626, "tokens_input": 2000, "tokens_output": 15737, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 50, "vendor_mcp": 1 }, "tool_result_tokens": 18212, "tool_time_ms": 2382, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.2539495, "cost_usd": 4.2539495, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 358294, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5997369, "tokens_cache_write": 84170, "tokens_input": 158, "tokens_output": 16511, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 51, "vendor_mcp": 1 }, "tool_result_tokens": 18006, "tool_time_ms": 2021, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.6456474999999999, "cost_usd": 0.6456474999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 59262, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 118885, "tokens_cache_write": 54623, "tokens_input": 10, "tokens_output": 1597, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 4, "vendor_mcp": 1 }, "tool_result_tokens": 8823, "tool_time_ms": 15780, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.412696, "cost_usd": 0.412696, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 57399, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 4, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 115242, "tokens_cache_write": 32126, "tokens_input": 8, "tokens_output": 1351, "tool_calls": 2, "tool_calls_by_class": { "vendor_mcp": 2 }, "tool_result_tokens": 12912, "tool_time_ms": 5293, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.37257, "cost_usd": 0.37257, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 60491, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 4, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 110520, "tokens_cache_write": 27337, "tokens_input": 8, "tokens_output": 1756, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 2, "vendor_mcp": 1 }, "tool_result_tokens": 7958, "tool_time_ms": 20759, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.43422649999999996, "cost_usd": 0.43422649999999996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 50918, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 179463, "tokens_cache_write": 29751, "tokens_input": 12, "tokens_output": 1877, "tool_calls": 6, "tool_calls_by_class": { "cc_builtin": 5, "vendor_mcp": 1 }, "tool_result_tokens": 8776, "tool_time_ms": 7732, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.5129665, "cost_usd": 0.5129665, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 41311, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 156413, "tokens_cache_write": 39751, "tokens_input": 10, "tokens_output": 1488, "tool_calls": 3, "tool_calls_by_class": { "vendor_mcp": 3 }, "tool_result_tokens": 17455, "tool_time_ms": 3666, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.429129, "cost_usd": 0.429129, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 35262, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 4, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 115238, "tokens_cache_write": 32049, "tokens_input": 1749, "tokens_output": 1691, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 1, "vendor_mcp": 2 }, "tool_result_tokens": 13664, "tool_time_ms": 4059, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.455763, "cost_usd": 0.455763, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 41125, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 202856, "tokens_cache_write": 31645, "tokens_input": 12, "tokens_output": 1513, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 2, "vendor_mcp": 2 }, "tool_result_tokens": 13438, "tool_time_ms": 3730, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.41338300000000006, "cost_usd": 0.41338300000000006, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 34541, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 4, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 115236, "tokens_cache_write": 32280, "tokens_input": 8, "tokens_output": 1317, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 1, "vendor_mcp": 2 }, "tool_result_tokens": 12925, "tool_time_ms": 2560, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.493528, "cost_usd": 0.493528, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 41353, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 158136, "tokens_cache_write": 37956, "tokens_input": 10, "tokens_output": 1394, "tool_calls": 3, "tool_calls_by_class": { "vendor_mcp": 3 }, "tool_result_tokens": 17597, "tool_time_ms": 4946, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.3862145, "cost_usd": 0.3862145, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 34453, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 4, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 116989, "tokens_cache_write": 29858, "tokens_input": 8, "tokens_output": 1164, "tool_calls": 2, "tool_calls_by_class": { "vendor_mcp": 2 }, "tool_result_tokens": 12600, "tool_time_ms": 1943, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.201972500000001, "cost_usd": 7.201972500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 654173, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|11", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3405015, "tokens_cache_write": 452335, "tokens_input": 183, "tokens_output": 39008, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 58, "vendor_mcp": 28 }, "tool_result_tokens": 187289, "tool_time_ms": 8205, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.1508855, "cost_usd": 6.1508855, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 670109, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|12", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2998701, "tokens_cache_write": 359566, "tokens_input": 170, "tokens_output": 42201, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 62, "vendor_mcp": 20 }, "tool_result_tokens": 145104, "tool_time_ms": 7160, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.167706999999999, "cost_usd": 6.167706999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 637857, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|13", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2955084, "tokens_cache_write": 372908, "tokens_input": 167, "tokens_output": 38410, "tool_calls": 81, "tool_calls_by_class": { "cc_builtin": 59, "vendor_mcp": 22 }, "tool_result_tokens": 156705, "tool_time_ms": 6740, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.0638995000000016, "cost_usd": 7.0638995000000016, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 761135, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|14", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3267169, "tokens_cache_write": 433762, "tokens_input": 174, "tokens_output": 43673, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 57, "vendor_mcp": 28 }, "tool_result_tokens": 188095, "tool_time_ms": 10412, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.528690499999999, "cost_usd": 6.528690499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 706683, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 96, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|15", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3434141, "tokens_cache_write": 373953, "tokens_input": 188, "tokens_output": 42846, "tool_calls": 94, "tool_calls_by_class": { "cc_builtin": 74, "vendor_mcp": 20 }, "tool_result_tokens": 153808, "tool_time_ms": 8238, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.297314500000001, "cost_usd": 6.297314500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 702442, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep16", "query_calls": 0, "rep": 16, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|16", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3005049, "tokens_cache_write": 376550, "tokens_input": 173, "tokens_output": 41137, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 63, "vendor_mcp": 22 }, "tool_result_tokens": 156186, "tool_time_ms": 9147, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.155304999999999, "cost_usd": 6.155304999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 621747, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep17", "query_calls": 0, "rep": 17, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|17", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2971930, "tokens_cache_write": 372661, "tokens_input": 166, "tokens_output": 37676, "tool_calls": 80, "tool_calls_by_class": { "cc_builtin": 59, "vendor_mcp": 21 }, "tool_result_tokens": 155380, "tool_time_ms": 8081, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.242885, "cost_usd": 6.242885, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 644460, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep18", "query_calls": 0, "rep": 18, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|18", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2880350, "tokens_cache_write": 380395, "tokens_input": 162, "tokens_output": 39918, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 58, "vendor_mcp": 24 }, "tool_result_tokens": 160004, "tool_time_ms": 9372, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2775175, "cost_usd": 6.2775175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 677061, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep19", "query_calls": 0, "rep": 19, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|19", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3159765, "tokens_cache_write": 368373, "tokens_input": 211, "tokens_output": 40514, "tool_calls": 87, "tool_calls_by_class": { "cc_builtin": 68, "vendor_mcp": 19 }, "tool_result_tokens": 147370, "tool_time_ms": 11427, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.681905500000001, "cost_usd": 6.681905500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 691486, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos-hard/rep20", "query_calls": 0, "rep": 20, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|20", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3217691, "tokens_cache_write": 399502, "tokens_input": 8138, "tokens_output": 41494, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 67, "vendor_mcp": 23 }, "tool_result_tokens": 174498, "tool_time_ms": 14519, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.692812, "cost_usd": 0.692812, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 17908, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 4, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 63064, "tokens_cache_write": 64734, "tokens_input": 8, "tokens_output": 556, "tool_calls": 2, "tool_calls_by_class": { "cc_builtin": 2 }, "tool_result_tokens": 693, "tool_time_ms": 247, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.46342550000000005, "cost_usd": 0.46342550000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 19921, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 4, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 88301, "tokens_cache_write": 40156, "tokens_input": 8, "tokens_output": 707, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 947, "tool_time_ms": 277, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.48877299999999996, "cost_usd": 0.48877299999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 20630, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 152356, "tokens_cache_write": 39346, "tokens_input": 12, "tokens_output": 763, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 544, "tool_time_ms": 299, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.227185, "cost_usd": 0.227185, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 18074, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 145730, "tokens_cache_write": 13842, "tokens_input": 10, "tokens_output": 634, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 448, "tool_time_ms": 314, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.23798600000000003, "cost_usd": 0.23798600000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 23877, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 145412, "tokens_cache_write": 14468, "tokens_input": 10, "tokens_output": 822, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 668, "tool_time_ms": 451, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.215798, "cost_usd": 0.215798, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 18207, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 147456, "tokens_cache_write": 12587, "tokens_input": 10, "tokens_output": 646, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 711, "tool_time_ms": 310, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2559045, "cost_usd": 0.2559045, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 30145, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 7, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 211399, "tokens_cache_write": 12861, "tokens_input": 14, "tokens_output": 861, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 692, "tool_time_ms": 472, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.31464749999999997, "cost_usd": 0.31464749999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 22987, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 155485, "tokens_cache_write": 21631, "tokens_input": 494, "tokens_output": 725, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 3, "vendor_mcp": 1 }, "tool_result_tokens": 5884, "tool_time_ms": 2455, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.21891899999999997, "cost_usd": 0.21891899999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 26059, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 150478, "tokens_cache_write": 12148, "tokens_input": 10, "tokens_output": 886, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 1246, "tool_time_ms": 1117, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.21655200000000002, "cost_usd": 0.21655200000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 17951, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/codegraph/claude-opus-4-8/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 147004, "tokens_cache_write": 12665, "tokens_input": 10, "tokens_output": 654, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 724, "tool_time_ms": 673, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.858005499999999, "cost_usd": 2.858005499999999, "denominator": 20, "duration_basis": "run", "duration_ms": 337695, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2562461, "tokens_cache_write": 103188, "tokens_input": 64, "tokens_output": 21783, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 45, "vendor_mcp": 2 }, "tool_result_tokens": 30269, "tool_time_ms": 15242, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0906689999999992, "cost_usd": 3.0906689999999992, "denominator": 20, "duration_basis": "run", "duration_ms": 493824, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 32, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3168688, "tokens_cache_write": 99704, "tokens_input": 1662, "tokens_output": 20039, "tool_calls": 40, "tool_calls_by_class": { "cc_builtin": 36, "vendor_mcp": 4 }, "tool_result_tokens": 45589, "tool_time_ms": 192649, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7253480000000003, "cost_usd": 2.7253480000000003, "denominator": 20, "duration_basis": "run", "duration_ms": 323832, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 31, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2566776, "tokens_cache_write": 89407, "tokens_input": 58, "tokens_output": 21904, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 45, "vendor_mcp": 2 }, "tool_result_tokens": 37183, "tool_time_ms": 3149, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5082465, "cost_usd": 3.5082465, "denominator": 20, "duration_basis": "run", "duration_ms": 317331, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 37, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3896313, "tokens_cache_write": 108607, "tokens_input": 69, "tokens_output": 18947, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 46, "vendor_mcp": 5 }, "tool_result_tokens": 48889, "tool_time_ms": 6531, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5079214999999997, "cost_usd": 2.5079214999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 303527, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 30, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2357553, "tokens_cache_write": 79534, "tokens_input": 56, "tokens_output": 21341, "tool_calls": 43, "tool_calls_by_class": { "cc_builtin": 41, "vendor_mcp": 2 }, "tool_result_tokens": 32025, "tool_time_ms": 2333, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.987354, "cost_usd": 2.987354, "denominator": 20, "duration_basis": "run", "duration_ms": 317084, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 28, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2561228, "tokens_cache_write": 116936, "tokens_input": 51, "tokens_output": 21485, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 43, "vendor_mcp": 5 }, "tool_result_tokens": 54115, "tool_time_ms": 9485, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0761549999999995, "cost_usd": 3.0761549999999995, "denominator": 20, "duration_basis": "run", "duration_ms": 366306, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 38, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3280230, "tokens_cache_write": 89951, "tokens_input": 71, "tokens_output": 21447, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 48, "vendor_mcp": 2 }, "tool_result_tokens": 37942, "tool_time_ms": 39452, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7938955000000005, "cost_usd": 2.7938955000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 298046, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 32, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2790391, "tokens_cache_write": 90880, "tokens_input": 60, "tokens_output": 19584, "tool_calls": 46, "tool_calls_by_class": { "cc_builtin": 44, "vendor_mcp": 2 }, "tool_result_tokens": 40334, "tool_time_ms": 10930, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 1.8824994999999998, "cost_usd": 1.8824994999999998, "denominator": 20, "duration_basis": "run", "duration_ms": 259233, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 20, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1482919, "tokens_cache_write": 69093, "tokens_input": 37, "tokens_output": 17997, "tool_calls": 30, "tool_calls_by_class": { "cc_builtin": 28, "vendor_mcp": 2 }, "tool_result_tokens": 29291, "tool_time_ms": 7547, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4925314999999992, "cost_usd": 2.4925314999999992, "denominator": 20, "duration_basis": "run", "duration_ms": 311605, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 36, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2638603, "tokens_cache_write": 68439, "tokens_input": 68, "tokens_output": 19540, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 44, "vendor_mcp": 1 }, "tool_result_tokens": 27411, "tool_time_ms": 3463, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2196030000000007, "cost_usd": 2.2196030000000007, "denominator": 30, "duration_basis": "run", "duration_ms": 301383, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 33, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2170076, "tokens_cache_write": 70452, "tokens_input": 334, "tokens_output": 17135, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 26569, "tool_time_ms": 25495, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6109304999999998, "cost_usd": 1.6109304999999998, "denominator": 30, "duration_basis": "run", "duration_ms": 214160, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 13, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 774341, "tokens_cache_write": 88304, "tokens_input": 24, "tokens_output": 13624, "tool_calls": 23, "tool_calls_by_class": { "cc_builtin": 22, "vendor_mcp": 1 }, "tool_result_tokens": 26953, "tool_time_ms": 16777, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.8280639999999997, "cost_usd": 1.8280639999999997, "denominator": 30, "duration_basis": "run", "duration_ms": 376589, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 32, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1740684, "tokens_cache_write": 121766, "tokens_input": 2490, "tokens_output": 18850, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 1 }, "tool_result_tokens": 38147, "tool_time_ms": 40790, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9557399999999998, "cost_usd": 1.9557399999999998, "denominator": 30, "duration_basis": "run", "duration_ms": 187372, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 17, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1074620, "tokens_cache_write": 106705, "tokens_input": 31, "tokens_output": 14049, "tool_calls": 30, "tool_calls_by_class": { "cc_builtin": 29, "vendor_mcp": 1 }, "tool_result_tokens": 28949, "tool_time_ms": 1488, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.7083504999999999, "cost_usd": 1.7083504999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 197826, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 17, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1123971, "tokens_cache_write": 78041, "tokens_input": 31, "tokens_output": 14632, "tool_calls": 32, "tool_calls_by_class": { "cc_builtin": 32 }, "tool_result_tokens": 29853, "tool_time_ms": 1188, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0409905, "cost_usd": 2.0409905, "denominator": 30, "duration_basis": "run", "duration_ms": 236192, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 29, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1891761, "tokens_cache_write": 69277, "tokens_input": 53, "tokens_output": 16083, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 26049, "tool_time_ms": 1555, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.809463, "cost_usd": 1.809463, "denominator": 30, "duration_basis": "run", "duration_ms": 223861, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 26, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1533056, "tokens_cache_write": 65532, "tokens_input": 48, "tokens_output": 15495, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 23655, "tool_time_ms": 1786, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.819375, "cost_usd": 1.819375, "denominator": 30, "duration_basis": "run", "duration_ms": 220770, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 23, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1482710, "tokens_cache_write": 70516, "tokens_input": 42, "tokens_output": 14906, "tool_calls": 42, "tool_calls_by_class": { "cc_builtin": 42 }, "tool_result_tokens": 27535, "tool_time_ms": 1374, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0303565000000003, "cost_usd": 2.0303565000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 234465, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 26, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1693773, "tokens_cache_write": 77081, "tokens_input": 47, "tokens_output": 16497, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 49, "vendor_mcp": 1 }, "tool_result_tokens": 31991, "tool_time_ms": 1821, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.326968, "cost_usd": 1.326968, "denominator": 30, "duration_basis": "run", "duration_ms": 190768, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 15, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 825656, "tokens_cache_write": 56608, "tokens_input": 27, "tokens_output": 13917, "tool_calls": 38, "tool_calls_by_class": { "cc_builtin": 38 }, "tool_result_tokens": 20612, "tool_time_ms": 1332, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.388982, "cost_usd": 6.388982, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 606246, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 79, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8044744, "tokens_cache_write": 154215, "tokens_input": 152, "tokens_output": 32948, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 64, "vendor_mcp": 2 }, "tool_result_tokens": 39339, "tool_time_ms": 5544, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.805120499999999, "cost_usd": 6.805120499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 711262, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9223761, "tokens_cache_write": 132250, "tokens_input": 178, "tokens_output": 34794, "tool_calls": 81, "tool_calls_by_class": { "cc_builtin": 81 }, "tool_result_tokens": 39807, "tool_time_ms": 10071, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.739199, "cost_usd": 7.739199, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 653675, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 90, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10725338, "tokens_cache_write": 151430, "tokens_input": 171, "tokens_output": 34455, "tool_calls": 79, "tool_calls_by_class": { "cc_builtin": 76, "vendor_mcp": 3 }, "tool_result_tokens": 51650, "tool_time_ms": 4996, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.9228575, "cost_usd": 6.9228575, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 668427, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9375385, "tokens_cache_write": 134426, "tokens_input": 186, "tokens_output": 35599, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 84 }, "tool_result_tokens": 40446, "tool_time_ms": 6477, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.220198, "cost_usd": 8.220198, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 794813, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11178196, "tokens_cache_write": 157799, "tokens_input": 182, "tokens_output": 42088, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 74, "vendor_mcp": 2 }, "tool_result_tokens": 51065, "tool_time_ms": 4765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.291997, "cost_usd": 8.291997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 786887, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 115, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11839224, "tokens_cache_write": 142272, "tokens_input": 218, "tokens_output": 37943, "tool_calls": 101, "tool_calls_by_class": { "cc_builtin": 101 }, "tool_result_tokens": 43239, "tool_time_ms": 10997, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.603729000000001, "cost_usd": 8.603729000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 831812, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12364708, "tokens_cache_write": 156327, "tokens_input": 2421, "tokens_output": 33840, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 86, "vendor_mcp": 2 }, "tool_result_tokens": 56374, "tool_time_ms": 145376, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.68056, "cost_usd": 6.68056, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 629506, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8922150, "tokens_cache_write": 138167, "tokens_input": 158, "tokens_output": 33481, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 67, "vendor_mcp": 2 }, "tool_result_tokens": 44647, "tool_time_ms": 5532, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.943114500000001, "cost_usd": 7.943114500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 729823, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 100, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11132779, "tokens_cache_write": 144936, "tokens_input": 193, "tokens_output": 37056, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 87, "vendor_mcp": 1 }, "tool_result_tokens": 47127, "tool_time_ms": 6942, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.090168, "cost_usd": 11.090168, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 871240, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 116, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16126996, "tokens_cache_write": 190028, "tokens_input": 258, "tokens_output": 45004, "tool_calls": 97, "tool_calls_by_class": { "cc_builtin": 93, "vendor_mcp": 4 }, "tool_result_tokens": 68160, "tool_time_ms": 9888, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7414245, "cost_usd": 4.7414245, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 439512, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6054619, "tokens_cache_write": 119033, "tokens_input": 167, "tokens_output": 20918, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 19381, "tool_time_ms": 2464, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.5694885, "cost_usd": 4.5694885, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 447239, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6201397, "tokens_cache_write": 97382, "tokens_input": 164, "tokens_output": 19766, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 21768, "tool_time_ms": 2996, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.9526475, "cost_usd": 4.9526475, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 457312, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6838905, "tokens_cache_write": 99937, "tokens_input": 180, "tokens_output": 21317, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 63 }, "tool_result_tokens": 22439, "tool_time_ms": 2892, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.1575495, "cost_usd": 5.1575495, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 478740, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7008949, "tokens_cache_write": 107061, "tokens_input": 178, "tokens_output": 23263, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 26118, "tool_time_ms": 2682, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.072552, "cost_usd": 6.072552, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 582908, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8401694, "tokens_cache_write": 115456, "tokens_input": 194, "tokens_output": 28647, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 27348, "tool_time_ms": 3082, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.291229499999999, "cost_usd": 4.291229499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 416367, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5692239, "tokens_cache_write": 93012, "tokens_input": 158, "tokens_output": 20568, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 19313, "tool_time_ms": 2494, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.155819, "cost_usd": 5.155819, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 432874, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6958768, "tokens_cache_write": 113568, "tokens_input": 171, "tokens_output": 21596, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 29758, "tool_time_ms": 5099, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.764645, "cost_usd": 4.764645, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 425826, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6564740, "tokens_cache_write": 95098, "tokens_input": 184, "tokens_output": 21215, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 63 }, "tool_result_tokens": 21843, "tool_time_ms": 2840, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.373012000000001, "cost_usd": 5.373012000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 463223, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7285574, "tokens_cache_write": 110596, "tokens_input": 178, "tokens_output": 24935, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 28541, "tool_time_ms": 2742, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.283930000000001, "cost_usd": 4.283930000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 449629, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5782950, "tokens_cache_write": 90743, "tokens_input": 160, "tokens_output": 19369, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 19993, "tool_time_ms": 2609, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.316952500000001, "cost_usd": 8.316952500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 819702, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 122, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4908365, "tokens_cache_write": 461577, "tokens_input": 240, "tokens_output": 49832, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 120, "vendor_mcp": 20 }, "tool_result_tokens": 184144, "tool_time_ms": 6229, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.1343705, "cost_usd": 8.1343705, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 808573, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 113, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4503541, "tokens_cache_write": 457553, "tokens_input": 2299, "tokens_output": 51823, "tool_calls": 135, "tool_calls_by_class": { "cc_builtin": 117, "vendor_mcp": 18 }, "tool_result_tokens": 174020, "tool_time_ms": 6216, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.357082499999999, "cost_usd": 7.357082499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 773404, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 114, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4469415, "tokens_cache_write": 391152, "tokens_input": 221, "tokens_output": 48390, "tool_calls": 135, "tool_calls_by_class": { "cc_builtin": 120, "vendor_mcp": 15 }, "tool_result_tokens": 155954, "tool_time_ms": 11843, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.9885155, "cost_usd": 7.9885155, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 814989, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 120, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4816661, "tokens_cache_write": 427154, "tokens_input": 234, "tokens_output": 52299, "tool_calls": 142, "tool_calls_by_class": { "cc_builtin": 127, "vendor_mcp": 15 }, "tool_result_tokens": 172421, "tool_time_ms": 5225, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.823144, "cost_usd": 7.823144, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 828829, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 121, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4805388, "tokens_cache_write": 408530, "tokens_input": 235, "tokens_output": 53359, "tool_calls": 138, "tool_calls_by_class": { "cc_builtin": 123, "vendor_mcp": 15 }, "tool_result_tokens": 165361, "tool_time_ms": 5721, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.964846500000001, "cost_usd": 7.964846500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 844700, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 128, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5033923, "tokens_cache_write": 409491, "tokens_input": 250, "tokens_output": 54069, "tool_calls": 160, "tool_calls_by_class": { "cc_builtin": 145, "vendor_mcp": 15 }, "tool_result_tokens": 164527, "tool_time_ms": 5818, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.174843000000001, "cost_usd": 8.174843000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 837046, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 117, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4650256, "tokens_cache_write": 455242, "tokens_input": 229, "tokens_output": 51846, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 118, "vendor_mcp": 19 }, "tool_result_tokens": 186672, "tool_time_ms": 6051, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.0351425, "cost_usd": 8.0351425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 798108, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 122, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4998435, "tokens_cache_write": 424039, "tokens_input": 237, "tokens_output": 51774, "tool_calls": 146, "tool_calls_by_class": { "cc_builtin": 130, "vendor_mcp": 16 }, "tool_result_tokens": 173683, "tool_time_ms": 5631, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.6636365, "cost_usd": 7.6636365, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 808592, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 123, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4857973, "tokens_cache_write": 397800, "tokens_input": 240, "tokens_output": 50218, "tool_calls": 144, "tool_calls_by_class": { "cc_builtin": 130, "vendor_mcp": 14 }, "tool_result_tokens": 159792, "tool_time_ms": 5154, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.952433499999999, "cost_usd": 7.952433499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 822775, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 123, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4894067, "tokens_cache_write": 424983, "tokens_input": 239, "tokens_output": 50175, "tool_calls": 147, "tool_calls_by_class": { "cc_builtin": 130, "vendor_mcp": 17 }, "tool_result_tokens": 175287, "tool_time_ms": 5857, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.652916499999999, "cost_usd": 7.652916499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 612698, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 112, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4092843, "tokens_cache_write": 465696, "tokens_input": 222, "tokens_output": 37937, "tool_calls": 118, "tool_calls_by_class": { "cc_builtin": 96, "vendor_mcp": 22 }, "tool_result_tokens": 179748, "tool_time_ms": 8073, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.3278375, "cost_usd": 7.3278375, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 609831, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3745145, "tokens_cache_write": 452942, "tokens_input": 1579, "tokens_output": 36718, "tool_calls": 107, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 19 }, "tool_result_tokens": 158984, "tool_time_ms": 7166, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.418983500000001, "cost_usd": 7.418983500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 601311, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3697607, "tokens_cache_write": 467511, "tokens_input": 204, "tokens_output": 35762, "tool_calls": 105, "tool_calls_by_class": { "cc_builtin": 84, "vendor_mcp": 21 }, "tool_result_tokens": 184690, "tool_time_ms": 7333, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.132093499999999, "cost_usd": 7.132093499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 630420, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 109, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3831667, "tokens_cache_write": 428322, "tokens_input": 218, "tokens_output": 37278, "tool_calls": 114, "tool_calls_by_class": { "cc_builtin": 98, "vendor_mcp": 16 }, "tool_result_tokens": 158242, "tool_time_ms": 6354, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.5270705, "cost_usd": 7.5270705, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 612929, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3812201, "tokens_cache_write": 472513, "tokens_input": 2458, "tokens_output": 35342, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 87, "vendor_mcp": 23 }, "tool_result_tokens": 185173, "tool_time_ms": 7646, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.432802000000001, "cost_usd": 7.432802000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 599326, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3802634, "tokens_cache_write": 460267, "tokens_input": 2203, "tokens_output": 36712, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 87, "vendor_mcp": 22 }, "tool_result_tokens": 179989, "tool_time_ms": 7682, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.007851, "cost_usd": 7.007851, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 585798, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3761302, "tokens_cache_write": 424538, "tokens_input": 214, "tokens_output": 35230, "tool_calls": 105, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 17 }, "tool_result_tokens": 156352, "tool_time_ms": 6526, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.080166, "cost_usd": 7.080166, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 616149, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3837972, "tokens_cache_write": 424644, "tokens_input": 213, "tokens_output": 36547, "tool_calls": 105, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 17 }, "tool_result_tokens": 155844, "tool_time_ms": 6819, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.353891500000002, "cost_usd": 7.353891500000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 601048, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3788943, "tokens_cache_write": 456797, "tokens_input": 415, "tokens_output": 35575, "tool_calls": 104, "tool_calls_by_class": { "cc_builtin": 85, "vendor_mcp": 19 }, "tool_result_tokens": 177100, "tool_time_ms": 6625, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.479546000000001, "cost_usd": 7.479546000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 609956, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/codegraph/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3832912, "tokens_cache_write": 466358, "tokens_input": 212, "tokens_output": 35938, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 22 }, "tool_result_tokens": 184140, "tool_time_ms": 7883, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.370739100000002, "cost_usd": 9.370739100000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 578985, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 100, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 22856167, "tokens_cache_write": 333944, "tokens_input": 200, "tokens_output": 33975, "tool_calls": 80, "tool_calls_by_class": { "cc_builtin": 61, "vendor_mcp": 19 }, "tool_result_tokens": 139355, "tool_time_ms": 5912, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5925838, "cost_usd": 5.5925838, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 474097, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 105, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14230556, "tokens_cache_write": 159072, "tokens_input": 210, "tokens_output": 24557, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 79, "vendor_mcp": 6 }, "tool_result_tokens": 60212, "tool_time_ms": 6131, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.908638300000001, "cost_usd": 5.908638300000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 451898, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 81, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12468821, "tokens_cache_write": 296246, "tokens_input": 162, "tokens_output": 26002, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 50, "vendor_mcp": 11 }, "tool_result_tokens": 80708, "tool_time_ms": 8214, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7575464, "cost_usd": 4.7575464, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 421494, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 75, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11116388, "tokens_cache_write": 181645, "tokens_input": 150, "tokens_output": 22154, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 43, "vendor_mcp": 12 }, "tool_result_tokens": 77976, "tool_time_ms": 4298, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8438086, "cost_usd": 5.8438086, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 492810, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 90, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14268852, "tokens_cache_write": 187913, "tokens_input": 180, "tokens_output": 29009, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 60, "vendor_mcp": 10 }, "tool_result_tokens": 76034, "tool_time_ms": 5399, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.091454099999998, "cost_usd": 6.091454099999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 395368, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 70, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13828587, "tokens_cache_write": 269123, "tokens_input": 140, "tokens_output": 21848, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 29, "vendor_mcp": 21 }, "tool_result_tokens": 133825, "tool_time_ms": 7246, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.507967300000001, "cost_usd": 5.507967300000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 429597, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 73, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12663201, "tokens_cache_write": 217744, "tokens_input": 146, "tokens_output": 26807, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 40, "vendor_mcp": 13 }, "tool_result_tokens": 99082, "tool_time_ms": 4422, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.6622786000000005, "cost_usd": 5.6622786000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 520038, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 105, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14190462, "tokens_cache_write": 163770, "tokens_input": 210, "tokens_output": 28126, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 78, "vendor_mcp": 7 }, "tool_result_tokens": 65274, "tool_time_ms": 4321, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.0376655999999995, "cost_usd": 6.0376655999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 404795, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 77, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14189682, "tokens_cache_write": 239304, "tokens_input": 154, "tokens_output": 22965, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 40, "vendor_mcp": 17 }, "tool_result_tokens": 110547, "tool_time_ms": 4568, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5787526000000005, "cost_usd": 5.5787526000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 432262, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 77, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13406572, "tokens_cache_write": 197879, "tokens_input": 154, "tokens_output": 24603, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 45, "vendor_mcp": 12 }, "tool_result_tokens": 84905, "tool_time_ms": 5074, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8260252, "cost_usd": 2.8260252, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 282411, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 85, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6443324, "tokens_cache_write": 115198, "tokens_input": 170, "tokens_output": 13422, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 14020, "tool_time_ms": 2392, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.590023, "cost_usd": 2.590023, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 341575, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6364990, "tokens_cache_write": 78818, "tokens_input": 166, "tokens_output": 13808, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 12652, "tool_time_ms": 2116, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5316502, "cost_usd": 2.5316502, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 287336, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6195624, "tokens_cache_write": 76679, "tokens_input": 168, "tokens_output": 14159, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 11322, "tool_time_ms": 1892, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4404013, "cost_usd": 2.4404013, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 298363, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 79, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5919741, "tokens_cache_write": 76305, "tokens_input": 158, "tokens_output": 13745, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 10884, "tool_time_ms": 2272, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.501409, "cost_usd": 2.501409, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 260623, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6212840, "tokens_cache_write": 76274, "tokens_input": 166, "tokens_output": 11961, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 12305, "tool_time_ms": 2468, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6732393999999995, "cost_usd": 2.6732393999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 294272, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 88, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6567838, "tokens_cache_write": 79940, "tokens_input": 176, "tokens_output": 14848, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 12613, "tool_time_ms": 1911, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5673466000000005, "cost_usd": 2.5673466000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 377469, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 84, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6327322, "tokens_cache_write": 79411, "tokens_input": 168, "tokens_output": 12812, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 13512, "tool_time_ms": 1792, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6669342999999994, "cost_usd": 2.6669342999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 302817, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 85, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6624001, "tokens_cache_write": 79039, "tokens_input": 170, "tokens_output": 13666, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 54, "vendor_mcp": 1 }, "tool_result_tokens": 15878, "tool_time_ms": 2350, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7513698999999994, "cost_usd": 2.7513698999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 312761, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 86, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6731853, "tokens_cache_write": 83773, "tokens_input": 172, "tokens_output": 15244, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 14654, "tool_time_ms": 2006, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6188199999999995, "cost_usd": 2.6188199999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 295242, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 82, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6358860, "tokens_cache_write": 81525, "tokens_input": 164, "tokens_output": 14768, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 13867, "tool_time_ms": 1925, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.694566699999999, "cost_usd": 5.694566699999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 649058, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 117, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6016309, "tokens_cache_write": 550317, "tokens_input": 234, "tokens_output": 39138, "tool_calls": 101, "tool_calls_by_class": { "cc_builtin": 68, "vendor_mcp": 33 }, "tool_result_tokens": 234442, "tool_time_ms": 6265, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.045950699999999, "cost_usd": 6.045950699999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 647943, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 123, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6768689, "tokens_cache_write": 570321, "tokens_input": 246, "tokens_output": 39512, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 74, "vendor_mcp": 32 }, "tool_result_tokens": 223853, "tool_time_ms": 6158, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.7101028000000005, "cost_usd": 5.7101028000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 594653, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 110, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5994646, "tokens_cache_write": 561124, "tokens_input": 220, "tokens_output": 36287, "tool_calls": 92, "tool_calls_by_class": { "cc_builtin": 50, "vendor_mcp": 42 }, "tool_result_tokens": 261980, "tool_time_ms": 7487, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.651483000000001, "cost_usd": 6.651483000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 648946, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 118, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6330750, "tokens_cache_write": 698465, "tokens_input": 236, "tokens_output": 37384, "tool_calls": 99, "tool_calls_by_class": { "cc_builtin": 55, "vendor_mcp": 44 }, "tool_result_tokens": 288923, "tool_time_ms": 6605, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.571137299999999, "cost_usd": 6.571137299999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 712766, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 129, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7338051, "tokens_cache_write": 620588, "tokens_input": 258, "tokens_output": 43028, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 66, "vendor_mcp": 44 }, "tool_result_tokens": 295322, "tool_time_ms": 12311, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2082265, "cost_usd": 5.2082265, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 655859, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 114, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5823805, "tokens_cache_write": 481091, "tokens_input": 228, "tokens_output": 38257, "tool_calls": 101, "tool_calls_by_class": { "cc_builtin": 70, "vendor_mcp": 31 }, "tool_result_tokens": 214352, "tool_time_ms": 5573, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.547350099999999, "cost_usd": 5.547350099999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 650973, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 111, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6012327, "tokens_cache_write": 521736, "tokens_input": 222, "tokens_output": 40838, "tool_calls": 96, "tool_calls_by_class": { "cc_builtin": 59, "vendor_mcp": 37 }, "tool_result_tokens": 238328, "tool_time_ms": 7379, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.032418999999999, "cost_usd": 5.032418999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 666892, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 108, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5741730, "tokens_cache_write": 456307, "tokens_input": 216, "tokens_output": 38094, "tool_calls": 91, "tool_calls_by_class": { "cc_builtin": 64, "vendor_mcp": 27 }, "tool_result_tokens": 197662, "tool_time_ms": 6291, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.250655199999999, "cost_usd": 5.250655199999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 615285, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 115, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5972764, "tokens_cache_write": 494426, "tokens_input": 230, "tokens_output": 32772, "tool_calls": 95, "tool_calls_by_class": { "cc_builtin": 62, "vendor_mcp": 33 }, "tool_result_tokens": 224094, "tool_time_ms": 6423, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.902688699999999, "cost_usd": 5.902688699999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 797388, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 123, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6736089, "tokens_cache_write": 536049, "tokens_input": 246, "tokens_output": 44322, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 75, "vendor_mcp": 35 }, "tool_result_tokens": 243494, "tool_time_ms": 10729, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.173744800000001, "cost_usd": 5.173744800000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 395202, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4515636, "tokens_cache_write": 583400, "tokens_input": 188, "tokens_output": 21206, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 29, "vendor_mcp": 35 }, "tool_result_tokens": 233130, "tool_time_ms": 6913, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.696387100000001, "cost_usd": 5.696387100000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 502121, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 109, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5314187, "tokens_cache_write": 619787, "tokens_input": 218, "tokens_output": 25517, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 43, "vendor_mcp": 39 }, "tool_result_tokens": 246491, "tool_time_ms": 8122, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.1547392, "cost_usd": 5.1547392, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 466080, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 102, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4956104, "tokens_cache_write": 551686, "tokens_input": 204, "tokens_output": 23812, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 38, "vendor_mcp": 36 }, "tool_result_tokens": 231200, "tool_time_ms": 7388, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.421473100000001, "cost_usd": 5.421473100000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 466622, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 108, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5271377, "tokens_cache_write": 575242, "tokens_input": 216, "tokens_output": 25864, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 41, "vendor_mcp": 37 }, "tool_result_tokens": 244508, "tool_time_ms": 7525, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.553204149999999, "cost_usd": 5.553204149999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 481933, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 117, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5608738, "tokens_cache_write": 588100, "tokens_input": 234, "tokens_output": 25019, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 46, "subagent": 2, "vendor_mcp": 38 }, "tool_result_tokens": 243076, "tool_time_ms": 7619, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.745159, "cost_usd": 5.745159, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 492559, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 116, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5737070, "tokens_cache_write": 607723, "tokens_input": 232, "tokens_output": 27139, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 50, "subagent": 2, "vendor_mcp": 38 }, "tool_result_tokens": 255728, "tool_time_ms": 8009, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.0685735, "cost_usd": 5.0685735, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 463139, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 102, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4968695, "tokens_cache_write": 538168, "tokens_input": 204, "tokens_output": 23223, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 38, "vendor_mcp": 35 }, "tool_result_tokens": 220639, "tool_time_ms": 7449, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.6551752, "cost_usd": 5.6551752, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 485935, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 104, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5071644, "tokens_cache_write": 623278, "tokens_input": 208, "tokens_output": 26226, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 32, "vendor_mcp": 42 }, "tool_result_tokens": 269124, "tool_time_ms": 8293, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.614179299999999, "cost_usd": 5.614179299999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 507984, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 113, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5538311, "tokens_cache_write": 589843, "tokens_input": 226, "tokens_output": 27530, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 49, "vendor_mcp": 37 }, "tool_result_tokens": 251832, "tool_time_ms": 7799, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codegraph", "arm_label": "CC+CodeGraph", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.282933399999999, "cost_usd": 5.282933399999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 458744, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 104, "model": "claude-sonnet-5", "path": "runs/codegraph/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codegraph|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5008988, "tokens_cache_write": 566433, "tokens_input": 208, "tokens_output": 25401, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 37, "vendor_mcp": 37 }, "tool_result_tokens": 240200, "tool_time_ms": 7828, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.05303216, "denominator": 20, "duration_basis": "run", "duration_ms": 168610, "exam": "HumanExam", "exam_epoch": null, "fails": 10, "family": "harness_bare", "llm_calls": 17, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1111808, "tokens_cache_write": 0, "tokens_input": 109706, "tokens_output": 7379, "tool_calls": 16, "tool_calls_by_class": { "harness_builtin": 16 }, "tool_result_tokens": 78124, "tool_time_ms": 7873, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.04903096, "denominator": 20, "duration_basis": "run", "duration_ms": 144843, "exam": "HumanExam", "exam_epoch": null, "fails": 10, "family": "harness_bare", "llm_calls": 17, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 973568, "tokens_cache_write": 0, "tokens_input": 107658, "tokens_output": 6690, "tool_calls": 16, "tool_calls_by_class": { "harness_builtin": 16 }, "tool_result_tokens": 77215, "tool_time_ms": 4430, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.052951279999999996, "denominator": 20, "duration_basis": "run", "duration_ms": 132839, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 18, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1073664, "tokens_cache_write": 0, "tokens_input": 119230, "tokens_output": 6360, "tool_calls": 17, "tool_calls_by_class": { "harness_builtin": 17 }, "tool_result_tokens": 88528, "tool_time_ms": 2202, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.08637704, "denominator": 20, "duration_basis": "run", "duration_ms": 203780, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 27, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2129152, "tokens_cache_write": 0, "tokens_input": 162708, "tokens_output": 9377, "tool_calls": 26, "tool_calls_by_class": { "harness_builtin": 26 }, "tool_result_tokens": 100001, "tool_time_ms": 3412, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.04570148, "denominator": 20, "duration_basis": "run", "duration_ms": 144173, "exam": "HumanExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 15, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 816384, "tokens_cache_write": 0, "tokens_input": 106603, "tokens_output": 6711, "tool_calls": 14, "tool_calls_by_class": { "harness_builtin": 14 }, "tool_result_tokens": 77230, "tool_time_ms": 3346, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.05083128, "denominator": 20, "duration_basis": "run", "duration_ms": 155710, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 16, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 963584, "tokens_cache_write": 0, "tokens_input": 114946, "tokens_output": 7142, "tool_calls": 15, "tool_calls_by_class": { "harness_builtin": 15 }, "tool_result_tokens": 87086, "tool_time_ms": 2254, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.06967544, "denominator": 20, "duration_basis": "run", "duration_ms": 197133, "exam": "HumanExam", "exam_epoch": null, "fails": 10, "family": "harness_bare", "llm_calls": 24, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1728512, "tokens_cache_write": 0, "tokens_input": 126110, "tokens_output": 8236, "tool_calls": 23, "tool_calls_by_class": { "harness_builtin": 23 }, "tool_result_tokens": 84919, "tool_time_ms": 13239, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.04991052, "denominator": 20, "duration_basis": "run", "duration_ms": 156848, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 18, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 969216, "tokens_cache_write": 0, "tokens_input": 111315, "tokens_output": 6886, "tool_calls": 17, "tool_calls_by_class": { "harness_builtin": 17 }, "tool_result_tokens": 78008, "tool_time_ms": 10041, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.046605440000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 138354, "exam": "HumanExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 15, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 869632, "tokens_cache_write": 0, "tokens_input": 108444, "tokens_output": 6270, "tool_calls": 14, "tool_calls_by_class": { "harness_builtin": 14 }, "tool_result_tokens": 81195, "tool_time_ms": 8305, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.03900324, "denominator": 20, "duration_basis": "run", "duration_ms": 137813, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 12, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 586752, "tokens_cache_write": 0, "tokens_input": 94851, "tokens_output": 6915, "tool_calls": 11, "tool_calls_by_class": { "harness_builtin": 11 }, "tool_result_tokens": 70575, "tool_time_ms": 1416, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.047705159999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 139022, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 14, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 849408, "tokens_cache_write": 0, "tokens_input": 113379, "tokens_output": 6701, "tool_calls": 13, "tool_calls_by_class": { "harness_builtin": 13 }, "tool_result_tokens": 85445, "tool_time_ms": 1048, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.04501712, "denominator": 30, "duration_basis": "run", "duration_ms": 147341, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 13, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 710656, "tokens_cache_write": 0, "tokens_input": 108528, "tokens_output": 7582, "tool_calls": 12, "tool_calls_by_class": { "harness_builtin": 12 }, "tool_result_tokens": 76201, "tool_time_ms": 944, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.03742848, "denominator": 30, "duration_basis": "run", "duration_ms": 127411, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 11, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 505344, "tokens_cache_write": 0, "tokens_input": 98184, "tokens_output": 6404, "tool_calls": 10, "tool_calls_by_class": { "harness_builtin": 10 }, "tool_result_tokens": 67373, "tool_time_ms": 918, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.031375679999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 104938, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 423424, "tokens_cache_write": 0, "tokens_input": 84428, "tokens_output": 5018, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 65342, "tool_time_ms": 668, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.042280080000000005, "denominator": 30, "duration_basis": "run", "duration_ms": 169719, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 12, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 625664, "tokens_cache_write": 0, "tokens_input": 99082, "tokens_output": 8292, "tool_calls": 11, "tool_calls_by_class": { "harness_builtin": 11 }, "tool_result_tokens": 76343, "tool_time_ms": 1332, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.03430816, "denominator": 30, "duration_basis": "run", "duration_ms": 107343, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 419328, "tokens_cache_write": 0, "tokens_input": 97646, "tokens_output": 5327, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 77572, "tool_time_ms": 760, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.03398072, "denominator": 30, "duration_basis": "run", "duration_ms": 118886, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 11, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 491776, "tokens_cache_write": 0, "tokens_input": 84948, "tokens_output": 5963, "tool_calls": 10, "tool_calls_by_class": { "harness_builtin": 10 }, "tool_result_tokens": 63600, "tool_time_ms": 826, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.041123679999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 135926, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 13, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 706304, "tokens_cache_write": 0, "tokens_input": 95424, "tokens_output": 6594, "tool_calls": 12, "tool_calls_by_class": { "harness_builtin": 12 }, "tool_result_tokens": 71702, "tool_time_ms": 862, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.03347244, "denominator": 30, "duration_basis": "run", "duration_ms": 125240, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 12, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 463872, "tokens_cache_write": 0, "tokens_input": 84417, "tokens_output": 6093, "tool_calls": 11, "tool_calls_by_class": { "harness_builtin": 11 }, "tool_result_tokens": 61116, "tool_time_ms": 1041, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.05269432, "denominator": 30, "duration_basis": "run", "duration_ms": 135331, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 19, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1121536, "tokens_cache_write": 0, "tokens_input": 113500, "tokens_output": 6303, "tool_calls": 18, "tool_calls_by_class": { "harness_builtin": 18 }, "tool_result_tokens": 82429, "tool_time_ms": 1236, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19470592000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 512141, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 72, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6916096, "tokens_cache_write": 0, "tokens_input": 203860, "tokens_output": 13010, "tool_calls": 52, "tool_calls_by_class": { "harness_builtin": 52 }, "tool_result_tokens": 136064, "tool_time_ms": 4047, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.20638736000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 345809, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 72, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7282688, "tokens_cache_write": 0, "tokens_input": 219206, "tokens_output": 14077, "tool_calls": 52, "tool_calls_by_class": { "harness_builtin": 52 }, "tool_result_tokens": 144867, "tool_time_ms": 5974, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.1855744, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 328743, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 73, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6553600, "tokens_cache_write": 0, "tokens_input": 198322, "tokens_output": 12365, "tool_calls": 52, "tool_calls_by_class": { "harness_builtin": 52 }, "tool_result_tokens": 126537, "tool_time_ms": 6290, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.18032264, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 351999, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 67, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6045952, "tokens_cache_write": 0, "tokens_input": 206100, "tokens_output": 15153, "tool_calls": 47, "tool_calls_by_class": { "harness_builtin": 47 }, "tool_result_tokens": 134037, "tool_time_ms": 4584, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2110028, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 414314, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 75, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7522560, "tokens_cache_write": 0, "tokens_input": 208330, "tokens_output": 15738, "tool_calls": 55, "tool_calls_by_class": { "harness_builtin": 55 }, "tool_result_tokens": 129993, "tool_time_ms": 13173, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.22656184, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 415244, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 80, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 8148992, "tokens_cache_write": 0, "tokens_input": 216840, "tokens_output": 16845, "tool_calls": 60, "tool_calls_by_class": { "harness_builtin": 60 }, "tool_result_tokens": 135091, "tool_time_ms": 21883, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.1912726, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 374446, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 77, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6813440, "tokens_cache_write": 0, "tokens_input": 188727, "tokens_output": 14382, "tool_calls": 57, "tool_calls_by_class": { "harness_builtin": 57 }, "tool_result_tokens": 111191, "tool_time_ms": 13072, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.21874896, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 365475, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 76, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7948288, "tokens_cache_write": 0, "tokens_input": 218222, "tokens_output": 13449, "tool_calls": 56, "tool_calls_by_class": { "harness_builtin": 56 }, "tool_result_tokens": 139278, "tool_time_ms": 5738, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.20835208000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 396913, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 76, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7409664, "tokens_cache_write": 0, "tokens_input": 212246, "tokens_output": 14758, "tool_calls": 56, "tool_calls_by_class": { "harness_builtin": 56 }, "tool_result_tokens": 131187, "tool_time_ms": 14464, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.21424211999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 353778, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 73, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7749376, "tokens_cache_write": 0, "tokens_input": 209387, "tokens_output": 14481, "tool_calls": 53, "tool_calls_by_class": { "harness_builtin": 53 }, "tool_result_tokens": 133474, "tool_time_ms": 4394, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.14144232, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 328360, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 80, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4386816, "tokens_cache_write": 0, "tokens_input": 189096, "tokens_output": 13239, "tool_calls": 50, "tool_calls_by_class": { "harness_builtin": 50 }, "tool_result_tokens": 83204, "tool_time_ms": 3838, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.1330498, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 271368, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4422400, "tokens_cache_write": 0, "tokens_input": 164077, "tokens_output": 9822, "tool_calls": 51, "tool_calls_by_class": { "harness_builtin": 51 }, "tool_result_tokens": 93666, "tool_time_ms": 3113, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.17967080000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 407485, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 94, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6346240, "tokens_cache_write": 0, "tokens_input": 170496, "tokens_output": 15539, "tool_calls": 64, "tool_calls_by_class": { "harness_builtin": 64 }, "tool_result_tokens": 89209, "tool_time_ms": 3551, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.15591088, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 324465, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 78, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 5018624, "tokens_cache_write": 0, "tokens_input": 212160, "tokens_output": 10922, "tool_calls": 48, "tool_calls_by_class": { "harness_builtin": 48 }, "tool_result_tokens": 98972, "tool_time_ms": 3136, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.13072287999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 330702, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4126464, "tokens_cache_write": 0, "tokens_input": 167090, "tokens_output": 12313, "tool_calls": 51, "tool_calls_by_class": { "harness_builtin": 51 }, "tool_result_tokens": 80519, "tool_time_ms": 2774, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.15524828, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 330736, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 79, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 5122304, "tokens_cache_write": 0, "tokens_input": 181967, "tokens_output": 13674, "tool_calls": 49, "tool_calls_by_class": { "harness_builtin": 49 }, "tool_result_tokens": 107277, "tool_time_ms": 3147, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.18129532, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 367421, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 86, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6370816, "tokens_cache_write": 0, "tokens_input": 188875, "tokens_output": 13420, "tool_calls": 56, "tool_calls_by_class": { "harness_builtin": 56 }, "tool_result_tokens": 111351, "tool_time_ms": 3800, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.11195688, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 333243, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 75, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3184384, "tokens_cache_write": 0, "tokens_input": 178928, "tokens_output": 10403, "tool_calls": 45, "tool_calls_by_class": { "harness_builtin": 45 }, "tool_result_tokens": 66237, "tool_time_ms": 2489, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.13816035999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 345976, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 74, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4720128, "tokens_cache_write": 0, "tokens_input": 151637, "tokens_output": 11192, "tool_calls": 44, "tool_calls_by_class": { "harness_builtin": 44 }, "tool_result_tokens": 86343, "tool_time_ms": 3779, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.10995672000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 301568, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3515136, "tokens_cache_write": 0, "tokens_input": 129702, "tokens_output": 11428, "tool_calls": 51, "tool_calls_by_class": { "harness_builtin": 51 }, "tool_result_tokens": 65112, "tool_time_ms": 3041, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.1888806, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 746744, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 119, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2078720, "tokens_cache_write": 0, "tokens_input": 588679, "tokens_output": 24642, "tool_calls": 99, "tool_calls_by_class": { "harness_builtin": 99 }, "tool_result_tokens": 376347, "tool_time_ms": 64742, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.18882736, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 639809, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 119, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2151168, "tokens_cache_write": 0, "tokens_input": 586778, "tokens_output": 23707, "tool_calls": 98, "tool_calls_by_class": { "harness_builtin": 98 }, "tool_result_tokens": 383804, "tool_time_ms": 22585, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.17343848, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 561502, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 107, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1929984, "tokens_cache_write": 0, "tokens_input": 544348, "tokens_output": 21641, "tool_calls": 87, "tool_calls_by_class": { "harness_builtin": 87 }, "tool_result_tokens": 394920, "tool_time_ms": 11478, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.20295876000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 752805, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "harness_bare", "llm_calls": 129, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2496768, "tokens_cache_write": 0, "tokens_input": 597867, "tokens_output": 27875, "tool_calls": 109, "tool_calls_by_class": { "harness_builtin": 109 }, "tool_result_tokens": 410401, "tool_time_ms": 51773, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.18617072, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 765558, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 116, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2155776, "tokens_cache_write": 0, "tokens_input": 556228, "tokens_output": 26508, "tool_calls": 96, "tool_calls_by_class": { "harness_builtin": 96 }, "tool_result_tokens": 393454, "tool_time_ms": 51552, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.1868388, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 663448, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 123, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2511360, "tokens_cache_write": 0, "tokens_input": 529068, "tokens_output": 25665, "tool_calls": 103, "tool_calls_by_class": { "harness_builtin": 103 }, "tool_result_tokens": 386618, "tool_time_ms": 25151, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.189012, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 721853, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 119, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1923840, "tokens_cache_write": 0, "tokens_input": 601752, "tokens_output": 25154, "tool_calls": 99, "tool_calls_by_class": { "harness_builtin": 99 }, "tool_result_tokens": 400103, "tool_time_ms": 49519, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.17582256, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 645582, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 116, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2107648, "tokens_cache_write": 0, "tokens_input": 522764, "tokens_output": 24264, "tool_calls": 96, "tool_calls_by_class": { "harness_builtin": 96 }, "tool_result_tokens": 382763, "tool_time_ms": 9655, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.21749932000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 729822, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 127, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2442496, "tokens_cache_write": 0, "tokens_input": 690259, "tokens_output": 25498, "tool_calls": 107, "tool_calls_by_class": { "harness_builtin": 107 }, "tool_result_tokens": 438039, "tool_time_ms": 48001, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19875051999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 796942, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 127, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2329856, "tokens_cache_write": 0, "tokens_input": 601719, "tokens_output": 26508, "tool_calls": 107, "tool_calls_by_class": { "harness_builtin": 107 }, "tool_result_tokens": 405305, "tool_time_ms": 41425, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19873072, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 602370, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 120, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1607936, "tokens_cache_write": 0, "tokens_input": 711612, "tokens_output": 20208, "tool_calls": 90, "tool_calls_by_class": { "harness_builtin": 90 }, "tool_result_tokens": 410788, "tool_time_ms": 7652, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.20393852, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 627400, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 122, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1700096, "tokens_cache_write": 0, "tokens_input": 718559, "tokens_output": 21854, "tool_calls": 92, "tool_calls_by_class": { "harness_builtin": 92 }, "tool_result_tokens": 416491, "tool_time_ms": 8493, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.20064424, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 594335, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 126, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1914112, "tokens_cache_write": 0, "tokens_input": 682672, "tokens_output": 21523, "tool_calls": 96, "tool_calls_by_class": { "harness_builtin": 96 }, "tool_result_tokens": 426970, "tool_time_ms": 8705, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.18603816, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 592951, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 118, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1485568, "tokens_cache_write": 0, "tokens_input": 666986, "tokens_output": 19108, "tool_calls": 88, "tool_calls_by_class": { "harness_builtin": 88 }, "tool_result_tokens": 382256, "tool_time_ms": 7459, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19497556, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 595403, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 128, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1987328, "tokens_cache_write": 0, "tokens_input": 645861, "tokens_output": 21714, "tool_calls": 98, "tool_calls_by_class": { "harness_builtin": 98 }, "tool_result_tokens": 431079, "tool_time_ms": 8240, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.21828172, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 619891, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 131, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1881856, "tokens_cache_write": 0, "tokens_input": 776689, "tokens_output": 21089, "tool_calls": 101, "tool_calls_by_class": { "harness_builtin": 101 }, "tool_result_tokens": 437456, "tool_time_ms": 7762, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.20753444000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 608005, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 127, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1787392, "tokens_cache_write": 0, "tokens_input": 737979, "tokens_output": 20159, "tool_calls": 97, "tool_calls_by_class": { "harness_builtin": 97 }, "tool_result_tokens": 430514, "tool_time_ms": 8336, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.20849768000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 624302, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 130, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1886464, "tokens_cache_write": 0, "tokens_input": 722274, "tokens_output": 21928, "tool_calls": 100, "tool_calls_by_class": { "harness_builtin": 100 }, "tool_result_tokens": 426272, "tool_time_ms": 8488, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.21656296, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 629452, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 133, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2060288, "tokens_cache_write": 0, "tokens_input": 747852, "tokens_output": 21489, "tool_calls": 103, "tool_calls_by_class": { "harness_builtin": 103 }, "tool_result_tokens": 450086, "tool_time_ms": 8655, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.19763899999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 606099, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 120, "model": "gpt-5.6-luna", "path": "runs/codex/gpt-5.6-luna/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1783040, "tokens_cache_write": 0, "tokens_input": 688313, "tokens_output": 20263, "tool_calls": 90, "tool_calls_by_class": { "harness_builtin": 90 }, "tool_result_tokens": 442444, "tool_time_ms": 8863, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2938904, "denominator": 20, "duration_basis": "run", "duration_ms": 49022, "exam": "HumanExam", "exam_epoch": null, "fails": 10, "family": "harness_bare", "llm_calls": 8, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 193536, "tokens_cache_write": 0, "tokens_input": 43179, "tokens_output": 2188, "tool_calls": 7, "tool_calls_by_class": { "harness_builtin": 7 }, "tool_result_tokens": 31373, "tool_time_ms": 492, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2759176, "denominator": 20, "duration_basis": "run", "duration_ms": 51124, "exam": "HumanExam", "exam_epoch": null, "fails": 13, "family": "harness_bare", "llm_calls": 9, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 7, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 214784, "tokens_cache_write": 0, "tokens_input": 36396, "tokens_output": 2221, "tool_calls": 8, "tool_calls_by_class": { "harness_builtin": 8 }, "tool_result_tokens": 22389, "tool_time_ms": 610, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.44819600000000004, "denominator": 20, "duration_basis": "run", "duration_ms": 60506, "exam": "HumanExam", "exam_epoch": null, "fails": 12, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 8, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 234240, "tokens_cache_write": 0, "tokens_input": 75425, "tokens_output": 2640, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 25434, "tool_time_ms": 558, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2743768, "denominator": 20, "duration_basis": "run", "duration_ms": 63196, "exam": "HumanExam", "exam_epoch": null, "fails": 13, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 7, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 210432, "tokens_cache_write": 0, "tokens_input": 34371, "tokens_output": 2636, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 20984, "tool_time_ms": 587, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.4994384, "denominator": 20, "duration_basis": "run", "duration_ms": 57985, "exam": "HumanExam", "exam_epoch": null, "fails": 14, "family": "harness_bare", "llm_calls": 11, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 6, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 284416, "tokens_cache_write": 0, "tokens_input": 83173, "tokens_output": 2649, "tool_calls": 10, "tool_calls_by_class": { "harness_builtin": 10 }, "tool_result_tokens": 34457, "tool_time_ms": 569, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.27380160000000003, "denominator": 20, "duration_basis": "run", "duration_ms": 55443, "exam": "HumanExam", "exam_epoch": null, "fails": 12, "family": "harness_bare", "llm_calls": 8, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 8, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 185344, "tokens_cache_write": 0, "tokens_input": 37026, "tokens_output": 2578, "tool_calls": 7, "tool_calls_by_class": { "harness_builtin": 7 }, "tool_result_tokens": 26557, "tool_time_ms": 455, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.31252159999999995, "denominator": 20, "duration_basis": "run", "duration_ms": 51387, "exam": "HumanExam", "exam_epoch": null, "fails": 11, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 9, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 254464, "tokens_cache_write": 0, "tokens_input": 41114, "tokens_output": 2314, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 28440, "tool_time_ms": 589, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.27082720000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 45740, "exam": "HumanExam", "exam_epoch": null, "fails": 13, "family": "harness_bare", "llm_calls": 8, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 7, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 171008, "tokens_cache_write": 0, "tokens_input": 40266, "tokens_output": 2068, "tool_calls": 7, "tool_calls_by_class": { "harness_builtin": 7 }, "tool_result_tokens": 29056, "tool_time_ms": 447, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.308832, "denominator": 20, "duration_basis": "run", "duration_ms": 53948, "exam": "HumanExam", "exam_epoch": null, "fails": 12, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 8, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 227840, "tokens_cache_write": 0, "tokens_input": 43484, "tokens_output": 2188, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 29899, "tool_time_ms": 595, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2708936, "denominator": 20, "duration_basis": "run", "duration_ms": 44169, "exam": "HumanExam", "exam_epoch": null, "fails": 11, "family": "harness_bare", "llm_calls": 9, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 9, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 189184, "tokens_cache_write": 0, "tokens_input": 39165, "tokens_output": 1928, "tool_calls": 8, "tool_calls_by_class": { "harness_builtin": 8 }, "tool_result_tokens": 25600, "tool_time_ms": 435, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.30212, "denominator": 30, "duration_basis": "run", "duration_ms": 81638, "exam": "HumanExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 24, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 238080, "tokens_cache_write": 0, "tokens_input": 33357, "tokens_output": 3673, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 24023, "tool_time_ms": 653, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.36204400000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 57384, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 234240, "tokens_cache_write": 0, "tokens_input": 53287, "tokens_output": 2760, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 31762, "tool_time_ms": 556, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.246032, "denominator": 30, "duration_basis": "run", "duration_ms": 59093, "exam": "HumanExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 7, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 24, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 165120, "tokens_cache_write": 0, "tokens_input": 30606, "tokens_output": 2878, "tool_calls": 6, "tool_calls_by_class": { "harness_builtin": 6 }, "tool_result_tokens": 22459, "tool_time_ms": 487, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3704808, "denominator": 30, "duration_basis": "run", "duration_ms": 66609, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 10, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 247552, "tokens_cache_write": 0, "tokens_input": 52605, "tokens_output": 3052, "tool_calls": 9, "tool_calls_by_class": { "harness_builtin": 9 }, "tool_result_tokens": 31839, "tool_time_ms": 603, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.36015600000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 62643, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 9, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 218880, "tokens_cache_write": 0, "tokens_input": 53376, "tokens_output": 2955, "tool_calls": 8, "tool_calls_by_class": { "harness_builtin": 8 }, "tool_result_tokens": 28977, "tool_time_ms": 466, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2739352, "denominator": 30, "duration_basis": "run", "duration_ms": 61685, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 9, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 195328, "tokens_cache_write": 0, "tokens_input": 34636, "tokens_output": 2863, "tool_calls": 8, "tool_calls_by_class": { "harness_builtin": 8 }, "tool_result_tokens": 25506, "tool_time_ms": 416, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.2858792, "denominator": 30, "duration_basis": "run", "duration_ms": 55822, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 8, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 171008, "tokens_cache_write": 0, "tokens_input": 41174, "tokens_output": 2639, "tool_calls": 7, "tool_calls_by_class": { "harness_builtin": 7 }, "tool_result_tokens": 27004, "tool_time_ms": 464, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3055184, "denominator": 30, "duration_basis": "run", "duration_ms": 69135, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 8, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 188416, "tokens_cache_write": 0, "tokens_input": 40993, "tokens_output": 3309, "tool_calls": 7, "tool_calls_by_class": { "harness_builtin": 7 }, "tool_result_tokens": 28794, "tool_time_ms": 517, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.4051264, "denominator": 30, "duration_basis": "run", "duration_ms": 75439, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 11, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 23, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 389376, "tokens_cache_write": 0, "tokens_input": 45334, "tokens_output": 3402, "tool_calls": 10, "tool_calls_by_class": { "harness_builtin": 10 }, "tool_result_tokens": 32116, "tool_time_ms": 560, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.4814832, "denominator": 30, "duration_basis": "run", "duration_ms": 85221, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 66, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1630208, "tokens_cache_write": 0, "tokens_input": 167930, "tokens_output": 7884, "tool_calls": 60, "tool_calls_by_class": { "harness_builtin": 43, "subagent": 17 }, "tool_result_tokens": 76123, "tool_time_ms": 13635, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5146384, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 228137, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 66, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4507136, "tokens_cache_write": 0, "tokens_input": 138086, "tokens_output": 7972, "tool_calls": 46, "tool_calls_by_class": { "harness_builtin": 46 }, "tool_result_tokens": 78364, "tool_time_ms": 3274, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.915124, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 268648, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 74, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 5373440, "tokens_cache_write": 0, "tokens_input": 145617, "tokens_output": 9164, "tool_calls": 54, "tool_calls_by_class": { "harness_builtin": 54 }, "tool_result_tokens": 81026, "tool_time_ms": 3468, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.51872, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 242840, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 66, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4400640, "tokens_cache_write": 0, "tokens_input": 145686, "tokens_output": 8786, "tool_calls": 46, "tool_calls_by_class": { "harness_builtin": 46 }, "tool_result_tokens": 84544, "tool_time_ms": 3181, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5521216000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 242890, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 70, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4587264, "tokens_cache_write": 0, "tokens_input": 137519, "tokens_output": 8357, "tool_calls": 49, "tool_calls_by_class": { "harness_builtin": 49 }, "tool_result_tokens": 76445, "tool_time_ms": 3066, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.4889928, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 293802, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 69, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4375552, "tokens_cache_write": 0, "tokens_input": 141638, "tokens_output": 8611, "tool_calls": 49, "tool_calls_by_class": { "harness_builtin": 49 }, "tool_result_tokens": 72299, "tool_time_ms": 6071, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7200416, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 291535, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 73, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4792064, "tokens_cache_write": 0, "tokens_input": 156874, "tokens_output": 8786, "tool_calls": 53, "tool_calls_by_class": { "harness_builtin": 53 }, "tool_result_tokens": 79359, "tool_time_ms": 4729, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5244328, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 259228, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 68, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4462592, "tokens_cache_write": 0, "tokens_input": 142819, "tokens_output": 8406, "tool_calls": 48, "tool_calls_by_class": { "harness_builtin": 48 }, "tool_result_tokens": 78480, "tool_time_ms": 3056, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3676472000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 228354, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 71, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4329728, "tokens_cache_write": 0, "tokens_input": 122609, "tokens_output": 7266, "tool_calls": 51, "tool_calls_by_class": { "harness_builtin": 51 }, "tool_result_tokens": 65308, "tool_time_ms": 3958, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.631432, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 328351, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 66, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4514560, "tokens_cache_write": 0, "tokens_input": 154882, "tokens_output": 10304, "tool_calls": 46, "tool_calls_by_class": { "harness_builtin": 46 }, "tool_result_tokens": 81298, "tool_time_ms": 3694, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3556280000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 257187, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 67, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4194560, "tokens_cache_write": 0, "tokens_input": 127451, "tokens_output": 8400, "tool_calls": 47, "tool_calls_by_class": { "harness_builtin": 47 }, "tool_result_tokens": 71426, "tool_time_ms": 3204, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7287736, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 276936, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 80, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2582784, "tokens_cache_write": 0, "tokens_input": 133230, "tokens_output": 8137, "tool_calls": 50, "tool_calls_by_class": { "harness_builtin": 50 }, "tool_result_tokens": 34738, "tool_time_ms": 3181, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3172752, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 275623, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 92, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3550208, "tokens_cache_write": 0, "tokens_input": 180213, "tokens_output": 8817, "tool_calls": 62, "tool_calls_by_class": { "harness_builtin": 62 }, "tool_result_tokens": 44815, "tool_time_ms": 3293, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7958888, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 260446, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 83, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2871552, "tokens_cache_write": 0, "tokens_input": 119167, "tokens_output": 8530, "tool_calls": 53, "tool_calls_by_class": { "harness_builtin": 53 }, "tool_result_tokens": 40339, "tool_time_ms": 2522, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8058704, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 259767, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3033856, "tokens_cache_write": 0, "tokens_input": 108792, "tokens_output": 7858, "tool_calls": 51, "tool_calls_by_class": { "harness_builtin": 51 }, "tool_result_tokens": 43881, "tool_time_ms": 2881, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.5334752, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 236363, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 73, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2458368, "tokens_cache_write": 0, "tokens_input": 99157, "tokens_output": 7675, "tool_calls": 43, "tool_calls_by_class": { "harness_builtin": 43 }, "tool_result_tokens": 40159, "tool_time_ms": 2098, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.671416, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 248809, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 78, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2757120, "tokens_cache_write": 0, "tokens_input": 104077, "tokens_output": 7613, "tool_calls": 48, "tool_calls_by_class": { "harness_builtin": 48 }, "tool_result_tokens": 41393, "tool_time_ms": 2527, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.4687936000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 245436, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 70, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2369024, "tokens_cache_write": 0, "tokens_input": 93186, "tokens_output": 7422, "tool_calls": 40, "tool_calls_by_class": { "harness_builtin": 40 }, "tool_result_tokens": 38293, "tool_time_ms": 3021, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8545832, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 234610, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 90, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2861568, "tokens_cache_write": 0, "tokens_input": 138804, "tokens_output": 7737, "tool_calls": 60, "tool_calls_by_class": { "harness_builtin": 60 }, "tool_result_tokens": 35308, "tool_time_ms": 4757, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.0140447999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 323135, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 89, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3334912, "tokens_cache_write": 0, "tokens_input": 123995, "tokens_output": 9205, "tool_calls": 59, "tool_calls_by_class": { "harness_builtin": 59 }, "tool_result_tokens": 41323, "tool_time_ms": 4843, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7901752, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 261859, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 83, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2892288, "tokens_cache_write": 0, "tokens_input": 114495, "tokens_output": 8764, "tool_calls": 53, "tool_calls_by_class": { "harness_builtin": 53 }, "tool_result_tokens": 41077, "tool_time_ms": 2893, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.2116024000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 570896, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 128, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2206976, "tokens_cache_write": 0, "tokens_input": 484363, "tokens_output": 19568, "tool_calls": 108, "tool_calls_by_class": { "harness_builtin": 108 }, "tool_result_tokens": 300844, "tool_time_ms": 8549, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.1519168, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 709191, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 133, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2349312, "tokens_cache_write": 0, "tokens_input": 453103, "tokens_output": 19989, "tool_calls": 113, "tool_calls_by_class": { "harness_builtin": 113 }, "tool_result_tokens": 277519, "tool_time_ms": 100037, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.2465424, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 625742, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 136, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2169856, "tokens_cache_write": 0, "tokens_input": 496775, "tokens_output": 19575, "tool_calls": 116, "tool_calls_by_class": { "harness_builtin": 116 }, "tool_result_tokens": 286303, "tool_time_ms": 63672, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.2039400000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 555060, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 135, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2295040, "tokens_cache_write": 0, "tokens_input": 473986, "tokens_output": 19499, "tool_calls": 115, "tool_calls_by_class": { "harness_builtin": 115 }, "tool_result_tokens": 291946, "tool_time_ms": 14040, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.0772064, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 625054, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 139, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2278656, "tokens_cache_write": 0, "tokens_input": 440741, "tokens_output": 20139, "tool_calls": 119, "tool_calls_by_class": { "harness_builtin": 119 }, "tool_result_tokens": 271185, "tool_time_ms": 9163, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.1459960000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 591584, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 134, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2228480, "tokens_cache_write": 0, "tokens_input": 466646, "tokens_output": 19401, "tool_calls": 114, "tool_calls_by_class": { "harness_builtin": 114 }, "tool_result_tokens": 274905, "tool_time_ms": 53889, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.35278, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 602570, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 139, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2379520, "tokens_cache_write": 0, "tokens_input": 495933, "tokens_output": 20862, "tool_calls": 119, "tool_calls_by_class": { "harness_builtin": 119 }, "tool_result_tokens": 286761, "tool_time_ms": 15579, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.289644, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 649372, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 148, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2750720, "tokens_cache_write": 0, "tokens_input": 440169, "tokens_output": 21434, "tool_calls": 128, "tool_calls_by_class": { "harness_builtin": 128 }, "tool_result_tokens": 279886, "tool_time_ms": 67188, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.3350768, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 616311, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 129, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2296832, "tokens_cache_write": 0, "tokens_input": 511651, "tokens_output": 18487, "tool_calls": 109, "tool_calls_by_class": { "harness_builtin": 109 }, "tool_result_tokens": 307620, "tool_time_ms": 70050, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.5271632000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 589942, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 134, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2425088, "tokens_cache_write": 0, "tokens_input": 538972, "tokens_output": 20062, "tool_calls": 114, "tool_calls_by_class": { "harness_builtin": 114 }, "tool_result_tokens": 320636, "tool_time_ms": 10985, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.0093232, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 620082, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 149, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2094848, "tokens_cache_write": 0, "tokens_input": 701911, "tokens_output": 18187, "tool_calls": 119, "tool_calls_by_class": { "harness_builtin": 119 }, "tool_result_tokens": 328478, "tool_time_ms": 8729, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.7397928, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 646695, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 147, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2166272, "tokens_cache_write": 0, "tokens_input": 622991, "tokens_output": 19066, "tool_calls": 117, "tool_calls_by_class": { "harness_builtin": 117 }, "tool_result_tokens": 337220, "tool_time_ms": 8494, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.8136160000000006, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 638784, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 152, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2272000, "tokens_cache_write": 0, "tokens_input": 630564, "tokens_output": 19128, "tool_calls": 122, "tool_calls_by_class": { "harness_builtin": 122 }, "tool_result_tokens": 323925, "tool_time_ms": 8914, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.773008, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 607591, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 141, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1982720, "tokens_cache_write": 0, "tokens_input": 654695, "tokens_output": 18057, "tool_calls": 111, "tool_calls_by_class": { "harness_builtin": 111 }, "tool_result_tokens": 319907, "tool_time_ms": 8426, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.6314344, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 638000, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 142, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2158336, "tokens_cache_write": 0, "tokens_input": 603320, "tokens_output": 17741, "tool_calls": 112, "tool_calls_by_class": { "harness_builtin": 112 }, "tool_result_tokens": 344672, "tool_time_ms": 8454, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.712092, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 555996, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 130, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1858560, "tokens_cache_write": 0, "tokens_input": 656827, "tokens_output": 17068, "tool_calls": 100, "tool_calls_by_class": { "harness_builtin": 100 }, "tool_result_tokens": 351204, "tool_time_ms": 7305, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.9286984, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 612590, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 138, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1942016, "tokens_cache_write": 0, "tokens_input": 695683, "tokens_output": 18458, "tool_calls": 108, "tool_calls_by_class": { "harness_builtin": 108 }, "tool_result_tokens": 332734, "tool_time_ms": 8495, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.0800712, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 685197, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 143, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2105088, "tokens_cache_write": 0, "tokens_input": 715444, "tokens_output": 18813, "tool_calls": 113, "tool_calls_by_class": { "harness_builtin": 113 }, "tool_result_tokens": 341606, "tool_time_ms": 8461, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 3.8817704, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 564108, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 147, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2253056, "tokens_cache_write": 0, "tokens_input": 650792, "tokens_output": 18869, "tool_calls": 117, "tool_calls_by_class": { "harness_builtin": 117 }, "tool_result_tokens": 343671, "tool_time_ms": 10766, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 4.0440608000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 711361, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 147, "model": "gpt-5.6-sol", "path": "runs/codex/gpt-5.6-sol/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2039552, "tokens_cache_write": 0, "tokens_input": 715335, "tokens_output": 18345, "tool_calls": 117, "tool_calls_by_class": { "harness_builtin": 117 }, "tool_result_tokens": 329706, "tool_time_ms": 8301, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.561636, "denominator": 20, "duration_basis": "run", "duration_ms": 122760, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 31, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1487360, "tokens_cache_write": 0, "tokens_input": 103168, "tokens_output": 4819, "tool_calls": 30, "tool_calls_by_class": { "harness_builtin": 30 }, "tool_result_tokens": 55497, "tool_time_ms": 1667, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.37451840000000003, "denominator": 20, "duration_basis": "run", "duration_ms": 143163, "exam": "HumanExam", "exam_epoch": null, "fails": 9, "family": "harness_bare", "llm_calls": 20, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 786432, "tokens_cache_write": 0, "tokens_input": 72064, "tokens_output": 6092, "tool_calls": 19, "tool_calls_by_class": { "harness_builtin": 19 }, "tool_result_tokens": 45532, "tool_time_ms": 1497, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.508522, "denominator": 20, "duration_basis": "run", "duration_ms": 124276, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 28, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1296640, "tokens_cache_write": 0, "tokens_input": 96271, "tokens_output": 4721, "tool_calls": 27, "tool_calls_by_class": { "harness_builtin": 27 }, "tool_result_tokens": 50908, "tool_time_ms": 1490, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.46621, "denominator": 20, "duration_basis": "run", "duration_ms": 131052, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 30, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1195520, "tokens_cache_write": 0, "tokens_input": 83751, "tokens_output": 4967, "tool_calls": 29, "tool_calls_by_class": { "harness_builtin": 29 }, "tool_result_tokens": 47586, "tool_time_ms": 2072, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3821728, "denominator": 20, "duration_basis": "run", "duration_ms": 127040, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 20, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 830464, "tokens_cache_write": 0, "tokens_input": 78100, "tokens_output": 4990, "tool_calls": 19, "tool_calls_by_class": { "harness_builtin": 19 }, "tool_result_tokens": 54183, "tool_time_ms": 1321, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.42960999999999994, "denominator": 20, "duration_basis": "run", "duration_ms": 144846, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 23, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 922880, "tokens_cache_write": 0, "tokens_input": 87447, "tokens_output": 5845, "tool_calls": 22, "tool_calls_by_class": { "harness_builtin": 22 }, "tool_result_tokens": 59509, "tool_time_ms": 11606, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.43517079999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 114973, "exam": "HumanExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 21, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 784384, "tokens_cache_write": 0, "tokens_input": 109255, "tokens_output": 4982, "tool_calls": 20, "tool_calls_by_class": { "harness_builtin": 20 }, "tool_result_tokens": 54109, "tool_time_ms": 1121, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.5330539999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 110469, "exam": "HumanExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 32, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1382400, "tokens_cache_write": 0, "tokens_input": 100867, "tokens_output": 4570, "tool_calls": 31, "tool_calls_by_class": { "harness_builtin": 31 }, "tool_result_tokens": 54274, "tool_time_ms": 2299, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.4794404, "denominator": 20, "duration_basis": "run", "duration_ms": 124782, "exam": "HumanExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 34, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 1270272, "tokens_cache_write": 0, "tokens_input": 84547, "tokens_output": 4691, "tool_calls": 33, "tool_calls_by_class": { "harness_builtin": 33 }, "tool_result_tokens": 46939, "tool_time_ms": 2427, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3325108, "denominator": 20, "duration_basis": "run", "duration_ms": 100376, "exam": "HumanExam", "exam_epoch": null, "fails": 9, "family": "harness_bare", "llm_calls": 17, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 670464, "tokens_cache_write": 0, "tokens_input": 71303, "tokens_output": 4651, "tool_calls": 16, "tool_calls_by_class": { "harness_builtin": 16 }, "tool_result_tokens": 45448, "tool_time_ms": 1091, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3080276, "denominator": 30, "duration_basis": "run", "duration_ms": 101127, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 17, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 625408, "tokens_cache_write": 0, "tokens_input": 63267, "tokens_output": 4701, "tool_calls": 16, "tool_calls_by_class": { "harness_builtin": 16 }, "tool_result_tokens": 43770, "tool_time_ms": 1021, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3540612, "denominator": 30, "duration_basis": "run", "duration_ms": 100093, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 21, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 746496, "tokens_cache_write": 0, "tokens_input": 77037, "tokens_output": 4224, "tool_calls": 20, "tool_calls_by_class": { "harness_builtin": 20 }, "tool_result_tokens": 45697, "tool_time_ms": 2375, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3960224, "denominator": 30, "duration_basis": "run", "duration_ms": 131471, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 17, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 703232, "tokens_cache_write": 0, "tokens_input": 90290, "tokens_output": 6233, "tool_calls": 16, "tool_calls_by_class": { "harness_builtin": 16 }, "tool_result_tokens": 55369, "tool_time_ms": 869, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.35003, "denominator": 30, "duration_basis": "run", "duration_ms": 105118, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 15, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 656640, "tokens_cache_write": 0, "tokens_input": 79513, "tokens_output": 4973, "tool_calls": 14, "tool_calls_by_class": { "harness_builtin": 14 }, "tool_result_tokens": 58244, "tool_time_ms": 885, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3959144, "denominator": 30, "duration_basis": "run", "duration_ms": 115474, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 18, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 764672, "tokens_cache_write": 0, "tokens_input": 89396, "tokens_output": 5349, "tool_calls": 17, "tool_calls_by_class": { "harness_builtin": 17 }, "tool_result_tokens": 58411, "tool_time_ms": 1088, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.306002, "denominator": 30, "duration_basis": "run", "duration_ms": 95903, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 11, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 482560, "tokens_cache_write": 0, "tokens_input": 76533, "tokens_output": 4702, "tool_calls": 10, "tool_calls_by_class": { "harness_builtin": 10 }, "tool_result_tokens": 58828, "tool_time_ms": 673, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.32866439999999997, "denominator": 30, "duration_basis": "run", "duration_ms": 104386, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 17, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 667392, "tokens_cache_write": 0, "tokens_input": 69399, "tokens_output": 4699, "tool_calls": 16, "tool_calls_by_class": { "harness_builtin": 16 }, "tool_result_tokens": 51210, "tool_time_ms": 879, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3372872, "denominator": 30, "duration_basis": "run", "duration_ms": 130202, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 15, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 563456, "tokens_cache_write": 0, "tokens_input": 74942, "tokens_output": 6226, "tool_calls": 14, "tool_calls_by_class": { "harness_builtin": 14 }, "tool_result_tokens": 56692, "tool_time_ms": 896, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.3676084, "denominator": 30, "duration_basis": "run", "duration_ms": 114707, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 18, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 748032, "tokens_cache_write": 0, "tokens_input": 80981, "tokens_output": 4670, "tool_calls": 17, "tool_calls_by_class": { "harness_builtin": 17 }, "tool_result_tokens": 58456, "tool_time_ms": 1281, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 0.36917920000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 112493, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 17, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 755456, "tokens_cache_write": 0, "tokens_input": 78036, "tokens_output": 5168, "tool_calls": 16, "tool_calls_by_class": { "harness_builtin": 16 }, "tool_result_tokens": 57719, "tool_time_ms": 925, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.1945164000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 350156, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 74, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7967232, "tokens_cache_write": 0, "tokens_input": 212443, "tokens_output": 14682, "tool_calls": 54, "tool_calls_by_class": { "harness_builtin": 54 }, "tool_result_tokens": 133704, "tool_time_ms": 4913, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.0036392000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 368772, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7334656, "tokens_cache_write": 0, "tokens_input": 187162, "tokens_output": 13532, "tool_calls": 61, "tool_calls_by_class": { "harness_builtin": 61 }, "tool_result_tokens": 112099, "tool_time_ms": 4742, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2024352, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 428864, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 82, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 7973376, "tokens_cache_write": 0, "tokens_input": 212266, "tokens_output": 15269, "tool_calls": 62, "tool_calls_by_class": { "harness_builtin": 62 }, "tool_result_tokens": 130808, "tool_time_ms": 14240, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.3073232, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 384732, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 85, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 8440576, "tokens_cache_write": 0, "tokens_input": 219964, "tokens_output": 14940, "tool_calls": 65, "tool_calls_by_class": { "harness_builtin": 65 }, "tool_result_tokens": 136513, "tool_time_ms": 4759, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2690488, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 365609, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 82, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 8292864, "tokens_cache_write": 0, "tokens_input": 214242, "tokens_output": 15166, "tool_calls": 62, "tool_calls_by_class": { "harness_builtin": 62 }, "tool_result_tokens": 129532, "tool_time_ms": 4774, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6464284000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 475076, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 90, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 9810432, "tokens_cache_write": 0, "tokens_input": 229533, "tokens_output": 18773, "tool_calls": 70, "tool_calls_by_class": { "harness_builtin": 70 }, "tool_result_tokens": 139465, "tool_time_ms": 4985, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2393652, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 482243, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 85, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 8225536, "tokens_cache_write": 0, "tokens_input": 206037, "tokens_output": 15182, "tool_calls": 65, "tool_calls_by_class": { "harness_builtin": 65 }, "tool_result_tokens": 124621, "tool_time_ms": 9437, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7885096000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 510921, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 89, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 10363648, "tokens_cache_write": 0, "tokens_input": 242282, "tokens_output": 19268, "tool_calls": 69, "tool_calls_by_class": { "harness_builtin": 69 }, "tool_result_tokens": 154017, "tool_time_ms": 7089, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.636252, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 483655, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 9, "family": "harness_bare", "llm_calls": 93, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 9666560, "tokens_cache_write": 0, "tokens_input": 241508, "tokens_output": 18327, "tool_calls": 72, "tool_calls_by_class": { "harness_builtin": 72 }, "tool_result_tokens": 154521, "tool_time_ms": 6974, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8762956000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 375392, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6436608, "tokens_cache_write": 0, "tokens_input": 207061, "tokens_output": 14571, "tool_calls": 61, "tool_calls_by_class": { "harness_builtin": 61 }, "tool_result_tokens": 105514, "tool_time_ms": 4865, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8819032000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 428202, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 103, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6553856, "tokens_cache_write": 0, "tokens_input": 198338, "tokens_output": 14538, "tool_calls": 73, "tool_calls_by_class": { "harness_builtin": 73 }, "tool_result_tokens": 87669, "tool_time_ms": 4514, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.9077424, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 462763, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 95, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6496512, "tokens_cache_write": 0, "tokens_input": 213692, "tokens_output": 15088, "tool_calls": 65, "tool_calls_by_class": { "harness_builtin": 65 }, "tool_result_tokens": 95786, "tool_time_ms": 5953, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.239728, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 323500, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 85, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4125440, "tokens_cache_write": 0, "tokens_input": 138962, "tokens_output": 11393, "tool_calls": 55, "tool_calls_by_class": { "harness_builtin": 55 }, "tool_result_tokens": 68203, "tool_time_ms": 2987, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.3036396, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 371122, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 95, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4227328, "tokens_cache_write": 0, "tokens_input": 156553, "tokens_output": 12089, "tool_calls": 65, "tool_calls_by_class": { "harness_builtin": 65 }, "tool_result_tokens": 58637, "tool_time_ms": 5167, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8789099999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 436238, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 106, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6557440, "tokens_cache_write": 0, "tokens_input": 187903, "tokens_output": 15968, "tool_calls": 76, "tool_calls_by_class": { "harness_builtin": 76 }, "tool_result_tokens": 85964, "tool_time_ms": 5277, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.2220904, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 452619, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 105, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 8142592, "tokens_cache_write": 0, "tokens_input": 190868, "tokens_output": 17653, "tool_calls": 75, "tool_calls_by_class": { "harness_builtin": 75 }, "tool_result_tokens": 102399, "tool_time_ms": 3807, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.452072, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 339340, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 102, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 5035520, "tokens_cache_write": 0, "tokens_input": 145198, "tokens_output": 12881, "tool_calls": 72, "tool_calls_by_class": { "harness_builtin": 72 }, "tool_result_tokens": 65064, "tool_time_ms": 4364, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.805126, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 374357, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 93, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6327040, "tokens_cache_write": 0, "tokens_input": 183177, "tokens_output": 14447, "tool_calls": 63, "tool_calls_by_class": { "harness_builtin": 63 }, "tool_result_tokens": 103273, "tool_time_ms": 3335, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.7615368, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 383129, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 102, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6285824, "tokens_cache_write": 0, "tokens_input": 168600, "tokens_output": 13931, "tool_calls": 72, "tool_calls_by_class": { "harness_builtin": 72 }, "tool_result_tokens": 82261, "tool_time_ms": 5075, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 1.8879236000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 417216, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 96, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 6622208, "tokens_cache_write": 0, "tokens_input": 191795, "tokens_output": 14991, "tool_calls": 66, "tool_calls_by_class": { "harness_builtin": 66 }, "tool_result_tokens": 107675, "tool_time_ms": 3960, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5052128000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 958610, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 179, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3967744, "tokens_cache_write": 0, "tokens_input": 638206, "tokens_output": 36271, "tool_calls": 159, "tool_calls_by_class": { "harness_builtin": 159 }, "tool_result_tokens": 401263, "tool_time_ms": 13468, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6785512000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1073244, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 187, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3828736, "tokens_cache_write": 0, "tokens_input": 733928, "tokens_output": 37079, "tool_calls": 167, "tool_calls_by_class": { "harness_builtin": 167 }, "tool_result_tokens": 412015, "tool_time_ms": 58406, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8659711999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1086321, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 199, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4613376, "tokens_cache_write": 0, "tokens_input": 735968, "tokens_output": 39280, "tool_calls": 179, "tool_calls_by_class": { "harness_builtin": 179 }, "tool_result_tokens": 427223, "tool_time_ms": 56458, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8003396, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 940613, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 177, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3954688, "tokens_cache_write": 0, "tokens_input": 783679, "tokens_output": 36837, "tool_calls": 157, "tool_calls_by_class": { "harness_builtin": 157 }, "tool_result_tokens": 460871, "tool_time_ms": 14015, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7952544, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 954333, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 183, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4046592, "tokens_cache_write": 0, "tokens_input": 778798, "tokens_output": 35695, "tool_calls": 163, "tool_calls_by_class": { "harness_builtin": 163 }, "tool_result_tokens": 446258, "tool_time_ms": 12384, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.8084256, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1153696, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 191, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4273408, "tokens_cache_write": 0, "tokens_input": 738744, "tokens_output": 39688, "tool_calls": 171, "tool_calls_by_class": { "harness_builtin": 171 }, "tool_result_tokens": 428821, "tool_time_ms": 99704, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.572102, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 960779, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 181, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3942400, "tokens_cache_write": 0, "tokens_input": 676609, "tokens_output": 35867, "tool_calls": 160, "tool_calls_by_class": { "harness_builtin": 160 }, "tool_result_tokens": 422429, "tool_time_ms": 14948, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6389476, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 949077, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 185, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4269568, "tokens_cache_write": 0, "tokens_input": 674285, "tokens_output": 36372, "tool_calls": 165, "tool_calls_by_class": { "harness_builtin": 165 }, "tool_result_tokens": 437606, "tool_time_ms": 36958, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5827088, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1071737, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 183, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4057344, "tokens_cache_write": 0, "tokens_input": 665114, "tokens_output": 36751, "tool_calls": 163, "tool_calls_by_class": { "harness_builtin": 163 }, "tool_result_tokens": 428427, "tool_time_ms": 81431, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos-hard", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6129732, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 908800, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 179, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 4091136, "tokens_cache_write": 0, "tokens_input": 679837, "tokens_output": 36256, "tool_calls": 159, "tool_calls_by_class": { "harness_builtin": 159 }, "tool_result_tokens": 434367, "tool_time_ms": 15635, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7034324, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 916551, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 175, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3640832, "tokens_cache_write": 0, "tokens_input": 798573, "tokens_output": 31510, "tool_calls": 145, "tool_calls_by_class": { "harness_builtin": 145 }, "tool_result_tokens": 479557, "tool_time_ms": 10761, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5591720000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 894269, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 169, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3092480, "tokens_cache_write": 0, "tokens_input": 787182, "tokens_output": 30526, "tool_calls": 139, "tool_calls_by_class": { "harness_builtin": 139 }, "tool_result_tokens": 435771, "tool_time_ms": 12775, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7759356, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 941037, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 177, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3455488, "tokens_cache_write": 0, "tokens_input": 857013, "tokens_output": 30901, "tool_calls": 147, "tool_calls_by_class": { "harness_builtin": 147 }, "tool_result_tokens": 475104, "tool_time_ms": 10171, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.7803692, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 895271, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 177, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3503616, "tokens_cache_write": 0, "tokens_input": 846725, "tokens_output": 32183, "tool_calls": 147, "tool_calls_by_class": { "harness_builtin": 147 }, "tool_result_tokens": 496153, "tool_time_ms": 11377, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.5453128, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 855826, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 174, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3366144, "tokens_cache_write": 0, "tokens_input": 743076, "tokens_output": 32161, "tool_calls": 144, "tool_calls_by_class": { "harness_builtin": 144 }, "tool_result_tokens": 466414, "tool_time_ms": 10930, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6460076, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 928461, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 177, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3365888, "tokens_cache_write": 0, "tokens_input": 796461, "tokens_output": 31659, "tool_calls": 147, "tool_calls_by_class": { "harness_builtin": 147 }, "tool_result_tokens": 443264, "tool_time_ms": 11662, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.4654103999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 842468, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 167, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 3050752, "tokens_cache_write": 0, "tokens_input": 747402, "tokens_output": 30038, "tool_calls": 137, "tool_calls_by_class": { "harness_builtin": 137 }, "tool_result_tokens": 444835, "tool_time_ms": 10203, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.6331952000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 865191, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 162, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2926336, "tokens_cache_write": 0, "tokens_input": 836608, "tokens_output": 31226, "tool_calls": 132, "tool_calls_by_class": { "harness_builtin": 132 }, "tool_result_tokens": 454392, "tool_time_ms": 9994, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.4856172, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 896526, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 166, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2900736, "tokens_cache_write": 0, "tokens_input": 765157, "tokens_output": 31263, "tool_calls": 136, "tool_calls_by_class": { "harness_builtin": 136 }, "tool_result_tokens": 395777, "tool_time_ms": 9291, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "codex", "arm_label": "Codex Bare", "battery": "memos", "cost_basis": "estimated:v6-claude-sonnet-5-5-0929", "cost_reported_usd": null, "cost_usd": 2.403008, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 894489, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 161, "model": "gpt-5.6-terra", "path": "runs/codex/gpt-5.6-terra/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|codex|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "codex_reported_derived", "tokens_cache_read": 2842880, "tokens_cache_write": 0, "tokens_input": 747428, "tokens_output": 28298, "tool_calls": 131, "tool_calls_by_class": { "harness_builtin": 131 }, "tool_result_tokens": 436132, "tool_time_ms": 9562, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5371565, "cost_usd": 5.5371565, "denominator": 20, "duration_basis": "run", "duration_ms": 307819, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 77, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2575309, "tokens_cache_write": 250862, "tokens_input": 1901, "tokens_output": 25156, "tool_calls": 123, "tool_calls_by_class": { "cc_builtin": 118, "subagent": 5 }, "tool_result_tokens": 82094, "tool_time_ms": 23409, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.440919750000001, "cost_usd": 6.440919750000001, "denominator": 20, "duration_basis": "run", "duration_ms": 271616, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 92, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2538781, "tokens_cache_write": 303553, "tokens_input": 1847, "tokens_output": 28102, "tool_calls": 170, "tool_calls_by_class": { "cc_builtin": 158, "subagent": 12 }, "tool_result_tokens": 113154, "tool_time_ms": 25381, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.462310999999997, "cost_usd": 6.462310999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 236303, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 69, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1804053, "tokens_cache_write": 171130, "tokens_input": 136, "tokens_output": 22313, "tool_calls": 99, "tool_calls_by_class": { "cc_builtin": 90, "subagent": 9 }, "tool_result_tokens": 54001, "tool_time_ms": 17818, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.505119750000002, "cost_usd": 4.505119750000002, "denominator": 20, "duration_basis": "run", "duration_ms": 228735, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 72, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1673918, "tokens_cache_write": 199868, "tokens_input": 143, "tokens_output": 31755, "tool_calls": 123, "tool_calls_by_class": { "cc_builtin": 114, "subagent": 9 }, "tool_result_tokens": 68512, "tool_time_ms": 21261, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5787572500000002, "cost_usd": 3.5787572500000002, "denominator": 20, "duration_basis": "run", "duration_ms": 235447, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 69, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1590611, "tokens_cache_write": 200534, "tokens_input": 1246, "tokens_output": 17543, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 100, "subagent": 9 }, "tool_result_tokens": 63154, "tool_time_ms": 10043, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.7147235, "cost_usd": 7.7147235, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 462896, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 80, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4992997, "tokens_cache_write": 95722, "tokens_input": 152, "tokens_output": 25242, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 19244, "tool_time_ms": 9514, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.901683, "cost_usd": 7.901683, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 463821, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 77, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5173443, "tokens_cache_write": 71014, "tokens_input": 146, "tokens_output": 26130, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 22293, "tool_time_ms": 8387, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.628003999999999, "cost_usd": 8.628003999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 480944, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 83, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5763384, "tokens_cache_write": 77034, "tokens_input": 269, "tokens_output": 26425, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 73 }, "tool_result_tokens": 27391, "tool_time_ms": 12196, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.334071, "cost_usd": 8.334071, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 437415, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 73, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5435551, "tokens_cache_write": 82271, "tokens_input": 250, "tokens_output": 25012, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 31230, "tool_time_ms": 9381, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.211282999999998, "cost_usd": 9.211282999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 542561, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 91, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6363483, "tokens_cache_write": 73146, "tokens_input": 173, "tokens_output": 27663, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 74 }, "tool_result_tokens": 24243, "tool_time_ms": 35664, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.427755, "cost_usd": 5.427755, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 349906, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 74, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3559945, "tokens_cache_write": 50210, "tokens_input": 141, "tokens_output": 17244, "tool_calls": 44, "tool_calls_by_class": { "cc_builtin": 44 }, "tool_result_tokens": 12931, "tool_time_ms": 3829, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.075831000000002, "cost_usd": 6.075831000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 361181, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 82, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4190051, "tokens_cache_write": 48056, "tokens_input": 156, "tokens_output": 18462, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 13856, "tool_time_ms": 4657, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.4119980000000005, "cost_usd": 6.4119980000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 406135, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4334358, "tokens_cache_write": 53742, "tokens_input": 160, "tokens_output": 20024, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 16586, "tool_time_ms": 3615, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.024354999999999, "cost_usd": 6.024354999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 346878, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4201055, "tokens_cache_write": 47045, "tokens_input": 160, "tokens_output": 17616, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 13898, "tool_time_ms": 3244, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.104220999999999, "cost_usd": 6.104220999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 370565, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4213681, "tokens_cache_write": 50162, "tokens_input": 160, "tokens_output": 17714, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 15780, "tool_time_ms": 3600, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.671331499999999, "cost_usd": 9.671331499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 872544, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 133, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4415940, "tokens_cache_write": 181931, "tokens_input": 373, "tokens_output": 45217, "tool_calls": 127, "tool_calls_by_class": { "cc_builtin": 127 }, "tool_result_tokens": 55700, "tool_time_ms": 41911, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.65490725, "cost_usd": 8.65490725, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 750961, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 128, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4106292, "tokens_cache_write": 187409, "tokens_input": 2014, "tokens_output": 41368, "tool_calls": 130, "tool_calls_by_class": { "cc_builtin": 128, "subagent": 2 }, "tool_result_tokens": 68648, "tool_time_ms": 46540, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.1505115, "cost_usd": 9.1505115, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 827452, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 129, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4141312, "tokens_cache_write": 211093, "tokens_input": 252, "tokens_output": 41547, "tool_calls": 136, "tool_calls_by_class": { "cc_builtin": 135, "subagent": 1 }, "tool_result_tokens": 76304, "tool_time_ms": 12060, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.2098175, "cost_usd": 8.2098175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 717548, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 128, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4017024, "tokens_cache_write": 155975, "tokens_input": 2127, "tokens_output": 39486, "tool_calls": 122, "tool_calls_by_class": { "cc_builtin": 120, "subagent": 2 }, "tool_result_tokens": 52947, "tool_time_ms": 40092, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.7160795, "cost_usd": 8.7160795, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 737593, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 135, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4350994, "tokens_cache_write": 158430, "tokens_input": 376, "tokens_output": 43113, "tool_calls": 138, "tool_calls_by_class": { "cc_builtin": 137, "subagent": 1 }, "tool_result_tokens": 58194, "tool_time_ms": 15199, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.440538, "cost_usd": 1.440538, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 81262, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 24, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 708888, "tokens_cache_write": 27541, "tokens_input": 48, "tokens_output": 3607, "tool_calls": 17, "tool_calls_by_class": { "cc_builtin": 17 }, "tool_result_tokens": 2716, "tool_time_ms": 1151, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.822237, "cost_usd": 1.822237, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 74385, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 22, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 622907, "tokens_cache_write": 51362, "tokens_input": 44, "tokens_output": 3433, "tool_calls": 15, "tool_calls_by_class": { "cc_builtin": 15 }, "tool_result_tokens": 2329, "tool_time_ms": 1153, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.251615, "cost_usd": 1.251615, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 72521, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 21, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 621645, "tokens_cache_write": 24050, "tokens_input": 42, "tokens_output": 2971, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 14 }, "tool_result_tokens": 4436, "tool_time_ms": 977, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.1681, "cost_usd": 1.1681, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 67924, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 21, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 624380, "tokens_cache_write": 18900, "tokens_input": 42, "tokens_output": 3306, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 14 }, "tool_result_tokens": 2528, "tool_time_ms": 921, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.046006, "cost_usd": 1.046006, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 80900, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 23, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 698206, "tokens_cache_write": 8667, "tokens_input": 46, "tokens_output": 3480, "tool_calls": 16, "tool_calls_by_class": { "cc_builtin": 16 }, "tool_result_tokens": 3894, "tool_time_ms": 1144, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.862456500000002, "cost_usd": 9.862456500000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 746331, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 145, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4626141, "tokens_cache_write": 226992, "tokens_input": 1721, "tokens_output": 34773, "tool_calls": 131, "tool_calls_by_class": { "cc_builtin": 130, "subagent": 1 }, "tool_result_tokens": 69369, "tool_time_ms": 7441, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.30789, "cost_usd": 9.30789, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 694914, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 141, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4422030, "tokens_cache_write": 155558, "tokens_input": 280, "tokens_output": 35438, "tool_calls": 124, "tool_calls_by_class": { "cc_builtin": 124 }, "tool_result_tokens": 46379, "tool_time_ms": 7206, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.103171000000001, "cost_usd": 9.103171000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 692796, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 139, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4347061, "tokens_cache_write": 150026, "tokens_input": 389, "tokens_output": 35034, "tool_calls": 118, "tool_calls_by_class": { "cc_builtin": 118 }, "tool_result_tokens": 43041, "tool_time_ms": 6830, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.329292250000002, "cost_usd": 8.329292250000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 688707, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 133, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4088483, "tokens_cache_write": 144281, "tokens_input": 696, "tokens_output": 32176, "tool_calls": 114, "tool_calls_by_class": { "cc_builtin": 113, "subagent": 1 }, "tool_result_tokens": 44150, "tool_time_ms": 7628, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.304273, "cost_usd": 8.304273, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 629648, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 136, "model": "claude-fable-5", "path": "runs/cold/claude-fable-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4238206, "tokens_cache_write": 130261, "tokens_input": 1570, "tokens_output": 33315, "tool_calls": 117, "tool_calls_by_class": { "cc_builtin": 117 }, "tool_result_tokens": 44969, "tool_time_ms": 6817, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.7737064, "cost_usd": 5.7737064, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 962420, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 288, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 29608564, "tokens_cache_write": 1316115, "tokens_input": 1730, "tokens_output": 35778, "tool_calls": 268, "tool_calls_by_class": { "cc_builtin": 268 }, "tool_result_tokens": 68925, "tool_time_ms": 14200, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.753349300000001, "cost_usd": 4.753349300000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 822406, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 255, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 24298203, "tokens_cache_write": 1083168, "tokens_input": 1543, "tokens_output": 31130, "tool_calls": 235, "tool_calls_by_class": { "cc_builtin": 235 }, "tool_result_tokens": 65482, "tool_time_ms": 14720, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.485702749999999, "cost_usd": 5.485702749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 858567, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 272, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 27178130, "tokens_cache_write": 1296407, "tokens_input": 1675, "tokens_output": 38036, "tool_calls": 293, "tool_calls_by_class": { "cc_builtin": 291, "subagent": 2 }, "tool_result_tokens": 95682, "tool_time_ms": 22030, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.301656, "cost_usd": 5.301656, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 798563, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 263, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 25306305, "tokens_cache_write": 1310210, "tokens_input": 1595, "tokens_output": 33972, "tool_calls": 252, "tool_calls_by_class": { "cc_builtin": 251, "subagent": 1 }, "tool_result_tokens": 102719, "tool_time_ms": 31712, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4488870499999997, "cost_usd": 2.4488870499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 116058, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 388, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10281758, "tokens_cache_write": 665589, "tokens_input": 9740, "tokens_output": 37025, "tool_calls": 527, "tool_calls_by_class": { "cc_builtin": 507, "subagent": 20 }, "tool_result_tokens": 285542, "tool_time_ms": 34696, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.677432399999999, "cost_usd": 5.677432399999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 526077, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 9, "family": "cc_bare", "llm_calls": 314, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 22651161, "tokens_cache_write": 1618185, "tokens_input": 2232, "tokens_output": 37580, "tool_calls": 435, "tool_calls_by_class": { "cc_builtin": 417, "subagent": 18 }, "tool_result_tokens": 309725, "tool_time_ms": 25116, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.470971649999999, "cost_usd": 4.470971649999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 704549, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 237, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 22647889, "tokens_cache_write": 1037871, "tokens_input": 1445, "tokens_output": 29915, "tool_calls": 228, "tool_calls_by_class": { "cc_builtin": 227, "subagent": 1 }, "tool_result_tokens": 93852, "tool_time_ms": 19921, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.138122400000001, "cost_usd": 3.138122400000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1334216, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 372, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14081464, "tokens_cache_write": 832513, "tokens_input": 3320, "tokens_output": 35591, "tool_calls": 580, "tool_calls_by_class": { "cc_builtin": 534, "subagent": 46 }, "tool_result_tokens": 368597, "tool_time_ms": 38223, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4833625, "cost_usd": 2.4833625, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 524457, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 178, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13621375, "tokens_cache_write": 505010, "tokens_input": 1130, "tokens_output": 22015, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 148 }, "tool_result_tokens": 51648, "tool_time_ms": 5516, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5546032999999997, "cost_usd": 2.5546032999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 486229, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 154, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12334393, "tokens_cache_write": 605814, "tokens_input": 1001, "tokens_output": 21707, "tool_calls": 124, "tool_calls_by_class": { "cc_builtin": 124 }, "tool_result_tokens": 61470, "tool_time_ms": 5020, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6748664000000004, "cost_usd": 2.6748664000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 530550, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 176, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14867264, "tokens_cache_write": 538299, "tokens_input": 1142, "tokens_output": 22080, "tool_calls": 146, "tool_calls_by_class": { "cc_builtin": 145, "harness_builtin": 1 }, "tool_result_tokens": 63766, "tool_time_ms": 5463, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1325018, "cost_usd": 3.1325018, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 617414, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 194, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17627248, "tokens_cache_write": 618094, "tokens_input": 1239, "tokens_output": 26470, "tool_calls": 164, "tool_calls_by_class": { "cc_builtin": 164 }, "tool_result_tokens": 67975, "tool_time_ms": 6502, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.890870500000001, "cost_usd": 2.890870500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 578476, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 177, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15311725, "tokens_cache_write": 615172, "tokens_input": 1144, "tokens_output": 25642, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 148 }, "tool_result_tokens": 60920, "tool_time_ms": 5125, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1883483999999997, "cost_usd": 2.1883483999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 430841, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 153, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11814614, "tokens_cache_write": 458352, "tokens_input": 1013, "tokens_output": 17834, "tool_calls": 123, "tool_calls_by_class": { "cc_builtin": 122, "harness_builtin": 1 }, "tool_result_tokens": 52611, "tool_time_ms": 4487, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6874731000000005, "cost_usd": 2.6874731000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 520099, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 171, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14619346, "tokens_cache_write": 556268, "tokens_input": 1122, "tokens_output": 25216, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 138, "harness_builtin": 1, "subagent": 1 }, "tool_result_tokens": 67932, "tool_time_ms": 4850, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2746364499999996, "cost_usd": 3.2746364499999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 492994, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 231, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17369687, "tokens_cache_write": 694525, "tokens_input": 1595, "tokens_output": 46303, "tool_calls": 253, "tool_calls_by_class": { "cc_builtin": 243, "subagent": 10 }, "tool_result_tokens": 194948, "tool_time_ms": 9035, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.0574919000000005, "cost_usd": 4.0574919000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 718329, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 12, "family": "cc_bare", "llm_calls": 513, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 8, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17380044, "tokens_cache_write": 1143068, "tokens_input": 3245, "tokens_output": 58537, "tool_calls": 658, "tool_calls_by_class": { "cc_builtin": 641, "subagent": 17 }, "tool_result_tokens": 462741, "tool_time_ms": 30014, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9980436500000005, "cost_usd": 3.9980436500000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 633305, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "cc_bare", "llm_calls": 472, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16122429, "tokens_cache_write": 1154177, "tokens_input": 3021, "tokens_output": 56687, "tool_calls": 631, "tool_calls_by_class": { "cc_builtin": 615, "subagent": 16 }, "tool_result_tokens": 438005, "tool_time_ms": 33941, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.137965749999999, "cost_usd": 4.137965749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 864055, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_bare", "llm_calls": 486, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16678955, "tokens_cache_write": 1216828, "tokens_input": 4461, "tokens_output": 51345, "tool_calls": 623, "tool_calls_by_class": { "cc_builtin": 602, "subagent": 21 }, "tool_result_tokens": 476296, "tool_time_ms": 53821, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.0033163499999995, "cost_usd": 4.0033163499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 680998, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_bare", "llm_calls": 479, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16086476, "tokens_cache_write": 1174492, "tokens_input": 3105, "tokens_output": 55980, "tool_calls": 681, "tool_calls_by_class": { "cc_builtin": 662, "subagent": 19 }, "tool_result_tokens": 456896, "tool_time_ms": 25718, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4917627000000007, "cost_usd": 3.4917627000000007, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 757733, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 10, "family": "cc_bare", "llm_calls": 437, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14815927, "tokens_cache_write": 952139, "tokens_input": 2780, "tokens_output": 52794, "tool_calls": 569, "tool_calls_by_class": { "cc_builtin": 555, "subagent": 14 }, "tool_result_tokens": 370329, "tool_time_ms": 36668, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.20394385, "cost_usd": 4.20394385, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 764861, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "cc_bare", "llm_calls": 515, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17576321, "tokens_cache_write": 1168380, "tokens_input": 3318, "tokens_output": 68002, "tool_calls": 743, "tool_calls_by_class": { "cc_builtin": 727, "subagent": 16 }, "tool_result_tokens": 459804, "tool_time_ms": 41625, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.4662296999999995, "cost_usd": 4.4662296999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 664631, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_bare", "llm_calls": 536, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17874262, "tokens_cache_write": 1293228, "tokens_input": 6948, "tokens_output": 63022, "tool_calls": 762, "tool_calls_by_class": { "cc_builtin": 743, "subagent": 19 }, "tool_result_tokens": 499589, "tool_time_ms": 45193, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.82414195, "cost_usd": 3.82414195, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 765816, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_bare", "llm_calls": 508, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16440932, "tokens_cache_write": 1037037, "tokens_input": 3273, "tokens_output": 61503, "tool_calls": 735, "tool_calls_by_class": { "cc_builtin": 716, "subagent": 19 }, "tool_result_tokens": 410437, "tool_time_ms": 50391, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.35893845, "cost_usd": 3.35893845, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 758669, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_bare", "llm_calls": 438, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14576532, "tokens_cache_write": 928055, "tokens_input": 2742, "tokens_output": 50975, "tool_calls": 527, "tool_calls_by_class": { "cc_builtin": 513, "subagent": 14 }, "tool_result_tokens": 357341, "tool_time_ms": 29758, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.985468550000001, "cost_usd": 3.985468550000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 510645, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 11, "family": "cc_bare", "llm_calls": 507, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 9, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17217233, "tokens_cache_write": 1062591, "tokens_input": 3467, "tokens_output": 61830, "tool_calls": 719, "tool_calls_by_class": { "cc_builtin": 704, "subagent": 15 }, "tool_result_tokens": 396439, "tool_time_ms": 37367, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9530856000000005, "cost_usd": 3.9530856000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 513880, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 450, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15463331, "tokens_cache_write": 1189771, "tokens_input": 2946, "tokens_output": 50284, "tool_calls": 547, "tool_calls_by_class": { "cc_builtin": 527, "subagent": 20 }, "tool_result_tokens": 471367, "tool_time_ms": 14340, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3749116500000005, "cost_usd": 3.3749116500000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 529550, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 389, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12001869, "tokens_cache_write": 1050434, "tokens_input": 2608, "tokens_output": 46774, "tool_calls": 510, "tool_calls_by_class": { "cc_builtin": 493, "subagent": 17 }, "tool_result_tokens": 391642, "tool_time_ms": 14245, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1665123000000004, "cost_usd": 3.1665123000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 622307, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 373, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11841873, "tokens_cache_write": 950944, "tokens_input": 3173, "tokens_output": 47609, "tool_calls": 450, "tool_calls_by_class": { "cc_builtin": 435, "subagent": 15 }, "tool_result_tokens": 380582, "tool_time_ms": 14508, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9653593999999996, "cost_usd": 2.9653593999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 784657, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 370, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10784659, "tokens_cache_write": 921023, "tokens_input": 3205, "tokens_output": 49095, "tool_calls": 455, "tool_calls_by_class": { "cc_builtin": 434, "subagent": 21 }, "tool_result_tokens": 377267, "tool_time_ms": 13304, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4485754500000003, "cost_usd": 3.4485754500000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 601638, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_bare", "llm_calls": 423, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 23, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12609957, "tokens_cache_write": 1055677, "tokens_input": 2778, "tokens_output": 50981, "tool_calls": 504, "tool_calls_by_class": { "cc_builtin": 481, "subagent": 23 }, "tool_result_tokens": 429019, "tool_time_ms": 15214, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.190535, "cost_usd": 3.190535, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 693535, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 372, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12165020, "tokens_cache_write": 968444, "tokens_input": 2608, "tokens_output": 48643, "tool_calls": 464, "tool_calls_by_class": { "cc_builtin": 444, "subagent": 20 }, "tool_result_tokens": 439793, "tool_time_ms": 13752, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7291737999999994, "cost_usd": 3.7291737999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 543494, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "cc_bare", "llm_calls": 438, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 22, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13686753, "tokens_cache_write": 1192115, "tokens_input": 2901, "tokens_output": 57189, "tool_calls": 542, "tool_calls_by_class": { "cc_builtin": 520, "subagent": 22 }, "tool_result_tokens": 532241, "tool_time_ms": 16830, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9224471999999997, "cost_usd": 3.9224471999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 751202, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 485, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14889052, "tokens_cache_write": 1199173, "tokens_input": 3173, "tokens_output": 59584, "tool_calls": 631, "tool_calls_by_class": { "cc_builtin": 600, "subagent": 31 }, "tool_result_tokens": 502836, "tool_time_ms": 16208, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6539717999999994, "cost_usd": 3.6539717999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 579285, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_bare", "llm_calls": 426, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 24, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12715783, "tokens_cache_write": 1185968, "tokens_input": 2854, "tokens_output": 54307, "tool_calls": 537, "tool_calls_by_class": { "cc_builtin": 512, "subagent": 25 }, "tool_result_tokens": 490262, "tool_time_ms": 15190, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5693193500000002, "cost_usd": 3.5693193500000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 472487, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 414, "model": "claude-haiku-4-5", "path": "runs/cold/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12967561, "tokens_cache_write": 1128483, "tokens_input": 2705, "tokens_output": 52836, "tool_calls": 495, "tool_calls_by_class": { "cc_builtin": 471, "subagent": 24 }, "tool_result_tokens": 535572, "tool_time_ms": 15865, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.468401750000001, "cost_usd": 5.468401750000001, "denominator": 20, "duration_basis": "run", "duration_ms": 521321, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 132, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4402346, "tokens_cache_write": 295915, "tokens_input": 2451, "tokens_output": 29646, "tool_calls": 248, "tool_calls_by_class": { "cc_builtin": 240, "subagent": 8 }, "tool_result_tokens": 137362, "tool_time_ms": 34793, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.781194000000003, "cost_usd": 8.781194000000003, "denominator": 20, "duration_basis": "run", "duration_ms": 435471, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 174, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7491658, "tokens_cache_write": 453629, "tokens_input": 417, "tokens_output": 37571, "tool_calls": 362, "tool_calls_by_class": { "cc_builtin": 349, "subagent": 13 }, "tool_result_tokens": 210952, "tool_time_ms": 56746, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.577365750000002, "cost_usd": 9.577365750000002, "denominator": 20, "duration_basis": "run", "duration_ms": 430614, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 203, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7218064, "tokens_cache_write": 554036, "tokens_input": 2906, "tokens_output": 38294, "tool_calls": 434, "tool_calls_by_class": { "cc_builtin": 415, "subagent": 19 }, "tool_result_tokens": 281950, "tool_time_ms": 54301, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.285276999999999, "cost_usd": 5.285276999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 515524, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 137, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4328994, "tokens_cache_write": 274093, "tokens_input": 173, "tokens_output": 25906, "tool_calls": 248, "tool_calls_by_class": { "cc_builtin": 240, "subagent": 8 }, "tool_result_tokens": 120605, "tool_time_ms": 25210, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.079957500000003, "cost_usd": 5.079957500000003, "denominator": 20, "duration_basis": "run", "duration_ms": 441378, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 139, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4150160, "tokens_cache_write": 270099, "tokens_input": 1611, "tokens_output": 22828, "tool_calls": 252, "tool_calls_by_class": { "cc_builtin": 244, "subagent": 8 }, "tool_result_tokens": 119836, "tool_time_ms": 27383, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.137857749999996, "cost_usd": 7.137857749999996, "denominator": 20, "duration_basis": "run", "duration_ms": 483564, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 140, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6463868, "tokens_cache_write": 352155, "tokens_input": 170, "tokens_output": 29593, "tool_calls": 260, "tool_calls_by_class": { "cc_builtin": 250, "subagent": 10 }, "tool_result_tokens": 148972, "tool_time_ms": 35367, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.507673250000001, "cost_usd": 7.507673250000001, "denominator": 20, "duration_basis": "run", "duration_ms": 394033, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 147, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5223269, "tokens_cache_write": 429286, "tokens_input": 10518, "tokens_output": 30368, "tool_calls": 311, "tool_calls_by_class": { "cc_builtin": 298, "subagent": 13 }, "tool_result_tokens": 207895, "tool_time_ms": 70462, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.611767749999996, "cost_usd": 6.611767749999996, "denominator": 20, "duration_basis": "run", "duration_ms": 362093, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 151, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5335163, "tokens_cache_write": 323829, "tokens_input": 234, "tokens_output": 31446, "tool_calls": 317, "tool_calls_by_class": { "cc_builtin": 307, "subagent": 10 }, "tool_result_tokens": 153644, "tool_time_ms": 47216, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.726880250000003, "cost_usd": 7.726880250000003, "denominator": 20, "duration_basis": "run", "duration_ms": 477170, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 174, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6581208, "tokens_cache_write": 368508, "tokens_input": 449, "tokens_output": 36476, "tool_calls": 349, "tool_calls_by_class": { "cc_builtin": 336, "subagent": 13 }, "tool_result_tokens": 177158, "tool_time_ms": 51692, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.68063725, "cost_usd": 5.68063725, "denominator": 20, "duration_basis": "run", "duration_ms": 436763, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 74, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4812227, "tokens_cache_write": 295235, "tokens_input": 84, "tokens_output": 31286, "tool_calls": 211, "tool_calls_by_class": { "cc_builtin": 207, "subagent": 4 }, "tool_result_tokens": 170328, "tool_time_ms": 22449, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.719590000000001, "cost_usd": 5.719590000000001, "denominator": 30, "duration_basis": "run", "duration_ms": 839707, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 129, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4188665, "tokens_cache_write": 356600, "tokens_input": 431, "tokens_output": 26735, "tool_calls": 245, "tool_calls_by_class": { "cc_builtin": 232, "subagent": 13 }, "tool_result_tokens": 183445, "tool_time_ms": 6873, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.216886249999998, "cost_usd": 6.216886249999998, "denominator": 30, "duration_basis": "run", "duration_ms": 290402, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 122, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4435960, "tokens_cache_write": 383977, "tokens_input": 510, "tokens_output": 29889, "tool_calls": 281, "tool_calls_by_class": { "cc_builtin": 269, "subagent": 12 }, "tool_result_tokens": 210694, "tool_time_ms": 7544, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.5271095000000001, "cost_usd": 1.5271095000000001, "denominator": 30, "duration_basis": "run", "duration_ms": 208221, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 13, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 827599, "tokens_cache_write": 83811, "tokens_input": 15, "tokens_output": 11005, "tool_calls": 92, "tool_calls_by_class": { "cc_builtin": 92 }, "tool_result_tokens": 48706, "tool_time_ms": 2585, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.927302249999999, "cost_usd": 4.927302249999999, "denominator": 30, "duration_basis": "run", "duration_ms": 270608, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 49, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2818597, "tokens_cache_write": 350795, "tokens_input": 282, "tokens_output": 25695, "tool_calls": 176, "tool_calls_by_class": { "cc_builtin": 171, "subagent": 5 }, "tool_result_tokens": 204567, "tool_time_ms": 4747, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.204343249999997, "cost_usd": 5.204343249999997, "denominator": 30, "duration_basis": "run", "duration_ms": 255601, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 96, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3390779, "tokens_cache_write": 360954, "tokens_input": 124, "tokens_output": 21902, "tool_calls": 233, "tool_calls_by_class": { "cc_builtin": 222, "subagent": 11 }, "tool_result_tokens": 202824, "tool_time_ms": 7249, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.0326075, "cost_usd": 6.0326075, "denominator": 30, "duration_basis": "run", "duration_ms": 302526, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 150, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3608620, "tokens_cache_write": 380401, "tokens_input": 13820, "tokens_output": 32282, "tool_calls": 317, "tool_calls_by_class": { "cc_builtin": 296, "subagent": 21 }, "tool_result_tokens": 184125, "tool_time_ms": 8014, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.867752000000001, "cost_usd": 5.867752000000001, "denominator": 30, "duration_basis": "run", "duration_ms": 316375, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 122, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4021264, "tokens_cache_write": 365845, "tokens_input": 290, "tokens_output": 28036, "tool_calls": 302, "tool_calls_by_class": { "cc_builtin": 290, "subagent": 12 }, "tool_result_tokens": 193748, "tool_time_ms": 9889, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1468524999999996, "cost_usd": 2.1468524999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 306512, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 21, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1656725, "tokens_cache_write": 98247, "tokens_input": 379, "tokens_output": 13365, "tool_calls": 114, "tool_calls_by_class": { "cc_builtin": 114 }, "tool_result_tokens": 56107, "tool_time_ms": 3221, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.18528275, "cost_usd": 3.18528275, "denominator": 30, "duration_basis": "run", "duration_ms": 383772, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 58, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2014253, "tokens_cache_write": 199938, "tokens_input": 84, "tokens_output": 15044, "tool_calls": 197, "tool_calls_by_class": { "cc_builtin": 191, "subagent": 6 }, "tool_result_tokens": 97500, "tool_time_ms": 5871, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.315999999999997, "cost_usd": 8.315999999999997, "denominator": 30, "duration_basis": "run", "duration_ms": 382621, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 165, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6202030, "tokens_cache_write": 509718, "tokens_input": 205, "tokens_output": 40185, "tool_calls": 339, "tool_calls_by_class": { "cc_builtin": 322, "subagent": 17 }, "tool_result_tokens": 272828, "tool_time_ms": 10669, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.530760750000001, "cost_usd": 5.530760750000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 871931, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7204179, "tokens_cache_write": 164568, "tokens_input": 173, "tokens_output": 21788, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 145, "subagent": 3 }, "tool_result_tokens": 91045, "tool_time_ms": 20554, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.3574652500000015, "cost_usd": 6.3574652500000015, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 926726, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 149, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8649918, "tokens_cache_write": 168568, "tokens_input": 282, "tokens_output": 25160, "tool_calls": 170, "tool_calls_by_class": { "cc_builtin": 167, "subagent": 3 }, "tool_result_tokens": 88423, "tool_time_ms": 17132, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.5296647499999985, "cost_usd": 6.5296647499999985, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 947712, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 144, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8834132, "tokens_cache_write": 167608, "tokens_input": 2972, "tokens_output": 27343, "tool_calls": 168, "tool_calls_by_class": { "cc_builtin": 165, "subagent": 3 }, "tool_result_tokens": 87872, "tool_time_ms": 28334, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.73743475, "cost_usd": 7.73743475, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1149763, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 149, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9089287, "tokens_cache_write": 279341, "tokens_input": 3070, "tokens_output": 30152, "tool_calls": 254, "tool_calls_by_class": { "cc_builtin": 249, "subagent": 5 }, "tool_result_tokens": 150872, "tool_time_ms": 17523, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.8358525000000006, "cost_usd": 6.8358525000000006, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1146891, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 145, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9135725, "tokens_cache_write": 181633, "tokens_input": 566, "tokens_output": 28242, "tool_calls": 179, "tool_calls_by_class": { "cc_builtin": 176, "subagent": 3 }, "tool_result_tokens": 95661, "tool_time_ms": 20050, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.29300975, "cost_usd": 8.29300975, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1121463, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 168, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11408412, "tokens_cache_write": 204810, "tokens_input": 218, "tokens_output": 33200, "tool_calls": 202, "tool_calls_by_class": { "cc_builtin": 198, "subagent": 4 }, "tool_result_tokens": 96814, "tool_time_ms": 19127, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.69237825, "cost_usd": 7.69237825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1235903, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 150, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8750619, "tokens_cache_write": 292770, "tokens_input": 447, "tokens_output": 32182, "tool_calls": 232, "tool_calls_by_class": { "cc_builtin": 225, "subagent": 7 }, "tool_result_tokens": 160539, "tool_time_ms": 33608, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.715739749999999, "cost_usd": 7.715739749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1223967, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 151, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10224397, "tokens_cache_write": 215827, "tokens_input": 5464, "tokens_output": 29401, "tool_calls": 196, "tool_calls_by_class": { "cc_builtin": 191, "subagent": 5 }, "tool_result_tokens": 118486, "tool_time_ms": 45635, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.386965000000002, "cost_usd": 7.386965000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1276379, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 145, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8275230, "tokens_cache_write": 279186, "tokens_input": 197, "tokens_output": 31738, "tool_calls": 250, "tool_calls_by_class": { "cc_builtin": 244, "subagent": 6 }, "tool_result_tokens": 145540, "tool_time_ms": 20281, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.8936855, "cost_usd": 6.8936855, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 991461, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 143, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9695846, "tokens_cache_write": 158613, "tokens_input": 1766, "tokens_output": 26491, "tool_calls": 166, "tool_calls_by_class": { "cc_builtin": 163, "subagent": 3 }, "tool_result_tokens": 82565, "tool_time_ms": 14899, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.91082, "cost_usd": 3.91082, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 639668, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 115, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5703000, "tokens_cache_write": 73997, "tokens_input": 175, "tokens_output": 12739, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 85 }, "tool_result_tokens": 19650, "tool_time_ms": 3872, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2787854999999997, "cost_usd": 3.2787854999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 633177, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 105, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5039431, "tokens_cache_write": 44147, "tokens_input": 165, "tokens_output": 12671, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 82 }, "tool_result_tokens": 15627, "tool_time_ms": 3991, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7343384999999993, "cost_usd": 3.7343384999999993, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 564317, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 119, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5860077, "tokens_cache_write": 46293, "tokens_input": 179, "tokens_output": 13619, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 89 }, "tool_result_tokens": 16598, "tool_time_ms": 4115, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8414645000000003, "cost_usd": 3.8414645000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 658208, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 119, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5967419, "tokens_cache_write": 53126, "tokens_input": 179, "tokens_output": 13024, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 89 }, "tool_result_tokens": 22326, "tool_time_ms": 3889, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4890904999999997, "cost_usd": 3.4890904999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 650929, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 112, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5426761, "tokens_cache_write": 44670, "tokens_input": 172, "tokens_output": 13126, "tool_calls": 93, "tool_calls_by_class": { "cc_builtin": 93 }, "tool_result_tokens": 15548, "tool_time_ms": 3776, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1767864999999995, "cost_usd": 4.1767864999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 563192, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6534423, "tokens_cache_write": 53385, "tokens_input": 185, "tokens_output": 14992, "tool_calls": 97, "tool_calls_by_class": { "cc_builtin": 97 }, "tool_result_tokens": 20816, "tool_time_ms": 4414, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4480390000000005, "cost_usd": 3.4480390000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 524767, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 106, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5353668, "tokens_cache_write": 43235, "tokens_input": 166, "tokens_output": 13521, "tool_calls": 93, "tool_calls_by_class": { "cc_builtin": 93 }, "tool_result_tokens": 17544, "tool_time_ms": 3816, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7302920000000004, "cost_usd": 3.7302920000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 616559, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 108, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5840464, "tokens_cache_write": 52130, "tokens_input": 312, "tokens_output": 11488, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 90 }, "tool_result_tokens": 22118, "tool_time_ms": 4521, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.247772, "cost_usd": 3.247772, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 496130, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 106, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5110094, "tokens_cache_write": 40621, "tokens_input": 203, "tokens_output": 11420, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 85 }, "tool_result_tokens": 18738, "tool_time_ms": 4190, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.24462, "cost_usd": 3.24462, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 604239, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 112, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5211970, "tokens_cache_write": 36383, "tokens_input": 316, "tokens_output": 10929, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 83 }, "tool_result_tokens": 16143, "tool_time_ms": 3854, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.865363749999998, "cost_usd": 13.865363749999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2653165, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 262, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8386575, "tokens_cache_write": 1000274, "tokens_input": 9918, "tokens_output": 55643, "tool_calls": 640, "tool_calls_by_class": { "cc_builtin": 621, "subagent": 19 }, "tool_result_tokens": 434561, "tool_time_ms": 48009, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 15.016338250000002, "cost_usd": 15.016338250000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2623983, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 280, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10300484, "tokens_cache_write": 1068338, "tokens_input": 2294, "tokens_output": 53191, "tool_calls": 591, "tool_calls_by_class": { "cc_builtin": 574, "subagent": 17 }, "tool_result_tokens": 475226, "tool_time_ms": 48517, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.599786000000002, "cost_usd": 13.599786000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2404923, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 252, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7971147, "tokens_cache_write": 1023609, "tokens_input": 4623, "tokens_output": 60412, "tool_calls": 568, "tool_calls_by_class": { "cc_builtin": 550, "subagent": 18 }, "tool_result_tokens": 483175, "tool_time_ms": 36615, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.244232000000002, "cost_usd": 12.244232000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2531935, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 256, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7727984, "tokens_cache_write": 849234, "tokens_input": 775, "tokens_output": 49849, "tool_calls": 566, "tool_calls_by_class": { "cc_builtin": 546, "subagent": 20 }, "tool_result_tokens": 338802, "tool_time_ms": 29494, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.70585225, "cost_usd": 12.70585225, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2391881, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 236, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7561607, "tokens_cache_write": 920490, "tokens_input": 2700, "tokens_output": 48309, "tool_calls": 580, "tool_calls_by_class": { "cc_builtin": 562, "subagent": 18 }, "tool_result_tokens": 401537, "tool_time_ms": 37912, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.362696750000001, "cost_usd": 13.362696750000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2252102, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 252, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8443871, "tokens_cache_write": 962067, "tokens_input": 4779, "tokens_output": 53834, "tool_calls": 561, "tool_calls_by_class": { "cc_builtin": 544, "subagent": 17 }, "tool_result_tokens": 431559, "tool_time_ms": 30040, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.320165749999997, "cost_usd": 12.320165749999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2280993, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 231, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7234644, "tokens_cache_write": 936604, "tokens_input": 5545, "tokens_output": 45070, "tool_calls": 538, "tool_calls_by_class": { "cc_builtin": 520, "subagent": 18 }, "tool_result_tokens": 384820, "tool_time_ms": 23064, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.883015250000001, "cost_usd": 13.883015250000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2386734, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 261, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8594048, "tokens_cache_write": 1009346, "tokens_input": 3290, "tokens_output": 49472, "tool_calls": 606, "tool_calls_by_class": { "cc_builtin": 585, "subagent": 21 }, "tool_result_tokens": 469783, "tool_time_ms": 31593, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.687901499999999, "cost_usd": 12.687901499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2473274, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 263, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8345448, "tokens_cache_write": 886605, "tokens_input": 1608, "tokens_output": 49664, "tool_calls": 578, "tool_calls_by_class": { "cc_builtin": 559, "subagent": 19 }, "tool_result_tokens": 382591, "tool_time_ms": 23092, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.2146995, "cost_usd": 12.2146995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2484096, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 230, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7018009, "tokens_cache_write": 894134, "tokens_input": 7823, "tokens_output": 45605, "tool_calls": 513, "tool_calls_by_class": { "cc_builtin": 494, "subagent": 19 }, "tool_result_tokens": 386464, "tool_time_ms": 21071, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.25698275, "cost_usd": 8.25698275, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1431288, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 166, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5029180, "tokens_cache_write": 642971, "tokens_input": 301, "tokens_output": 33870, "tool_calls": 317, "tool_calls_by_class": { "cc_builtin": 306, "subagent": 11 }, "tool_result_tokens": 259129, "tool_time_ms": 13177, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.76938075, "cost_usd": 8.76938075, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1482590, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 181, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5126267, "tokens_cache_write": 679765, "tokens_input": 318, "tokens_output": 32696, "tool_calls": 348, "tool_calls_by_class": { "cc_builtin": 336, "subagent": 12 }, "tool_result_tokens": 273506, "tool_time_ms": 14110, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.494272, "cost_usd": 8.494272, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1518815, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 178, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5003949, "tokens_cache_write": 660784, "tokens_input": 264, "tokens_output": 33534, "tool_calls": 344, "tool_calls_by_class": { "cc_builtin": 331, "subagent": 13 }, "tool_result_tokens": 287275, "tool_time_ms": 13074, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.524916, "cost_usd": 8.524916, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1523285, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 191, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5583162, "tokens_cache_write": 605637, "tokens_input": 1025, "tokens_output": 37517, "tool_calls": 369, "tool_calls_by_class": { "cc_builtin": 356, "subagent": 13 }, "tool_result_tokens": 269127, "tool_time_ms": 13935, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.364764000000001, "cost_usd": 8.364764000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1489378, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 173, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4987513, "tokens_cache_write": 624048, "tokens_input": 257, "tokens_output": 35706, "tool_calls": 336, "tool_calls_by_class": { "cc_builtin": 321, "subagent": 15 }, "tool_result_tokens": 286302, "tool_time_ms": 13535, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.459531499999999, "cost_usd": 8.459531499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1382487, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 181, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5358056, "tokens_cache_write": 627272, "tokens_input": 316, "tokens_output": 33487, "tool_calls": 326, "tool_calls_by_class": { "cc_builtin": 314, "subagent": 12 }, "tool_result_tokens": 273848, "tool_time_ms": 13555, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.85113575, "cost_usd": 8.85113575, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1569584, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 188, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5432164, "tokens_cache_write": 652479, "tokens_input": 5703, "tokens_output": 38316, "tool_calls": 367, "tool_calls_by_class": { "cc_builtin": 354, "subagent": 13 }, "tool_result_tokens": 276950, "tool_time_ms": 14413, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.201208, "cost_usd": 8.201208, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1501611, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 187, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5361701, "tokens_cache_write": 570184, "tokens_input": 4793, "tokens_output": 36281, "tool_calls": 345, "tool_calls_by_class": { "cc_builtin": 332, "subagent": 13 }, "tool_result_tokens": 247287, "tool_time_ms": 14139, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.423043750000001, "cost_usd": 8.423043750000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1430664, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 173, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5058365, "tokens_cache_write": 636862, "tokens_input": 6884, "tokens_output": 33653, "tool_calls": 315, "tool_calls_by_class": { "cc_builtin": 303, "subagent": 12 }, "tool_result_tokens": 285069, "tool_time_ms": 13035, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.942678749999999, "cost_usd": 8.942678749999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1585452, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 179, "model": "claude-opus-4-6", "path": "runs/cold/claude-opus-4-6/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5372375, "tokens_cache_write": 665823, "tokens_input": 7568, "tokens_output": 33201, "tool_calls": 345, "tool_calls_by_class": { "cc_builtin": 331, "subagent": 14 }, "tool_result_tokens": 305785, "tool_time_ms": 15008, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.851452500000001, "cost_usd": 4.851452500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 581084, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5961485, "tokens_cache_write": 101122, "tokens_input": 169, "tokens_output": 35752, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 69, "subagent": 1 }, "tool_result_tokens": 28458, "tool_time_ms": 10806, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.757861999999999, "cost_usd": 5.757861999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 667388, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 102, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7365574, "tokens_cache_write": 115671, "tokens_input": 193, "tokens_output": 37921, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 84, "subagent": 2 }, "tool_result_tokens": 32467, "tool_time_ms": 23500, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.266741999999998, "cost_usd": 5.266741999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 651601, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 95, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6734739, "tokens_cache_write": 107517, "tokens_input": 180, "tokens_output": 34628, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 75, "subagent": 2 }, "tool_result_tokens": 30734, "tool_time_ms": 33340, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.889714249999999, "cost_usd": 5.889714249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 761479, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 98, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7413741, "tokens_cache_write": 115695, "tokens_input": 187, "tokens_output": 42216, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 77, "subagent": 1 }, "tool_result_tokens": 32928, "tool_time_ms": 78561, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.704783000000001, "cost_usd": 4.704783000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 580178, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 85, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5809231, "tokens_cache_write": 99694, "tokens_input": 160, "tokens_output": 33597, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 65, "subagent": 2 }, "tool_result_tokens": 29167, "tool_time_ms": 11111, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8377497499999995, "cost_usd": 5.8377497499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1036943, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 129, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5548662, "tokens_cache_write": 202096, "tokens_input": 1896, "tokens_output": 37087, "tool_calls": 136, "tool_calls_by_class": { "cc_builtin": 127, "subagent": 9 }, "tool_result_tokens": 63680, "tool_time_ms": 51726, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.96944275, "cost_usd": 5.96944275, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 792008, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 116, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7829503, "tokens_cache_write": 105496, "tokens_input": 220, "tokens_output": 40678, "tool_calls": 97, "tool_calls_by_class": { "cc_builtin": 96, "subagent": 1 }, "tool_result_tokens": 26892, "tool_time_ms": 66229, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.81091775, "cost_usd": 4.81091775, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 644775, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 93, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6025803, "tokens_cache_write": 96762, "tokens_input": 178, "tokens_output": 33227, "tool_calls": 75, "tool_calls_by_class": { "cc_builtin": 73, "subagent": 2 }, "tool_result_tokens": 27295, "tool_time_ms": 54385, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.792103, "cost_usd": 5.792103, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 786392, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 108, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7459146, "tokens_cache_write": 113267, "tokens_input": 204, "tokens_output": 38467, "tool_calls": 93, "tool_calls_by_class": { "cc_builtin": 91, "subagent": 2 }, "tool_result_tokens": 32994, "tool_time_ms": 57802, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.05388525, "cost_usd": 6.05388525, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 672828, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 106, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7846688, "tokens_cache_write": 112875, "tokens_input": 202, "tokens_output": 40293, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 87, "subagent": 1 }, "tool_result_tokens": 33132, "tool_time_ms": 12688, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4747225, "cost_usd": 2.4747225, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 316862, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 75, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3361655, "tokens_cache_write": 40848, "tokens_input": 143, "tokens_output": 15388, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 10439, "tool_time_ms": 3562, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0214524999999997, "cost_usd": 3.0214524999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 394656, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 88, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4165285, "tokens_cache_write": 46802, "tokens_input": 168, "tokens_output": 18798, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 11600, "tool_time_ms": 4848, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8773660000000003, "cost_usd": 2.8773660000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 374670, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 86, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3984802, "tokens_cache_write": 43912, "tokens_input": 164, "tokens_output": 17801, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 10840, "tool_time_ms": 3505, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1992309999999997, "cost_usd": 3.1992309999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 412945, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 91, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4430802, "tokens_cache_write": 50689, "tokens_input": 173, "tokens_output": 19043, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 13526, "tool_time_ms": 4097, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3365175000000002, "cost_usd": 3.3365175000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 408617, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 94, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4580515, "tokens_cache_write": 51919, "tokens_input": 179, "tokens_output": 21047, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 13253, "tool_time_ms": 4029, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9498125, "cost_usd": 2.9498125, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 412071, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4054035, "tokens_cache_write": 44749, "tokens_input": 166, "tokens_output": 18979, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 10464, "tool_time_ms": 3669, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.639987, "cost_usd": 2.639987, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 367221, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3577224, "tokens_cache_write": 43017, "tokens_input": 151, "tokens_output": 16818, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 10922, "tool_time_ms": 3560, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.811821, "cost_usd": 2.811821, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 323123, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3925292, "tokens_cache_write": 46542, "tokens_input": 151, "tokens_output": 15320, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 13501, "tool_time_ms": 2875, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8483840000000002, "cost_usd": 2.8483840000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 374751, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3856368, "tokens_cache_write": 46135, "tokens_input": 160, "tokens_output": 18322, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 11618, "tool_time_ms": 3460, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.219414, "cost_usd": 3.219414, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 392638, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 88, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4349628, "tokens_cache_write": 53279, "tokens_input": 167, "tokens_output": 20439, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 14644, "tool_time_ms": 3612, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.9900212499999999, "cost_usd": 0.9900212499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 104338, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 17, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 309660, "tokens_cache_write": 82212, "tokens_input": 34, "tokens_output": 3565, "tool_calls": 20, "tool_calls_by_class": { "cc_builtin": 17, "subagent": 3 }, "tool_result_tokens": 9561, "tool_time_ms": 923, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.72537725, "cost_usd": 0.72537725, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 132825, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 16, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 298037, "tokens_cache_write": 55702, "tokens_input": 31, "tokens_output": 5155, "tool_calls": 20, "tool_calls_by_class": { "cc_builtin": 18, "subagent": 2 }, "tool_result_tokens": 8096, "tool_time_ms": 3484, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.803481, "cost_usd": 0.803481, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 108594, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 21, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 494642, "tokens_cache_write": 47614, "tokens_input": 41, "tokens_output": 6392, "tool_calls": 23, "tool_calls_by_class": { "cc_builtin": 21, "subagent": 2 }, "tool_result_tokens": 10622, "tool_time_ms": 5848, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.930354, "cost_usd": 0.930354, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 8256, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 27, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 638323, "tokens_cache_write": 53635, "tokens_input": 190, "tokens_output": 4678, "tool_calls": 25, "tool_calls_by_class": { "cc_builtin": 22, "subagent": 3 }, "tool_result_tokens": 11116, "tool_time_ms": 2908, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.748971, "cost_usd": 0.748971, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 59433, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 17, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 334687, "tokens_cache_write": 53932, "tokens_input": 34, "tokens_output": 5022, "tool_calls": 20, "tool_calls_by_class": { "cc_builtin": 17, "subagent": 3 }, "tool_result_tokens": 10206, "tool_time_ms": 3050, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.76334225, "cost_usd": 0.76334225, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 77183, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 18, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 355587, "tokens_cache_write": 50286, "tokens_input": 72, "tokens_output": 4664, "tool_calls": 22, "tool_calls_by_class": { "cc_builtin": 19, "subagent": 3 }, "tool_result_tokens": 11211, "tool_time_ms": 2646, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.673195, "cost_usd": 0.673195, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 117927, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 16, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 299995, "tokens_cache_write": 48798, "tokens_input": 174, "tokens_output": 3958, "tool_calls": 19, "tool_calls_by_class": { "cc_builtin": 16, "subagent": 3 }, "tool_result_tokens": 9812, "tool_time_ms": 3006, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.73727225, "cost_usd": 0.73727225, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 68138, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 20, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 412612, "tokens_cache_write": 48979, "tokens_input": 39, "tokens_output": 4173, "tool_calls": 25, "tool_calls_by_class": { "cc_builtin": 21, "subagent": 4 }, "tool_result_tokens": 11238, "tool_time_ms": 3885, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.90336375, "cost_usd": 0.90336375, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 50201, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 24, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 557660, "tokens_cache_write": 55066, "tokens_input": 46, "tokens_output": 3957, "tool_calls": 27, "tool_calls_by_class": { "cc_builtin": 24, "subagent": 3 }, "tool_result_tokens": 11984, "tool_time_ms": 4648, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.70352125, "cost_usd": 0.70352125, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 43125, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 20, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 481960, "tokens_cache_write": 38863, "tokens_input": 40, "tokens_output": 3741, "tool_calls": 20, "tool_calls_by_class": { "cc_builtin": 17, "subagent": 3 }, "tool_result_tokens": 9614, "tool_time_ms": 3604, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.42838575, "cost_usd": 9.42838575, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 851866, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 235, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|11", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5910934, "tokens_cache_write": 510850, "tokens_input": 460, "tokens_output": 61124, "tool_calls": 274, "tool_calls_by_class": { "cc_builtin": 253, "subagent": 21 }, "tool_result_tokens": 123419, "tool_time_ms": 104448, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.5503475, "cost_usd": 8.5503475, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1048128, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 203, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|12", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5437299, "tokens_cache_write": 482265, "tokens_input": 394, "tokens_output": 51919, "tool_calls": 238, "tool_calls_by_class": { "cc_builtin": 220, "subagent": 18 }, "tool_result_tokens": 123019, "tool_time_ms": 25517, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.965413750000002, "cost_usd": 8.965413750000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 944422, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 233, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|13", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6054437, "tokens_cache_write": 487547, "tokens_input": 452, "tokens_output": 62619, "tool_calls": 258, "tool_calls_by_class": { "cc_builtin": 240, "subagent": 18 }, "tool_result_tokens": 114673, "tool_time_ms": 24129, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.4319965, "cost_usd": 9.4319965, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 747757, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 237, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|14", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6122926, "tokens_cache_write": 512756, "tokens_input": 460, "tokens_output": 60036, "tool_calls": 272, "tool_calls_by_class": { "cc_builtin": 256, "subagent": 16 }, "tool_result_tokens": 126430, "tool_time_ms": 33128, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.144421499999998, "cost_usd": 10.144421499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1065546, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 245, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|15", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6235556, "tokens_cache_write": 561453, "tokens_input": 2131, "tokens_output": 65859, "tool_calls": 269, "tool_calls_by_class": { "cc_builtin": 248, "subagent": 21 }, "tool_result_tokens": 130321, "tool_time_ms": 103934, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.457234, "cost_usd": 8.457234, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 886098, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 210, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep16", "query_calls": 0, "rep": 16, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|16", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5395833, "tokens_cache_write": 460544, "tokens_input": 2645, "tokens_output": 55841, "tool_calls": 245, "tool_calls_by_class": { "cc_builtin": 224, "subagent": 21 }, "tool_result_tokens": 112184, "tool_time_ms": 24353, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.51974475, "cost_usd": 8.51974475, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 802471, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 214, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep17", "query_calls": 0, "rep": 17, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|17", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5246607, "tokens_cache_write": 471125, "tokens_input": 2736, "tokens_output": 60873, "tool_calls": 238, "tool_calls_by_class": { "cc_builtin": 217, "subagent": 21 }, "tool_result_tokens": 113757, "tool_time_ms": 69671, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.191375749999999, "cost_usd": 9.191375749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1197712, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 226, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep18", "query_calls": 0, "rep": 18, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|18", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5532734, "tokens_cache_write": 518105, "tokens_input": 442, "tokens_output": 62915, "tool_calls": 272, "tool_calls_by_class": { "cc_builtin": 249, "subagent": 23 }, "tool_result_tokens": 113513, "tool_time_ms": 69806, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.931420249999999, "cost_usd": 7.931420249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 898450, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_bare", "llm_calls": 194, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep19", "query_calls": 0, "rep": 19, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|19", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4873872, "tokens_cache_write": 466965, "tokens_input": 376, "tokens_output": 48835, "tool_calls": 207, "tool_calls_by_class": { "cc_builtin": 190, "subagent": 17 }, "tool_result_tokens": 112729, "tool_time_ms": 23699, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.954755749999999, "cost_usd": 9.954755749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 872998, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 246, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos-hard/rep20", "query_calls": 0, "rep": 20, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|20", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6306275, "tokens_cache_write": 531980, "tokens_input": 481, "tokens_output": 60412, "tool_calls": 285, "tool_calls_by_class": { "cc_builtin": 263, "subagent": 22 }, "tool_result_tokens": 138467, "tool_time_ms": 31919, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.6716755, "cost_usd": 0.6716755, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 52621, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 13, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 275606, "tokens_cache_write": 51748, "tokens_input": 26, "tokens_output": 2100, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "subagent": 2 }, "tool_result_tokens": 2469, "tool_time_ms": 859, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.307417, "cost_usd": 0.307417, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 38898, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 9, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 250624, "tokens_cache_write": 15089, "tokens_input": 18, "tokens_output": 1245, "tool_calls": 7, "tool_calls_by_class": { "cc_builtin": 7 }, "tool_result_tokens": 1768, "tool_time_ms": 1166, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.535519, "cost_usd": 0.535519, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 33663, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 17, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 405553, "tokens_cache_write": 29985, "tokens_input": 34, "tokens_output": 2415, "tool_calls": 15, "tool_calls_by_class": { "cc_builtin": 14, "subagent": 1 }, "tool_result_tokens": 3395, "tool_time_ms": 1693, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.694694, "cost_usd": 0.694694, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 49843, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 21, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 439883, "tokens_cache_write": 45009, "tokens_input": 42, "tokens_output": 2260, "tool_calls": 21, "tool_calls_by_class": { "cc_builtin": 18, "subagent": 3 }, "tool_result_tokens": 5116, "tool_time_ms": 1167, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.32919299999999996, "cost_usd": 0.32919299999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 38261, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 278656, "tokens_cache_write": 15784, "tokens_input": 20, "tokens_output": 1277, "tool_calls": 8, "tool_calls_by_class": { "cc_builtin": 8 }, "tool_result_tokens": 1215, "tool_time_ms": 512, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.45047025, "cost_usd": 0.45047025, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 56320, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 13, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 299963, "tokens_cache_write": 29644, "tokens_input": 26, "tokens_output": 1947, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 11, "subagent": 1 }, "tool_result_tokens": 1767, "tool_time_ms": 1254, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.5726497500000001, "cost_usd": 0.5726497500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 46411, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 20, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 423052, "tokens_cache_write": 36883, "tokens_input": 40, "tokens_output": 2973, "tool_calls": 16, "tool_calls_by_class": { "cc_builtin": 14, "subagent": 2 }, "tool_result_tokens": 3135, "tool_time_ms": 1574, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.266931, "cost_usd": 0.266931, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 46876, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 315812, "tokens_cache_write": 7434, "tokens_input": 22, "tokens_output": 1383, "tool_calls": 9, "tool_calls_by_class": { "cc_builtin": 9 }, "tool_result_tokens": 1162, "tool_time_ms": 1104, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.4198505, "cost_usd": 0.4198505, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 55540, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 13, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 285546, "tokens_cache_write": 25818, "tokens_input": 26, "tokens_output": 2070, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "subagent": 2 }, "tool_result_tokens": 3872, "tool_time_ms": 991, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.5930415000000001, "cost_usd": 0.5930415000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 76955, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_bare", "llm_calls": 15, "model": "claude-opus-4-8", "path": "runs/cold/claude-opus-4-8/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 293353, "tokens_cache_write": 46371, "tokens_input": 30, "tokens_output": 1833, "tool_calls": 13, "tool_calls_by_class": { "cc_builtin": 10, "subagent": 3 }, "tool_result_tokens": 4054, "tool_time_ms": 1893, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1237500000000002, "cost_usd": 3.1237500000000002, "denominator": 20, "duration_basis": "run", "duration_ms": 463765, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 38, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3018630, "tokens_cache_write": 92941, "tokens_input": 70, "tokens_output": 27387, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 38185, "tool_time_ms": 77083, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9690274999999997, "cost_usd": 2.9690274999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 503524, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 36, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2953285, "tokens_cache_write": 91410, "tokens_input": 67, "tokens_output": 23118, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 39950, "tool_time_ms": 164371, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.291545000000001, "cost_usd": 3.291545000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 451417, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 38, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3358360, "tokens_cache_write": 100466, "tokens_input": 71, "tokens_output": 24294, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 44624, "tool_time_ms": 93427, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.615433999999999, "cost_usd": 3.615433999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 644868, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 50, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4185528, "tokens_cache_write": 95928, "tokens_input": 93, "tokens_output": 22517, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 73 }, "tool_result_tokens": 42482, "tool_time_ms": 268860, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3352715000000006, "cost_usd": 3.3352715000000006, "denominator": 20, "duration_basis": "run", "duration_ms": 455807, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 38, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3303473, "tokens_cache_write": 104391, "tokens_input": 4220, "tokens_output": 24741, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 46784, "tool_time_ms": 82923, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7810045, "cost_usd": 2.7810045, "denominator": 20, "duration_basis": "run", "duration_ms": 385671, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2635639, "tokens_cache_write": 89457, "tokens_input": 63, "tokens_output": 22732, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 37588, "tool_time_ms": 42369, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9042379999999994, "cost_usd": 2.9042379999999994, "denominator": 20, "duration_basis": "run", "duration_ms": 380441, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2815616, "tokens_cache_write": 93175, "tokens_input": 2186, "tokens_output": 22150, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 40598, "tool_time_ms": 37055, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0365085000000005, "cost_usd": 3.0365085000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 390501, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 35, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2928997, "tokens_cache_write": 100389, "tokens_input": 64, "tokens_output": 22712, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 42376, "tool_time_ms": 58573, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1725319999999995, "cost_usd": 3.1725319999999995, "denominator": 20, "duration_basis": "run", "duration_ms": 419766, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 41, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3277594, "tokens_cache_write": 94173, "tokens_input": 76, "tokens_output": 23665, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 39600, "tool_time_ms": 61855, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.621175500000001, "cost_usd": 2.621175500000001, "denominator": 20, "duration_basis": "run", "duration_ms": 340936, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2491321, "tokens_cache_write": 81553, "tokens_input": 62, "tokens_output": 22387, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 32149, "tool_time_ms": 35082, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1349159999999996, "cost_usd": 2.1349159999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 224569, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 27, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1611152, "tokens_cache_write": 93247, "tokens_input": 49, "tokens_output": 15865, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 27411, "tool_time_ms": 1825, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1624735000000004, "cost_usd": 2.1624735000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 257169, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 26, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1600957, "tokens_cache_write": 94153, "tokens_input": 48, "tokens_output": 16809, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 47 }, "tool_result_tokens": 27406, "tool_time_ms": 2140, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.481743499999999, "cost_usd": 2.481743499999999, "denominator": 30, "duration_basis": "run", "duration_ms": 266793, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2117997, "tokens_cache_write": 97418, "tokens_input": 63, "tokens_output": 17930, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 28276, "tool_time_ms": 2950, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0042729999999995, "cost_usd": 2.0042729999999995, "denominator": 30, "duration_basis": "run", "duration_ms": 217511, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1434086, "tokens_cache_write": 90756, "tokens_input": 44, "tokens_output": 15178, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 26047, "tool_time_ms": 1718, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9901805000000001, "cost_usd": 1.9901805000000001, "denominator": 30, "duration_basis": "run", "duration_ms": 226185, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1403481, "tokens_cache_write": 88647, "tokens_input": 44, "tokens_output": 16070, "tool_calls": 44, "tool_calls_by_class": { "cc_builtin": 44 }, "tool_result_tokens": 24504, "tool_time_ms": 2013, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4150730000000005, "cost_usd": 2.4150730000000005, "denominator": 30, "duration_basis": "run", "duration_ms": 285775, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 35, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2419216, "tokens_cache_write": 76014, "tokens_input": 65, "tokens_output": 17800, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 29782, "tool_time_ms": 2492, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1191590000000002, "cost_usd": 2.1191590000000002, "denominator": 30, "duration_basis": "run", "duration_ms": 247220, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 29, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1896578, "tokens_cache_write": 73845, "tokens_input": 54, "tokens_output": 17286, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 28989, "tool_time_ms": 2561, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1516385000000002, "cost_usd": 2.1516385000000002, "denominator": 30, "duration_basis": "run", "duration_ms": 245218, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 30, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2002567, "tokens_cache_write": 74823, "tokens_input": 55, "tokens_output": 16074, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 30485, "tool_time_ms": 2260, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0387160000000004, "cost_usd": 2.0387160000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 258036, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 28, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1715062, "tokens_cache_write": 74918, "tokens_input": 51, "tokens_output": 17270, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 26962, "tool_time_ms": 2793, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9553194999999999, "cost_usd": 1.9553194999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 251864, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 28, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1752979, "tokens_cache_write": 64825, "tokens_input": 51, "tokens_output": 17213, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 24424, "tool_time_ms": 2362, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.408841500000001, "cost_usd": 6.408841500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 918692, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 97, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8618283, "tokens_cache_write": 115058, "tokens_input": 184, "tokens_output": 37928, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 90 }, "tool_result_tokens": 39006, "tool_time_ms": 240906, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.05589, "cost_usd": 7.05589, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 768676, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 95, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9377960, "tokens_cache_write": 132096, "tokens_input": 1495, "tokens_output": 41539, "tool_calls": 102, "tool_calls_by_class": { "cc_builtin": 102 }, "tool_result_tokens": 47441, "tool_time_ms": 70612, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.370529, "cost_usd": 6.370529, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 861743, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8560448, "tokens_cache_write": 116789, "tokens_input": 178, "tokens_output": 36861, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 89 }, "tool_result_tokens": 41764, "tool_time_ms": 77446, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.3307965, "cost_usd": 7.3307965, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 810848, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9990993, "tokens_cache_write": 125350, "tokens_input": 415, "tokens_output": 43189, "tool_calls": 115, "tool_calls_by_class": { "cc_builtin": 115 }, "tool_result_tokens": 42467, "tool_time_ms": 62707, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.1059035, "cost_usd": 7.1059035, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 816053, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 91, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9210667, "tokens_cache_write": 135217, "tokens_input": 170, "tokens_output": 45902, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 109 }, "tool_result_tokens": 48723, "tool_time_ms": 79896, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2985595000000005, "cost_usd": 6.2985595000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 753984, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 91, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8207129, "tokens_cache_write": 120610, "tokens_input": 814, "tokens_output": 39393, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 98 }, "tool_result_tokens": 41013, "tool_time_ms": 101747, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.173322000000001, "cost_usd": 7.173322000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 804100, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 108, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9848734, "tokens_cache_write": 118666, "tokens_input": 1689, "tokens_output": 42154, "tool_calls": 113, "tool_calls_by_class": { "cc_builtin": 113 }, "tool_result_tokens": 41013, "tool_time_ms": 66871, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.741151499999999, "cost_usd": 6.741151499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 809363, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8683643, "tokens_cache_write": 129466, "tokens_input": 294, "tokens_output": 44128, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 103 }, "tool_result_tokens": 43112, "tool_time_ms": 77356, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.483638999999999, "cost_usd": 6.483638999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 652700, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 97, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8746538, "tokens_cache_write": 117484, "tokens_input": 646, "tokens_output": 37292, "tool_calls": 91, "tool_calls_by_class": { "cc_builtin": 91 }, "tool_result_tokens": 44754, "tool_time_ms": 8695, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.933777, "cost_usd": 6.933777, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 733046, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9461314, "tokens_cache_write": 127509, "tokens_input": 1371, "tokens_output": 36847, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 103 }, "tool_result_tokens": 46755, "tool_time_ms": 72437, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.756122500000001, "cost_usd": 4.756122500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 503413, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 100, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6094215, "tokens_cache_write": 104739, "tokens_input": 190, "tokens_output": 26427, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 25100, "tool_time_ms": 4799, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.292938, "cost_usd": 4.292938, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 498692, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 98, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5814466, "tokens_cache_write": 74972, "tokens_input": 187, "tokens_output": 25402, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 69 }, "tool_result_tokens": 22766, "tool_time_ms": 4350, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.928454, "cost_usd": 4.928454, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 533858, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6715578, "tokens_cache_write": 85217, "tokens_input": 204, "tokens_output": 28699, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 77 }, "tool_result_tokens": 26612, "tool_time_ms": 5656, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.71685, "cost_usd": 4.71685, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 519280, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 104, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6475410, "tokens_cache_write": 83328, "tokens_input": 198, "tokens_output": 25795, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 74 }, "tool_result_tokens": 27303, "tool_time_ms": 4677, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.696688999999999, "cost_usd": 4.696688999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 590012, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 106, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6536368, "tokens_cache_write": 79457, "tokens_input": 202, "tokens_output": 25317, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 76 }, "tool_result_tokens": 25326, "tool_time_ms": 4790, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.3532554999999995, "cost_usd": 4.3532554999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 488881, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 100, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5924611, "tokens_cache_write": 76460, "tokens_input": 190, "tokens_output": 25016, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 23815, "tool_time_ms": 4618, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.900758499999999, "cost_usd": 4.900758499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 683042, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 108, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6766547, "tokens_cache_write": 83778, "tokens_input": 206, "tokens_output": 27147, "tool_calls": 79, "tool_calls_by_class": { "cc_builtin": 79 }, "tool_result_tokens": 26662, "tool_time_ms": 4243, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9168565, "cost_usd": 3.9168565, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 572564, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 89, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5128723, "tokens_cache_write": 73985, "tokens_input": 169, "tokens_output": 24472, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 23184, "tool_time_ms": 3911, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.006004, "cost_usd": 4.006004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 428351, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 95, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5539848, "tokens_cache_write": 66510, "tokens_input": 181, "tokens_output": 22803, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 65 }, "tool_result_tokens": 22408, "tool_time_ms": 4398, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.923886, "cost_usd": 3.923886, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 455966, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 95, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5401202, "tokens_cache_write": 64093, "tokens_input": 181, "tokens_output": 23258, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 65 }, "tool_result_tokens": 20705, "tool_time_ms": 4778, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.898342499999999, "cost_usd": 7.898342499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1118666, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 162, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5827035, "tokens_cache_write": 334734, "tokens_input": 1627, "tokens_output": 65174, "tool_calls": 223, "tool_calls_by_class": { "cc_builtin": 223 }, "tool_result_tokens": 117146, "tool_time_ms": 73002, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.955368999999999, "cost_usd": 7.955368999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1110092, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 155, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5683208, "tokens_cache_write": 357167, "tokens_input": 294, "tokens_output": 61625, "tool_calls": 225, "tool_calls_by_class": { "cc_builtin": 224, "subagent": 1 }, "tool_result_tokens": 128048, "tool_time_ms": 133125, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.830437, "cost_usd": 7.830437, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1206581, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 159, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5820624, "tokens_cache_write": 323855, "tokens_input": 305, "tokens_output": 67202, "tool_calls": 213, "tool_calls_by_class": { "cc_builtin": 213 }, "tool_result_tokens": 122361, "tool_time_ms": 87466, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.208328, "cost_usd": 7.208328, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1147182, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5449316, "tokens_cache_write": 289563, "tokens_input": 298, "tokens_output": 63462, "tool_calls": 204, "tool_calls_by_class": { "cc_builtin": 204 }, "tool_result_tokens": 102906, "tool_time_ms": 143178, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.893001499999998, "cost_usd": 6.893001499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 958027, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 152, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5201733, "tokens_cache_write": 275226, "tokens_input": 320, "tokens_output": 61531, "tool_calls": 213, "tool_calls_by_class": { "cc_builtin": 213 }, "tool_result_tokens": 95709, "tool_time_ms": 12756, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.532627499999999, "cost_usd": 7.532627499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1068101, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 146, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5244095, "tokens_cache_write": 332632, "tokens_input": 277, "tokens_output": 63315, "tool_calls": 207, "tool_calls_by_class": { "cc_builtin": 207 }, "tool_result_tokens": 128273, "tool_time_ms": 113069, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.305798499999999, "cost_usd": 7.305798499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1026063, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5414787, "tokens_cache_write": 304128, "tokens_input": 380, "tokens_output": 62209, "tool_calls": 229, "tool_calls_by_class": { "cc_builtin": 229 }, "tool_result_tokens": 112038, "tool_time_ms": 43710, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.068113500000001, "cost_usd": 7.068113500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 951520, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 148, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5238427, "tokens_cache_write": 293114, "tokens_input": 282, "tokens_output": 60654, "tool_calls": 219, "tool_calls_by_class": { "cc_builtin": 219 }, "tool_result_tokens": 111520, "tool_time_ms": 14282, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.842379999999999, "cost_usd": 6.842379999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1076597, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 145, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4993220, "tokens_cache_write": 275634, "tokens_input": 656, "tokens_output": 63446, "tool_calls": 213, "tool_calls_by_class": { "cc_builtin": 213 }, "tool_result_tokens": 102146, "tool_time_ms": 98880, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.068432, "cost_usd": 7.068432, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1016399, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 153, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5406044, "tokens_cache_write": 286231, "tokens_input": 290, "tokens_output": 60066, "tool_calls": 208, "tool_calls_by_class": { "cc_builtin": 208 }, "tool_result_tokens": 109365, "tool_time_ms": 56596, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.528762, "cost_usd": 6.528762, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 797037, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 157, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5035916, "tokens_cache_write": 321694, "tokens_input": 2665, "tokens_output": 46276, "tool_calls": 191, "tool_calls_by_class": { "cc_builtin": 191 }, "tool_result_tokens": 89788, "tool_time_ms": 9230, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.598536500000002, "cost_usd": 6.598536500000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 761332, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 154, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4963512, "tokens_cache_write": 308419, "tokens_input": 1277, "tokens_output": 45670, "tool_calls": 192, "tool_calls_by_class": { "cc_builtin": 192 }, "tool_result_tokens": 96909, "tool_time_ms": 9748, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.3347989999999985, "cost_usd": 6.3347989999999985, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 730847, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 152, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4862467, "tokens_cache_write": 293716, "tokens_input": 896, "tokens_output": 43580, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 184 }, "tool_result_tokens": 87963, "tool_time_ms": 9217, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2190095, "cost_usd": 6.2190095, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 773490, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 152, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4881382, "tokens_cache_write": 277046, "tokens_input": 299, "tokens_output": 45054, "tool_calls": 190, "tool_calls_by_class": { "cc_builtin": 190 }, "tool_result_tokens": 92028, "tool_time_ms": 8894, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.026325000000001, "cost_usd": 6.026325000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 738128, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 151, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4792303, "tokens_cache_write": 263434, "tokens_input": 409, "tokens_output": 44532, "tool_calls": 176, "tool_calls_by_class": { "cc_builtin": 176 }, "tool_result_tokens": 85794, "tool_time_ms": 8527, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.361050999999999, "cost_usd": 6.361050999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 718162, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 154, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4985065, "tokens_cache_write": 288360, "tokens_input": 526, "tokens_output": 43998, "tool_calls": 189, "tool_calls_by_class": { "cc_builtin": 189 }, "tool_result_tokens": 99807, "tool_time_ms": 9787, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.214803000000001, "cost_usd": 6.214803000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 752698, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 160, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5128835, "tokens_cache_write": 262985, "tokens_input": 312, "tokens_output": 45760, "tool_calls": 192, "tool_calls_by_class": { "cc_builtin": 192 }, "tool_result_tokens": 86436, "tool_time_ms": 10286, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.117136499999999, "cost_usd": 6.117136499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 727301, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 155, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5004222, "tokens_cache_write": 263415, "tokens_input": 415, "tokens_output": 44720, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 182 }, "tool_result_tokens": 87365, "tool_time_ms": 8897, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.035135, "cost_usd": 6.035135, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 724538, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5010069, "tokens_cache_write": 257180, "tokens_input": 454, "tokens_output": 43926, "tool_calls": 196, "tool_calls_by_class": { "cc_builtin": 196 }, "tool_result_tokens": 93610, "tool_time_ms": 9638, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8654925, "cost_usd": 5.8654925, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 728891, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 148, "model": "claude-opus-5", "path": "runs/cold/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4747158, "tokens_cache_write": 252852, "tokens_input": 2746, "tokens_output": 42610, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 184 }, "tool_result_tokens": 92191, "tool_time_ms": 9314, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.546379150000001, "cost_usd": 5.546379150000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 816579, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 149, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13776688, "tokens_cache_write": 133478, "tokens_input": 298, "tokens_output": 43607, "tool_calls": 129, "tool_calls_by_class": { "cc_builtin": 128, "subagent": 1 }, "tool_result_tokens": 36669, "tool_time_ms": 38221, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.0425162, "cost_usd": 4.0425162, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 605388, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 121, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9819304, "tokens_cache_write": 108434, "tokens_input": 242, "tokens_output": 31670, "tool_calls": 102, "tool_calls_by_class": { "cc_builtin": 101, "subagent": 1 }, "tool_result_tokens": 28203, "tool_time_ms": 19892, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.907380649999999, "cost_usd": 4.907380649999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 174059, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 183, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7461718, "tokens_cache_write": 345057, "tokens_input": 11673, "tokens_output": 37673, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 167, "subagent": 17 }, "tool_result_tokens": 118220, "tool_time_ms": 20329, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.08626165, "cost_usd": 5.08626165, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 618780, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 130, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12604283, "tokens_cache_write": 131703, "tokens_input": 260, "tokens_output": 37073, "tool_calls": 112, "tool_calls_by_class": { "cc_builtin": 111, "subagent": 1 }, "tool_result_tokens": 38002, "tool_time_ms": 19860, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.583478499999999, "cost_usd": 5.583478499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 585270, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 133, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13693580, "tokens_cache_write": 153491, "tokens_input": 266, "tokens_output": 39750, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 110, "subagent": 1 }, "tool_result_tokens": 52822, "tool_time_ms": 30809, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.677630050000001, "cost_usd": 5.677630050000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 609160, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 139, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14049041, "tokens_cache_write": 150433, "tokens_input": 278, "tokens_output": 40337, "tool_calls": 119, "tool_calls_by_class": { "cc_builtin": 118, "subagent": 1 }, "tool_result_tokens": 45448, "tool_time_ms": 44103, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.85242655, "cost_usd": 5.85242655, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 593476, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 140, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14475351, "tokens_cache_write": 154434, "tokens_input": 280, "tokens_output": 41561, "tool_calls": 123, "tool_calls_by_class": { "cc_builtin": 122, "subagent": 1 }, "tool_result_tokens": 51732, "tool_time_ms": 19005, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.494787399999999, "cost_usd": 5.494787399999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 576489, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 121, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13556848, "tokens_cache_write": 150344, "tokens_input": 242, "tokens_output": 37814, "tool_calls": 107, "tool_calls_by_class": { "cc_builtin": 105, "subagent": 2 }, "tool_result_tokens": 52161, "tool_time_ms": 18211, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.0650641, "cost_usd": 5.0650641, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 515483, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 121, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12257407, "tokens_cache_write": 145328, "tokens_input": 242, "tokens_output": 36231, "tool_calls": 102, "tool_calls_by_class": { "cc_builtin": 101, "subagent": 1 }, "tool_result_tokens": 47738, "tool_time_ms": 11609, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.921541750000001, "cost_usd": 5.921541750000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 674697, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 138, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14818225, "tokens_cache_write": 159730, "tokens_input": 276, "tokens_output": 37229, "tool_calls": 121, "tool_calls_by_class": { "cc_builtin": 119, "subagent": 2 }, "tool_result_tokens": 52160, "tool_time_ms": 12785, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4921882, "cost_usd": 2.4921882, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 290386, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 93, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5778484, "tokens_cache_write": 89175, "tokens_input": 186, "tokens_output": 14869, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 63 }, "tool_result_tokens": 15505, "tool_time_ms": 3185, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5357076999999997, "cost_usd": 2.5357076999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 346314, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 98, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6406349, "tokens_cache_write": 60365, "tokens_input": 196, "tokens_output": 16735, "tool_calls": 68, "tool_calls_by_class": { "cc_builtin": 68 }, "tool_result_tokens": 17077, "tool_time_ms": 3276, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2951056000000003, "cost_usd": 2.2951056000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 288398, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 92, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5792542, "tokens_cache_write": 58196, "tokens_input": 184, "tokens_output": 13841, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 16986, "tool_time_ms": 3067, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3490228, "cost_usd": 2.3490228, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 299993, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 97, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6094396, "tokens_cache_write": 52762, "tokens_input": 194, "tokens_output": 13570, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 67 }, "tool_result_tokens": 14757, "tool_time_ms": 3220, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4181185000000003, "cost_usd": 2.4181185000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 364000, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 93, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6049865, "tokens_cache_write": 61606, "tokens_input": 186, "tokens_output": 15531, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 63 }, "tool_result_tokens": 18516, "tool_time_ms": 3461, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.65522695, "cost_usd": 2.65522695, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 362489, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 106, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6289027, "tokens_cache_write": 88277, "tokens_input": 212, "tokens_output": 17325, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 72, "subagent": 2 }, "tool_result_tokens": 20350, "tool_time_ms": 3976, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1354005999999996, "cost_usd": 2.1354005999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 339377, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 83, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5478052, "tokens_cache_write": 49627, "tokens_input": 166, "tokens_output": 12915, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 17372, "tool_time_ms": 3744, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.67837, "cost_usd": 2.67837, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 379136, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 99, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6797680, "tokens_cache_write": 63687, "tokens_input": 198, "tokens_output": 17090, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 69 }, "tool_result_tokens": 18489, "tool_time_ms": 4170, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4528306, "cost_usd": 2.4528306, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 319452, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 95, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6292782, "tokens_cache_write": 53956, "tokens_input": 190, "tokens_output": 16046, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 65 }, "tool_result_tokens": 18938, "tool_time_ms": 3677, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1853944, "cost_usd": 2.1853944, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 308181, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_bare", "llm_calls": 90, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5651318, "tokens_cache_write": 44309, "tokens_input": 180, "tokens_output": 14907, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 14005, "tool_time_ms": 4471, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.644866099999999, "cost_usd": 7.644866099999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 616919, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_bare", "llm_calls": 14, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10209883, "tokens_cache_write": 783120, "tokens_input": 600, "tokens_output": 55286, "tool_calls": 15, "tool_calls_by_class": { "cc_builtin": 13, "subagent": 2 }, "tool_result_tokens": 5177, "tool_time_ms": 394, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.087541300000002, "cost_usd": 8.087541300000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 802779, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_bare", "llm_calls": 16, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11134045, "tokens_cache_write": 769450, "tokens_input": 1059, "tokens_output": 51326, "tool_calls": 17, "tool_calls_by_class": { "cc_builtin": 16, "subagent": 1 }, "tool_result_tokens": 3955, "tool_time_ms": 967, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.882989449999999, "cost_usd": 7.882989449999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 639517, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_bare", "llm_calls": 26, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10608349, "tokens_cache_write": 745947, "tokens_input": 701, "tokens_output": 59904, "tool_calls": 25, "tool_calls_by_class": { "cc_builtin": 22, "subagent": 3 }, "tool_result_tokens": 6406, "tool_time_ms": 1006, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.437676499999999, "cost_usd": 7.437676499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 770251, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 9, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10226818, "tokens_cache_write": 714013, "tokens_input": 578, "tokens_output": 54395, "tool_calls": 6, "tool_calls_by_class": { "cc_builtin": 5, "subagent": 1 }, "tool_result_tokens": 2692, "tool_time_ms": 287, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.719937300000001, "cost_usd": 6.719937300000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 771713, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 11, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9297490, "tokens_cache_write": 627086, "tokens_input": 687, "tokens_output": 58231, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 10, "subagent": 1 }, "tool_result_tokens": 4573, "tool_time_ms": 437, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.290464149999998, "cost_usd": 8.290464149999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 643986, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 12, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11477536, "tokens_cache_write": 768974, "tokens_input": 648, "tokens_output": 57502, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 10, "subagent": 1 }, "tool_result_tokens": 3121, "tool_time_ms": 359, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.8952637999999995, "cost_usd": 7.8952637999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 861421, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_bare", "llm_calls": 17, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10809101, "tokens_cache_write": 740919, "tokens_input": 1276, "tokens_output": 62596, "tool_calls": 16, "tool_calls_by_class": { "cc_builtin": 14, "subagent": 2 }, "tool_result_tokens": 3863, "tool_time_ms": 585, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.173790449999998, "cost_usd": 9.173790449999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 850971, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 13, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14094561, "tokens_cache_write": 723267, "tokens_input": 2213, "tokens_output": 73056, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 12, "subagent": 2 }, "tool_result_tokens": 4425, "tool_time_ms": 661, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.2737065, "cost_usd": 9.2737065, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1122870, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 353, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13543999, "tokens_cache_write": 781992, "tokens_input": 817, "tokens_output": 70105, "tool_calls": 376, "tool_calls_by_class": { "cc_builtin": 352, "subagent": 24 }, "tool_result_tokens": 198755, "tool_time_ms": 170612, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.993374650000002, "cost_usd": 8.993374650000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1119708, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_bare", "llm_calls": 359, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13509448, "tokens_cache_write": 720216, "tokens_input": 830, "tokens_output": 59588, "tool_calls": 358, "tool_calls_by_class": { "cc_builtin": 334, "subagent": 24 }, "tool_result_tokens": 197790, "tool_time_ms": 36586, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.695332049999999, "cost_usd": 6.695332049999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 593693, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 262, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9046315, "tokens_cache_write": 669853, "tokens_input": 3416, "tokens_output": 48426, "tool_calls": 277, "tool_calls_by_class": { "cc_builtin": 257, "subagent": 20 }, "tool_result_tokens": 146950, "tool_time_ms": 12633, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.87064675, "cost_usd": 5.87064675, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 464167, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 228, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7930942, "tokens_cache_write": 555114, "tokens_input": 456, "tokens_output": 42735, "tool_calls": 208, "tool_calls_by_class": { "cc_builtin": 194, "subagent": 14 }, "tool_result_tokens": 127450, "tool_time_ms": 11705, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.950823399999999, "cost_usd": 5.950823399999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 473959, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 228, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7918078, "tokens_cache_write": 569691, "tokens_input": 456, "tokens_output": 44381, "tool_calls": 227, "tool_calls_by_class": { "cc_builtin": 213, "subagent": 14 }, "tool_result_tokens": 131316, "tool_time_ms": 10423, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.68641755, "cost_usd": 6.68641755, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 601519, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 252, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8450905, "tokens_cache_write": 681522, "tokens_input": 504, "tokens_output": 49148, "tool_calls": 277, "tool_calls_by_class": { "cc_builtin": 257, "subagent": 20 }, "tool_result_tokens": 167101, "tool_time_ms": 73068, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.543131849999999, "cost_usd": 6.543131849999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 586176, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 259, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8560067, "tokens_cache_write": 632837, "tokens_input": 518, "tokens_output": 45713, "tool_calls": 272, "tool_calls_by_class": { "cc_builtin": 254, "subagent": 18 }, "tool_result_tokens": 142649, "tool_time_ms": 12031, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.50904405, "cost_usd": 6.50904405, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 502073, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_bare", "llm_calls": 258, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8390489, "tokens_cache_write": 640876, "tokens_input": 516, "tokens_output": 45133, "tool_calls": 263, "tool_calls_by_class": { "cc_builtin": 241, "subagent": 22 }, "tool_result_tokens": 149662, "tool_time_ms": 13805, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.680000849999998, "cost_usd": 6.680000849999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 657417, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_bare", "llm_calls": 276, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9125343, "tokens_cache_write": 628358, "tokens_input": 552, "tokens_output": 48460, "tool_calls": 277, "tool_calls_by_class": { "cc_builtin": 253, "subagent": 24 }, "tool_result_tokens": 129903, "tool_time_ms": 12562, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.468158699999999, "cost_usd": 6.468158699999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 720683, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_bare", "llm_calls": 257, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9381336, "tokens_cache_write": 563573, "tokens_input": 514, "tokens_output": 51050, "tool_calls": 252, "tool_calls_by_class": { "cc_builtin": 236, "subagent": 16 }, "tool_result_tokens": 142766, "tool_time_ms": 12765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.065433899999999, "cost_usd": 6.065433899999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 561168, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 233, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7722283, "tokens_cache_write": 605738, "tokens_input": 466, "tokens_output": 44817, "tool_calls": 240, "tool_calls_by_class": { "cc_builtin": 219, "subagent": 21 }, "tool_result_tokens": 147166, "tool_time_ms": 10784, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cold", "arm_label": "CC Bare", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.0229349, "cost_usd": 7.0229349, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 488861, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_bare", "llm_calls": 280, "model": "claude-sonnet-5", "path": "runs/cold/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cold|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9171467, "tokens_cache_write": 689153, "tokens_input": 672, "tokens_output": 53366, "tool_calls": 287, "tool_calls_by_class": { "cc_builtin": 265, "subagent": 22 }, "tool_result_tokens": 145398, "tool_time_ms": 14297, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.05910345, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 837005, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 191, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 19788902, "tokens_cache_write": 1516249, "tokens_input": 1217, "tokens_output": 36737, "tool_calls": 238, "tool_calls_by_class": { "harness_builtin": 238 }, "tool_result_tokens": 62940, "tool_time_ms": 114381, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.7368594500000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 896527, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 243, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 22337627, "tokens_cache_write": 1057847, "tokens_input": 1338, "tokens_output": 35890, "tool_calls": 221, "tool_calls_by_class": { "harness_builtin": 221 }, "tool_result_tokens": 54473, "tool_time_ms": 97840, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.6969068, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 681657, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 184, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 18189688, "tokens_cache_write": 594332, "tokens_input": 1053, "tokens_output": 26794, "tool_calls": 162, "tool_calls_by_class": { "harness_builtin": 162 }, "tool_result_tokens": 57234, "tool_time_ms": 85062, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.4300006, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 736990, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 196, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 21551736, "tokens_cache_write": 899324, "tokens_input": 1137, "tokens_output": 29907, "tool_calls": 186, "tool_calls_by_class": { "harness_builtin": 186 }, "tool_result_tokens": 62892, "tool_time_ms": 78765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.5365898, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 654113, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 166, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 18267448, "tokens_cache_write": 448884, "tokens_input": 965, "tokens_output": 29555, "tool_calls": 150, "tool_calls_by_class": { "harness_builtin": 150 }, "tool_result_tokens": 50930, "tool_time_ms": 74988, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.77368975, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 711416, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 165, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 19042820, "tokens_cache_write": 1359631, "tokens_input": 1119, "tokens_output": 33750, "tool_calls": 206, "tool_calls_by_class": { "harness_builtin": 206 }, "tool_result_tokens": 56778, "tool_time_ms": 96768, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.17299255, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 547344, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 121, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11992863, "tokens_cache_write": 674629, "tokens_input": 870, "tokens_output": 25910, "tool_calls": 174, "tool_calls_by_class": { "harness_builtin": 174 }, "tool_result_tokens": 50990, "tool_time_ms": 72532, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.3738968000000007, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 785633, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 215, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 21716148, "tokens_cache_write": 835912, "tokens_input": 1217, "tokens_output": 31235, "tool_calls": 195, "tool_calls_by_class": { "harness_builtin": 195 }, "tool_result_tokens": 49527, "tool_time_ms": 83334, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.7762404, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 828452, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 225, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 23826394, "tokens_cache_write": 980588, "tokens_input": 1266, "tokens_output": 33320, "tool_calls": 203, "tool_calls_by_class": { "harness_builtin": 203 }, "tool_result_tokens": 48894, "tool_time_ms": 92957, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.11126505, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 978970, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 255, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 23836843, "tokens_cache_write": 1227639, "tokens_input": 1407, "tokens_output": 38325, "tool_calls": 242, "tool_calls_by_class": { "harness_builtin": 242 }, "tool_result_tokens": 51616, "tool_time_ms": 118015, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.7482619500000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 408009, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 168, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10409837, "tokens_cache_write": 483529, "tokens_input": 972, "tokens_output": 20379, "tool_calls": 136, "tool_calls_by_class": { "harness_builtin": 136 }, "tool_result_tokens": 42843, "tool_time_ms": 52655, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.54759905, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 479931, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 190, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 16581968, "tokens_cache_write": 593677, "tokens_input": 1141, "tokens_output": 29233, "tool_calls": 158, "tool_calls_by_class": { "harness_builtin": 158 }, "tool_result_tokens": 56097, "tool_time_ms": 55256, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6202374499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 401165, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 142, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11129842, "tokens_cache_write": 306301, "tokens_input": 967, "tokens_output": 24682, "tool_calls": 158, "tool_calls_by_class": { "harness_builtin": 158 }, "tool_result_tokens": 51980, "tool_time_ms": 60016, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.91022555, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 335490, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 135, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13074193, "tokens_cache_write": 392505, "tokens_input": 915, "tokens_output": 22252, "tool_calls": 144, "tool_calls_by_class": { "harness_builtin": 144 }, "tool_result_tokens": 41765, "tool_time_ms": 48488, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.40945055, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1352828, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 61, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2317698, "tokens_cache_write": 84427, "tokens_input": 457, "tokens_output": 14338, "tool_calls": 30, "tool_calls_by_class": { "subagent": 30 }, "tool_result_tokens": 617222, "tool_time_ms": 1152568, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.4428418, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 438962, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 183, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 14544243, "tokens_cache_write": 683982, "tokens_input": 1140, "tokens_output": 26460, "tool_calls": 151, "tool_calls_by_class": { "harness_builtin": 151 }, "tool_result_tokens": 46646, "tool_time_ms": 53056, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.0705953, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 395502, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 159, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 12433513, "tokens_cache_write": 567264, "tokens_input": 994, "tokens_output": 23434, "tool_calls": 141, "tool_calls_by_class": { "harness_builtin": 141 }, "tool_result_tokens": 46129, "tool_time_ms": 51244, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.75061275, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 414845, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 171, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 17619555, "tokens_cache_write": 689845, "tokens_input": 1041, "tokens_output": 25062, "tool_calls": 158, "tool_calls_by_class": { "harness_builtin": 158 }, "tool_result_tokens": 46473, "tool_time_ms": 53567, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.82824855, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 433330, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 180, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 19230243, "tokens_cache_write": 627677, "tokens_input": 1088, "tokens_output": 23908, "tool_calls": 146, "tool_calls_by_class": { "harness_builtin": 146 }, "tool_result_tokens": 65580, "tool_time_ms": 55255, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.81157445, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 439723, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 173, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 18170812, "tokens_cache_write": 689269, "tokens_input": 1067, "tokens_output": 26368, "tool_calls": 141, "tool_calls_by_class": { "harness_builtin": 141 }, "tool_result_tokens": 45064, "tool_time_ms": 47108, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.2377566, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1429758, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 356, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10645381, "tokens_cache_write": 694782, "tokens_input": 2086, "tokens_output": 60531, "tool_calls": 414, "tool_calls_by_class": { "harness_builtin": 413, "subagent": 1 }, "tool_result_tokens": 191475, "tool_time_ms": 383766, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.33354435, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1593951, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 412, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13039851, "tokens_cache_write": 549069, "tokens_input": 2368, "tokens_output": 68171, "tool_calls": 420, "tool_calls_by_class": { "harness_builtin": 419, "subagent": 1 }, "tool_result_tokens": 205403, "tool_time_ms": 394143, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8916774000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1868537, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 493, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 17799074, "tokens_cache_write": 567756, "tokens_input": 2820, "tokens_output": 79851, "tool_calls": 505, "tool_calls_by_class": { "harness_builtin": 505 }, "tool_result_tokens": 220989, "tool_time_ms": 404911, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.23048095, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1350982, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 411, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 12574552, "tokens_cache_write": 521451, "tokens_input": 2352, "tokens_output": 63772, "tool_calls": 421, "tool_calls_by_class": { "harness_builtin": 421 }, "tool_result_tokens": 189773, "tool_time_ms": 246807, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.9395177000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1732484, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 376, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11017367, "tokens_cache_write": 450116, "tokens_input": 2156, "tokens_output": 54596, "tool_calls": 369, "tool_calls_by_class": { "harness_builtin": 367, "subagent": 2 }, "tool_result_tokens": 337349, "tool_time_ms": 590715, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.2236951, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1483706, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 433, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13332846, "tokens_cache_write": 444858, "tokens_input": 2483, "tokens_output": 66371, "tool_calls": 451, "tool_calls_by_class": { "harness_builtin": 451 }, "tool_result_tokens": 184877, "tool_time_ms": 270040, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.2159606, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1641397, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 397, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 12862981, "tokens_cache_write": 465286, "tokens_input": 2310, "tokens_output": 69149, "tool_calls": 427, "tool_calls_by_class": { "harness_builtin": 426, "subagent": 1 }, "tool_result_tokens": 213280, "tool_time_ms": 481294, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.9572566999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 3163944, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 383, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10990067, "tokens_cache_write": 453712, "tokens_input": 2095, "tokens_output": 57803, "tool_calls": 381, "tool_calls_by_class": { "harness_builtin": 379, "subagent": 2 }, "tool_result_tokens": 259583, "tool_time_ms": 1981046, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.1098476, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1501197, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 388, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11932986, "tokens_cache_write": 484560, "tokens_input": 2219, "tokens_output": 61726, "tool_calls": 419, "tool_calls_by_class": { "harness_builtin": 417, "subagent": 2 }, "tool_result_tokens": 251052, "tool_time_ms": 426538, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.3890729000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1561301, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 430, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13403104, "tokens_cache_write": 570862, "tokens_input": 2490, "tokens_output": 66539, "tool_calls": 449, "tool_calls_by_class": { "harness_builtin": 449 }, "tool_result_tokens": 181169, "tool_time_ms": 350227, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.04577675, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1687775, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 304, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7712920, "tokens_cache_write": 836663, "tokens_input": 1841, "tokens_output": 45363, "tool_calls": 297, "tool_calls_by_class": { "harness_builtin": 296, "subagent": 1 }, "tool_result_tokens": 203513, "tool_time_ms": 1016704, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.713524, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 793645, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 306, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8155970, "tokens_cache_write": 523700, "tokens_input": 1857, "tokens_output": 48289, "tool_calls": 298, "tool_calls_by_class": { "harness_builtin": 298 }, "tool_result_tokens": 151264, "tool_time_ms": 149620, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8183794, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1048264, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 337, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9451219, "tokens_cache_write": 509286, "tokens_input": 1995, "tokens_output": 46931, "tool_calls": 313, "tool_calls_by_class": { "harness_builtin": 312, "subagent": 1 }, "tool_result_tokens": 234137, "tool_time_ms": 271654, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.64246825, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 754927, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 321, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8388250, "tokens_cache_write": 457413, "tokens_input": 1907, "tokens_output": 45994, "tool_calls": 299, "tool_calls_by_class": { "harness_builtin": 299 }, "tool_result_tokens": 148831, "tool_time_ms": 139863, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.7363479500000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 809148, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 307, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8560137, "tokens_cache_write": 519081, "tokens_input": 1843, "tokens_output": 45928, "tool_calls": 289, "tool_calls_by_class": { "harness_builtin": 288, "subagent": 1 }, "tool_result_tokens": 194754, "tool_time_ms": 177899, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.89100525, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 896632, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 352, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9737690, "tokens_cache_write": 511713, "tokens_input": 2110, "tokens_output": 55097, "tool_calls": 367, "tool_calls_by_class": { "harness_builtin": 367 }, "tool_result_tokens": 166555, "tool_time_ms": 160513, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8995664, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1536647, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 352, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10162944, "tokens_cache_write": 498288, "tokens_input": 2082, "tokens_output": 51666, "tool_calls": 333, "tool_calls_by_class": { "harness_builtin": 332, "subagent": 1 }, "tool_result_tokens": 167513, "tool_time_ms": 193417, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8720297499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 938045, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 342, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9529265, "tokens_cache_write": 537661, "tokens_input": 2027, "tokens_output": 49000, "tool_calls": 325, "tool_calls_by_class": { "harness_builtin": 324, "subagent": 1 }, "tool_result_tokens": 201573, "tool_time_ms": 201894, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8300475, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 884449, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 327, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9184595, "tokens_cache_write": 519704, "tokens_input": 1968, "tokens_output": 51998, "tool_calls": 321, "tool_calls_by_class": { "harness_builtin": 320, "subagent": 1 }, "tool_result_tokens": 178793, "tool_time_ms": 190510, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.78491045, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 824031, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 322, "model": "claude-haiku-4-5", "path": "runs/cursor/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8582477, "tokens_cache_write": 542983, "tokens_input": 1964, "tokens_output": 49194, "tool_calls": 341, "tool_calls_by_class": { "harness_builtin": 340, "subagent": 1 }, "tool_result_tokens": 189433, "tool_time_ms": 181426, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.5374255, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 951632, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 86, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7485131, "tokens_cache_write": 145184, "tokens_input": 172, "tokens_output": 35464, "tool_calls": 85, "tool_calls_by_class": { "harness_builtin": 85 }, "tool_result_tokens": 36011, "tool_time_ms": 172519, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.99113875, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1230184, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 122, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11838150, "tokens_cache_write": 141467, "tokens_input": 244, "tokens_output": 47467, "tool_calls": 125, "tool_calls_by_class": { "harness_builtin": 125 }, "tool_result_tokens": 41807, "tool_time_ms": 127919, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.9137569999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1205853, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 104, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11106459, "tokens_cache_write": 153926, "tokens_input": 208, "tokens_output": 55898, "tool_calls": 96, "tool_calls_by_class": { "harness_builtin": 96 }, "tool_result_tokens": 45588, "tool_time_ms": 80742, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 6.580685999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1017862, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 100, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9528177, "tokens_cache_write": 129558, "tokens_input": 202, "tokens_output": 40234, "tool_calls": 95, "tool_calls_by_class": { "harness_builtin": 95 }, "tool_result_tokens": 35647, "tool_time_ms": 104350, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.206331, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1079739, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 109, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11939332, "tokens_cache_write": 156984, "tokens_input": 218, "tokens_output": 50177, "tool_calls": 123, "tool_calls_by_class": { "harness_builtin": 123 }, "tool_result_tokens": 47787, "tool_time_ms": 147266, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 6.918427749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 823305, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9643863, "tokens_cache_write": 165541, "tokens_input": 188, "tokens_output": 42437, "tool_calls": 100, "tool_calls_by_class": { "harness_builtin": 100 }, "tool_result_tokens": 35722, "tool_time_ms": 78820, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.48123925, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1005480, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 111, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11518546, "tokens_cache_write": 391065, "tokens_input": 222, "tokens_output": 51068, "tool_calls": 127, "tool_calls_by_class": { "harness_builtin": 127 }, "tool_result_tokens": 47135, "tool_time_ms": 92787, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.357927, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1112261, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 115, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 12425134, "tokens_cache_write": 151556, "tokens_input": 232, "tokens_output": 47879, "tool_calls": 106, "tool_calls_by_class": { "harness_builtin": 106 }, "tool_result_tokens": 48370, "tool_time_ms": 114607, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.708513749999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1160172, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 130, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 16429415, "tokens_cache_write": 184681, "tokens_input": 260, "tokens_output": 53530, "tool_calls": 136, "tool_calls_by_class": { "harness_builtin": 136 }, "tool_result_tokens": 52986, "tool_time_ms": 105651, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.875351000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 800517, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 108, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10410797, "tokens_cache_write": 254218, "tokens_input": 218, "tokens_output": 43200, "tool_calls": 114, "tool_calls_by_class": { "harness_builtin": 114 }, "tool_result_tokens": 37114, "tool_time_ms": 90556, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.9191485, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 870097, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 112, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6963882, "tokens_cache_write": 110742, "tokens_input": 224, "tokens_output": 29758, "tool_calls": 79, "tool_calls_by_class": { "harness_builtin": 79 }, "tool_result_tokens": 28027, "tool_time_ms": 69042, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.57035275, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1008005, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 114, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8201113, "tokens_cache_write": 96949, "tokens_input": 228, "tokens_output": 34509, "tool_calls": 81, "tool_calls_by_class": { "harness_builtin": 81 }, "tool_result_tokens": 31054, "tool_time_ms": 71323, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.35547675, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 813017, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7727151, "tokens_cache_write": 102621, "tokens_input": 214, "tokens_output": 33978, "tool_calls": 88, "tool_calls_by_class": { "harness_builtin": 88 }, "tool_result_tokens": 32876, "tool_time_ms": 55788, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.408101, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1001611, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 116, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7888432, "tokens_cache_write": 93592, "tokens_input": 232, "tokens_output": 35111, "tool_calls": 84, "tool_calls_by_class": { "harness_builtin": 84 }, "tool_result_tokens": 28112, "tool_time_ms": 66310, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.2142807499999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 765967, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7200759, "tokens_cache_write": 129613, "tokens_input": 204, "tokens_output": 32112, "tool_calls": 83, "tool_calls_by_class": { "harness_builtin": 83 }, "tool_result_tokens": 34283, "tool_time_ms": 54233, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.6134770000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 697755, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 109, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6828799, "tokens_cache_write": 82218, "tokens_input": 218, "tokens_output": 27365, "tool_calls": 77, "tool_calls_by_class": { "harness_builtin": 77 }, "tool_result_tokens": 25458, "tool_time_ms": 53288, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.2339375, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 824557, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 109, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7449470, "tokens_cache_write": 98034, "tokens_input": 218, "tokens_output": 35816, "tool_calls": 88, "tool_calls_by_class": { "harness_builtin": 88 }, "tool_result_tokens": 29356, "tool_time_ms": 78552, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.1361225, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 838434, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 104, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7416040, "tokens_cache_write": 102534, "tokens_input": 208, "tokens_output": 31449, "tool_calls": 85, "tool_calls_by_class": { "harness_builtin": 85 }, "tool_result_tokens": 34152, "tool_time_ms": 64428, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.6995135, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 844785, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 111, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8282487, "tokens_cache_write": 108880, "tokens_input": 224, "tokens_output": 35066, "tool_calls": 80, "tool_calls_by_class": { "harness_builtin": 80 }, "tool_result_tokens": 35713, "tool_time_ms": 55374, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.935209250000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 881406, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 124, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8655651, "tokens_cache_write": 102915, "tokens_input": 248, "tokens_output": 38517, "tool_calls": 99, "tool_calls_by_class": { "harness_builtin": 99 }, "tool_result_tokens": 31036, "tool_time_ms": 76544, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 11.13311725, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1799419, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 229, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7581517, "tokens_cache_write": 815723, "tokens_input": 458, "tokens_output": 89672, "tool_calls": 282, "tool_calls_by_class": { "harness_builtin": 282 }, "tool_result_tokens": 146751, "tool_time_ms": 267603, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.983575499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1628998, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 205, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6636976, "tokens_cache_write": 581598, "tokens_input": 410, "tokens_output": 81122, "tool_calls": 259, "tool_calls_by_class": { "harness_builtin": 259 }, "tool_result_tokens": 120674, "tool_time_ms": 302110, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.959825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1556215, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 211, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6645960, "tokens_cache_write": 749932, "tokens_input": 424, "tokens_output": 77906, "tool_calls": 241, "tool_calls_by_class": { "harness_builtin": 241 }, "tool_result_tokens": 125600, "tool_time_ms": 151968, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.40365, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1672346, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 217, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7194660, "tokens_cache_write": 752732, "tokens_input": 434, "tokens_output": 83983, "tool_calls": 273, "tool_calls_by_class": { "harness_builtin": 272, "subagent": 1 }, "tool_result_tokens": 184713, "tool_time_ms": 257699, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.60379725, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1470272, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 204, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6795027, "tokens_cache_write": 512267, "tokens_input": 408, "tokens_output": 80103, "tool_calls": 254, "tool_calls_by_class": { "harness_builtin": 254 }, "tool_result_tokens": 123655, "tool_time_ms": 135682, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.5285175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1702676, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 217, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7253075, "tokens_cache_write": 768080, "tokens_input": 436, "tokens_output": 83972, "tool_calls": 266, "tool_calls_by_class": { "harness_builtin": 265, "subagent": 1 }, "tool_result_tokens": 211866, "tool_time_ms": 328199, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.04739825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1797402, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 202, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6877419, "tokens_cache_write": 583167, "tokens_input": 404, "tokens_output": 78475, "tool_calls": 245, "tool_calls_by_class": { "harness_builtin": 244, "subagent": 1 }, "tool_result_tokens": 369942, "tool_time_ms": 306766, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.388956250000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1604401, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 204, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6948820, "tokens_cache_write": 464241, "tokens_input": 408, "tokens_output": 80440, "tool_calls": 252, "tool_calls_by_class": { "harness_builtin": 251, "subagent": 1 }, "tool_result_tokens": 172990, "tool_time_ms": 271880, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.97656325, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1543051, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 222, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7365424, "tokens_cache_write": 675213, "tokens_input": 444, "tokens_output": 82862, "tool_calls": 267, "tool_calls_by_class": { "harness_builtin": 267 }, "tool_result_tokens": 136051, "tool_time_ms": 159615, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.85374725, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1552531, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 210, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7269212, "tokens_cache_write": 524921, "tokens_input": 422, "tokens_output": 77451, "tool_calls": 263, "tool_calls_by_class": { "harness_builtin": 263 }, "tool_result_tokens": 125288, "tool_time_ms": 187248, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 12.857222, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1516780, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 260, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7299074, "tokens_cache_write": 1215896, "tokens_input": 522, "tokens_output": 64229, "tool_calls": 242, "tool_calls_by_class": { "harness_builtin": 242 }, "tool_result_tokens": 119554, "tool_time_ms": 145301, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.899407499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1363103, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 253, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7245085, "tokens_cache_write": 744652, "tokens_input": 508, "tokens_output": 64810, "tool_calls": 241, "tool_calls_by_class": { "harness_builtin": 241 }, "tool_result_tokens": 128988, "tool_time_ms": 117285, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.634532499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1339725, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 236, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6569445, "tokens_cache_write": 784752, "tokens_input": 472, "tokens_output": 57710, "tool_calls": 220, "tool_calls_by_class": { "harness_builtin": 220 }, "tool_result_tokens": 106581, "tool_time_ms": 140346, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.664264, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1486043, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 261, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7367583, "tokens_cache_write": 857046, "tokens_input": 522, "tokens_output": 64853, "tool_calls": 243, "tool_calls_by_class": { "harness_builtin": 243 }, "tool_result_tokens": 123147, "tool_time_ms": 156575, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.16661625, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1518528, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 256, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7588905, "tokens_cache_write": 597751, "tokens_input": 514, "tokens_output": 65346, "tool_calls": 236, "tool_calls_by_class": { "harness_builtin": 236 }, "tool_result_tokens": 115075, "tool_time_ms": 163277, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.069928, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 2926067, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 241, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6689471, "tokens_cache_write": 827806, "tokens_input": 476, "tokens_output": 61961, "tool_calls": 222, "tool_calls_by_class": { "harness_builtin": 221, "subagent": 1 }, "tool_result_tokens": 185641, "tool_time_ms": 1636107, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.92340125, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1488147, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 234, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6947910, "tokens_cache_write": 623441, "tokens_input": 468, "tokens_output": 62024, "tool_calls": 219, "tool_calls_by_class": { "harness_builtin": 218, "subagent": 1 }, "tool_result_tokens": 219315, "tool_time_ms": 205084, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.2161175, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1498065, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 252, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7427500, "tokens_cache_write": 456198, "tokens_input": 506, "tokens_output": 65944, "tool_calls": 250, "tool_calls_by_class": { "harness_builtin": 250 }, "tool_result_tokens": 119817, "tool_time_ms": 168210, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.62066375, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1375174, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 238, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6918685, "tokens_cache_write": 579365, "tokens_input": 478, "tokens_output": 61516, "tool_calls": 219, "tool_calls_by_class": { "harness_builtin": 219 }, "tool_result_tokens": 104725, "tool_time_ms": 136699, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.358854000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1563426, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 250, "model": "claude-opus-5", "path": "runs/cursor/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7291458, "tokens_cache_write": 497428, "tokens_input": 500, "tokens_output": 64068, "tool_calls": 245, "tool_calls_by_class": { "harness_builtin": 245 }, "tool_result_tokens": 166149, "tool_time_ms": 152988, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.1116663, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1035357, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 124, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 15457114, "tokens_cache_write": 205883, "tokens_input": 248, "tokens_output": 50504, "tool_calls": 121, "tool_calls_by_class": { "harness_builtin": 121 }, "tool_result_tokens": 39019, "tool_time_ms": 95155, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.0361112, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 784407, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 112, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11710066, "tokens_cache_write": 138996, "tokens_input": 224, "tokens_output": 34616, "tool_calls": 104, "tool_calls_by_class": { "harness_builtin": 104 }, "tool_result_tokens": 36833, "tool_time_ms": 59596, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.202187800000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1106034, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 138, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 16787664, "tokens_cache_write": 160858, "tokens_input": 280, "tokens_output": 44195, "tool_calls": 131, "tool_calls_by_class": { "harness_builtin": 131 }, "tool_result_tokens": 35924, "tool_time_ms": 106171, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8357124999999996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 632600, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 108, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10570240, "tokens_cache_write": 154309, "tokens_input": 216, "tokens_output": 33546, "tool_calls": 96, "tool_calls_by_class": { "harness_builtin": 96 }, "tool_result_tokens": 46388, "tool_time_ms": 54622, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.0575323, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1175710, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 142, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 15658369, "tokens_cache_write": 160409, "tokens_input": 288, "tokens_output": 52426, "tool_calls": 119, "tool_calls_by_class": { "harness_builtin": 119 }, "tool_result_tokens": 40826, "tool_time_ms": 129131, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.733294, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1031108, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 141, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 14537530, "tokens_cache_write": 144300, "tokens_input": 284, "tokens_output": 46447, "tool_calls": 127, "tool_calls_by_class": { "harness_builtin": 127 }, "tool_result_tokens": 41073, "tool_time_ms": 111319, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.4123757, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 932379, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 120, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13114991, "tokens_cache_write": 151383, "tokens_input": 240, "tokens_output": 41044, "tool_calls": 120, "tool_calls_by_class": { "harness_builtin": 120 }, "tool_result_tokens": 41131, "tool_time_ms": 116395, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.2206966999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1102665, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 152, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 16493156, "tokens_cache_write": 189311, "tokens_input": 304, "tokens_output": 44818, "tool_calls": 130, "tool_calls_by_class": { "harness_builtin": 130 }, "tool_result_tokens": 42878, "tool_time_ms": 74584, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.7339258000000006, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1368524, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 114, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13659584, "tokens_cache_write": 169126, "tokens_input": 232, "tokens_output": 57873, "tool_calls": 124, "tool_calls_by_class": { "harness_builtin": 120, "subagent": 4 }, "tool_result_tokens": 156143, "tool_time_ms": 318245, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.8941502000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1020461, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 151, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 15380356, "tokens_cache_write": 147042, "tokens_input": 302, "tokens_output": 44987, "tool_calls": 128, "tool_calls_by_class": { "harness_builtin": 128 }, "tool_result_tokens": 34106, "tool_time_ms": 63745, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8141001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 496136, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 109, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6499183, "tokens_cache_write": 104439, "tokens_input": 218, "tokens_output": 25273, "tool_calls": 75, "tool_calls_by_class": { "harness_builtin": 75 }, "tool_result_tokens": 24757, "tool_time_ms": 32269, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6858555, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 736039, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 107, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6249425, "tokens_cache_write": 72929, "tokens_input": 214, "tokens_output": 25322, "tool_calls": 72, "tool_calls_by_class": { "harness_builtin": 72 }, "tool_result_tokens": 21549, "tool_time_ms": 30207, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.4281047, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 533329, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 93, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5145831, "tokens_cache_write": 66745, "tokens_input": 188, "tokens_output": 23170, "tool_calls": 62, "tool_calls_by_class": { "harness_builtin": 62 }, "tool_result_tokens": 19213, "tool_time_ms": 23681, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.0436490000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 470536, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 112, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7811705, "tokens_cache_write": 91500, "tokens_input": 224, "tokens_output": 25211, "tool_calls": 80, "tool_calls_by_class": { "harness_builtin": 80 }, "tool_result_tokens": 34060, "tool_time_ms": 24647, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6819643, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 450827, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 111, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6436539, "tokens_cache_write": 70541, "tokens_input": 222, "tokens_output": 21786, "tool_calls": 79, "tool_calls_by_class": { "harness_builtin": 79 }, "tool_result_tokens": 22134, "tool_time_ms": 27746, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8089754000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 528932, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 112, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6922487, "tokens_cache_write": 78256, "tokens_input": 224, "tokens_output": 22839, "tool_calls": 80, "tool_calls_by_class": { "harness_builtin": 80 }, "tool_result_tokens": 24978, "tool_time_ms": 34290, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6273128, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 488233, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 101, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6200844, "tokens_cache_write": 71124, "tokens_input": 202, "tokens_output": 20893, "tool_calls": 70, "tool_calls_by_class": { "harness_builtin": 70 }, "tool_result_tokens": 22913, "tool_time_ms": 25299, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.7669728, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 484762, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 111, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6671249, "tokens_cache_write": 74502, "tokens_input": 224, "tokens_output": 24602, "tool_calls": 79, "tool_calls_by_class": { "harness_builtin": 79 }, "tool_result_tokens": 22518, "tool_time_ms": 32869, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.624401, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 755858, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 107, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6138180, "tokens_cache_write": 68714, "tokens_input": 210, "tokens_output": 22456, "tool_calls": 74, "tool_calls_by_class": { "harness_builtin": 74 }, "tool_result_tokens": 21391, "tool_time_ms": 35241, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8652258, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 532204, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 115, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7110854, "tokens_cache_write": 78114, "tokens_input": 230, "tokens_output": 24731, "tool_calls": 83, "tool_calls_by_class": { "harness_builtin": 83 }, "tool_result_tokens": 24311, "tool_time_ms": 28096, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.0320261, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1738230, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 250, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8188773, "tokens_cache_write": 671827, "tokens_input": 502, "tokens_output": 71370, "tool_calls": 225, "tool_calls_by_class": { "harness_builtin": 225 }, "tool_result_tokens": 92728, "tool_time_ms": 156640, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.8770121, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1586683, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 247, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8041388, "tokens_cache_write": 622021, "tokens_input": 496, "tokens_output": 71269, "tool_calls": 236, "tool_calls_by_class": { "harness_builtin": 236 }, "tool_result_tokens": 91201, "tool_time_ms": 165497, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.9861562, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1865549, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 243, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8988401, "tokens_cache_write": 535624, "tokens_input": 488, "tokens_output": 84844, "tool_calls": 241, "tool_calls_by_class": { "harness_builtin": 240, "subagent": 1 }, "tool_result_tokens": 149258, "tool_time_ms": 279873, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.7444277, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2162683, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 268, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9515866, "tokens_cache_write": 393849, "tokens_input": 536, "tokens_output": 85556, "tool_calls": 259, "tool_calls_by_class": { "harness_builtin": 259 }, "tool_result_tokens": 105631, "tool_time_ms": 215727, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.0984943, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1532391, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 243, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8110329, "tokens_cache_write": 307905, "tokens_input": 488, "tokens_output": 70569, "tool_calls": 244, "tool_calls_by_class": { "harness_builtin": 244 }, "tool_result_tokens": 91404, "tool_time_ms": 156713, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.2048303000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1671959, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 245, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8248804, "tokens_cache_write": 325423, "tokens_input": 496, "tokens_output": 74052, "tool_calls": 234, "tool_calls_by_class": { "harness_builtin": 234 }, "tool_result_tokens": 92904, "tool_time_ms": 169882, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7826529000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1546358, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 230, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7450997, "tokens_cache_write": 266591, "tokens_input": 468, "tokens_output": 62504, "tool_calls": 210, "tool_calls_by_class": { "harness_builtin": 210 }, "tool_result_tokens": 71497, "tool_time_ms": 210197, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.3191815, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1825833, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 235, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8588050, "tokens_cache_write": 340055, "tokens_input": 472, "tokens_output": 75049, "tool_calls": 232, "tool_calls_by_class": { "harness_builtin": 231, "subagent": 1 }, "tool_result_tokens": 155092, "tool_time_ms": 267740, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9397642, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1578372, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 226, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7490981, "tokens_cache_write": 302460, "tokens_input": 454, "tokens_output": 68451, "tool_calls": 218, "tool_calls_by_class": { "harness_builtin": 217, "subagent": 1 }, "tool_result_tokens": 129423, "tool_time_ms": 217199, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.7802983, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1974300, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 276, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9450584, "tokens_cache_write": 422055, "tokens_input": 552, "tokens_output": 83394, "tool_calls": 265, "tool_calls_by_class": { "harness_builtin": 264, "subagent": 1 }, "tool_result_tokens": 137503, "tool_time_ms": 269475, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.6143909, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1256669, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 248, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7150107, "tokens_cache_write": 656359, "tokens_input": 496, "tokens_output": 54248, "tool_calls": 191, "tool_calls_by_class": { "harness_builtin": 191 }, "tool_result_tokens": 104785, "tool_time_ms": 96388, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9196155000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1269142, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 255, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7834640, "tokens_cache_write": 310083, "tokens_input": 510, "tokens_output": 57646, "tool_calls": 203, "tool_calls_by_class": { "harness_builtin": 203 }, "tool_result_tokens": 104130, "tool_time_ms": 91803, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7369642, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1356402, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 248, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7518716, "tokens_cache_write": 278246, "tokens_input": 498, "tokens_output": 53661, "tool_calls": 191, "tool_calls_by_class": { "harness_builtin": 191 }, "tool_result_tokens": 108369, "tool_time_ms": 94695, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.6369121, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1326945, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 240, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7220923, "tokens_cache_write": 249139, "tokens_input": 480, "tokens_output": 56892, "tool_calls": 196, "tool_calls_by_class": { "harness_builtin": 195, "subagent": 1 }, "tool_result_tokens": 134229, "tool_time_ms": 195730, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7680239999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1150531, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 237, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7323605, "tokens_cache_write": 291470, "tokens_input": 474, "tokens_output": 57368, "tool_calls": 198, "tool_calls_by_class": { "harness_builtin": 198 }, "tool_result_tokens": 112877, "tool_time_ms": 77526, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9810979, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1335937, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 259, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7906427, "tokens_cache_write": 318545, "tokens_input": 520, "tokens_output": 60241, "tool_calls": 213, "tool_calls_by_class": { "harness_builtin": 213 }, "tool_result_tokens": 119634, "tool_time_ms": 92313, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8889389, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1393965, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 270, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8084982, "tokens_cache_write": 258597, "tokens_input": 540, "tokens_output": 62437, "tool_calls": 218, "tool_calls_by_class": { "harness_builtin": 218 }, "tool_result_tokens": 94061, "tool_time_ms": 121453, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9005659, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1322323, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 265, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8047642, "tokens_cache_write": 279775, "tokens_input": 530, "tokens_output": 59054, "tool_calls": 209, "tool_calls_by_class": { "harness_builtin": 209 }, "tool_result_tokens": 101189, "tool_time_ms": 85936, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.5726627000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1203218, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 247, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7368036, "tokens_cache_write": 237499, "tokens_input": 494, "tokens_output": 50432, "tool_calls": 185, "tool_calls_by_class": { "harness_builtin": 185 }, "tool_result_tokens": 91166, "tool_time_ms": 91124, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8709084000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1149040, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 241, "model": "claude-sonnet-5", "path": "runs/cursor/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7562577, "tokens_cache_write": 330762, "tokens_input": 484, "tokens_output": 53052, "tool_calls": 191, "tool_calls_by_class": { "harness_builtin": 191 }, "tool_result_tokens": 123665, "tool_time_ms": 80812, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.2348926000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1094017, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 167, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 14379808, "tokens_cache_write": 0, "tokens_input": 430662, "tokens_output": 57440, "tool_calls": 295, "tool_calls_by_class": { "harness_builtin": 295 }, "tool_result_tokens": 157831, "tool_time_ms": 358574, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7989987000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 737405, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 128, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 12467776, "tokens_cache_write": 0, "tokens_input": 385982, "tokens_output": 44981, "tool_calls": 217, "tool_calls_by_class": { "harness_builtin": 217 }, "tool_result_tokens": 102201, "tool_time_ms": 240893, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.1679362, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1240523, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 152, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13768256, "tokens_cache_write": 0, "tokens_input": 561750, "tokens_output": 53364, "tool_calls": 259, "tool_calls_by_class": { "harness_builtin": 259 }, "tool_result_tokens": 131936, "tool_time_ms": 406572, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.5136103, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 945499, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 117, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11477344, "tokens_cache_write": 0, "tokens_input": 209848, "tokens_output": 45287, "tool_calls": 196, "tool_calls_by_class": { "harness_builtin": 196 }, "tool_result_tokens": 100144, "tool_time_ms": 388914, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8969681, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 790469, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 127, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13362848, "tokens_cache_write": 0, "tokens_input": 214992, "tokens_output": 46761, "tool_calls": 196, "tool_calls_by_class": { "harness_builtin": 196 }, "tool_result_tokens": 108748, "tool_time_ms": 143844, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.4771258000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 913737, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 127, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11396224, "tokens_cache_write": 0, "tokens_input": 172717, "tokens_output": 44609, "tool_calls": 207, "tool_calls_by_class": { "harness_builtin": 207 }, "tool_result_tokens": 89160, "tool_time_ms": 211917, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.589033, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1130270, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 129, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11953920, "tokens_cache_write": 0, "tokens_input": 169708, "tokens_output": 45358, "tool_calls": 216, "tool_calls_by_class": { "harness_builtin": 216 }, "tool_result_tokens": 98998, "tool_time_ms": 446865, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9982235, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1103130, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 147, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13415360, "tokens_cache_write": 0, "tokens_input": 362093, "tokens_output": 53642, "tool_calls": 257, "tool_calls_by_class": { "harness_builtin": 257 }, "tool_result_tokens": 129071, "tool_time_ms": 363998, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.0501912, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1090856, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 144, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13263776, "tokens_cache_write": 0, "tokens_input": 536542, "tokens_output": 51666, "tool_calls": 235, "tool_calls_by_class": { "harness_builtin": 235 }, "tool_result_tokens": 125444, "tool_time_ms": 356546, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.5087029999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 918011, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 128, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11393760, "tokens_cache_write": 0, "tokens_input": 247582, "tokens_output": 42464, "tool_calls": 215, "tool_calls_by_class": { "harness_builtin": 215 }, "tool_result_tokens": 99613, "tool_time_ms": 286381, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9714840000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 597713, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 162, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13505920, "tokens_cache_write": 0, "tokens_input": 345560, "tokens_output": 39008, "tool_calls": 272, "tool_calls_by_class": { "harness_builtin": 272 }, "tool_result_tokens": 106031, "tool_time_ms": 153910, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8334275, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 490858, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 143, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13059680, "tokens_cache_write": 0, "tokens_input": 257148, "tokens_output": 37167, "tool_calls": 240, "tool_calls_by_class": { "harness_builtin": 240 }, "tool_result_tokens": 118355, "tool_time_ms": 85564, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.4777463999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 527110, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 146, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11378592, "tokens_cache_write": 0, "tokens_input": 228231, "tokens_output": 35165, "tool_calls": 227, "tool_calls_by_class": { "harness_builtin": 227 }, "tool_result_tokens": 102996, "tool_time_ms": 87501, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.3717706, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 517360, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 138, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10843168, "tokens_cache_write": 0, "tokens_input": 241344, "tokens_output": 32986, "tool_calls": 219, "tool_calls_by_class": { "harness_builtin": 219 }, "tool_result_tokens": 106224, "tool_time_ms": 95529, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.5575779, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 543347, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 140, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 11735072, "tokens_cache_write": 0, "tokens_input": 234327, "tokens_output": 37360, "tool_calls": 235, "tool_calls_by_class": { "harness_builtin": 235 }, "tool_result_tokens": 111590, "tool_time_ms": 78526, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9901532, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 473607, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 168, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13306496, "tokens_cache_write": 0, "tokens_input": 470028, "tokens_output": 37536, "tool_calls": 262, "tool_calls_by_class": { "harness_builtin": 262 }, "tool_result_tokens": 109620, "tool_time_ms": 89982, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8818649000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 552984, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 155, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13420032, "tokens_cache_write": 0, "tokens_input": 209227, "tokens_output": 37298, "tool_calls": 245, "tool_calls_by_class": { "harness_builtin": 245 }, "tool_result_tokens": 123644, "tool_time_ms": 81609, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8275118000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 462054, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 153, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 13058784, "tokens_cache_write": 0, "tokens_input": 252605, "tokens_output": 35781, "tool_calls": 240, "tool_calls_by_class": { "harness_builtin": 240 }, "tool_result_tokens": 114144, "tool_time_ms": 75564, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.3236393, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 478785, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 138, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10542624, "tokens_cache_write": 0, "tokens_input": 251994, "tokens_output": 35647, "tool_calls": 215, "tool_calls_by_class": { "harness_builtin": 215 }, "tool_result_tokens": 107196, "tool_time_ms": 80277, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.4176821000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 462226, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 145, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10979008, "tokens_cache_write": 0, "tokens_input": 278411, "tokens_output": 33070, "tool_calls": 215, "tool_calls_by_class": { "harness_builtin": 215 }, "tool_result_tokens": 104459, "tool_time_ms": 79976, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.2508873, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2023380, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 297, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6372384, "tokens_cache_write": 0, "tokens_input": 1493516, "tokens_output": 91861, "tool_calls": 543, "tool_calls_by_class": { "harness_builtin": 542, "subagent": 1 }, "tool_result_tokens": 390490, "tool_time_ms": 1085060, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.7768676, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1286579, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 262, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4716608, "tokens_cache_write": 0, "tokens_input": 1253502, "tokens_output": 82718, "tool_calls": 483, "tool_calls_by_class": { "harness_builtin": 482, "subagent": 1 }, "tool_result_tokens": 331827, "tool_time_ms": 461009, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8895955, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1679287, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 262, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4506080, "tokens_cache_write": 0, "tokens_input": 1568659, "tokens_output": 81620, "tool_calls": 510, "tool_calls_by_class": { "harness_builtin": 508, "subagent": 2 }, "tool_result_tokens": 450887, "tool_time_ms": 757940, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8618002000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1243176, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 272, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5112096, "tokens_cache_write": 0, "tokens_input": 1271222, "tokens_output": 81508, "tool_calls": 518, "tool_calls_by_class": { "harness_builtin": 518 }, "tool_result_tokens": 353927, "tool_time_ms": 387582, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8572, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1858311, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 257, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4615680, "tokens_cache_write": 0, "tokens_input": 1462818, "tokens_output": 81062, "tool_calls": 486, "tool_calls_by_class": { "harness_builtin": 485, "subagent": 1 }, "tool_result_tokens": 343511, "tool_time_ms": 987229, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8122674, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1356849, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 263, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4830272, "tokens_cache_write": 0, "tokens_input": 1321111, "tokens_output": 74263, "tool_calls": 487, "tool_calls_by_class": { "harness_builtin": 486, "subagent": 1 }, "tool_result_tokens": 599826, "tool_time_ms": 518154, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.5429563000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1627761, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 237, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3873344, "tokens_cache_write": 0, "tokens_input": 1143255, "tokens_output": 78664, "tool_calls": 439, "tool_calls_by_class": { "harness_builtin": 435, "subagent": 4 }, "tool_result_tokens": 460233, "tool_time_ms": 834118, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8086637000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1339963, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 263, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4669856, "tokens_cache_write": 0, "tokens_input": 1381790, "tokens_output": 73519, "tool_calls": 496, "tool_calls_by_class": { "harness_builtin": 495, "subagent": 1 }, "tool_result_tokens": 346413, "tool_time_ms": 523866, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8875635000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1236255, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 261, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4980640, "tokens_cache_write": 0, "tokens_input": 1389786, "tokens_output": 78617, "tool_calls": 487, "tool_calls_by_class": { "harness_builtin": 486, "subagent": 1 }, "tool_result_tokens": 389070, "tool_time_ms": 488774, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.0783723000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1388462, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 280, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5760384, "tokens_cache_write": 0, "tokens_input": 1426581, "tokens_output": 85202, "tool_calls": 498, "tool_calls_by_class": { "harness_builtin": 497, "subagent": 1 }, "tool_result_tokens": 404422, "tool_time_ms": 499234, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.762016, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1195831, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 282, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4498720, "tokens_cache_write": 0, "tokens_input": 1382384, "tokens_output": 68432, "tool_calls": 468, "tool_calls_by_class": { "harness_builtin": 468 }, "tool_result_tokens": 312207, "tool_time_ms": 252090, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.7646098000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1164708, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 284, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4562784, "tokens_cache_write": 0, "tokens_input": 1371291, "tokens_output": 66563, "tool_calls": 488, "tool_calls_by_class": { "harness_builtin": 487, "subagent": 1 }, "tool_result_tokens": 434260, "tool_time_ms": 325076, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8112007, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1308745, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 281, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4531616, "tokens_cache_write": 0, "tokens_input": 1485995, "tokens_output": 64752, "tool_calls": 462, "tool_calls_by_class": { "harness_builtin": 461, "subagent": 1 }, "tool_result_tokens": 411479, "tool_time_ms": 337106, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6728863, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1195584, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 271, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4161824, "tokens_cache_write": 0, "tokens_input": 1354473, "tokens_output": 65314, "tool_calls": 438, "tool_calls_by_class": { "harness_builtin": 437, "subagent": 1 }, "tool_result_tokens": 335445, "tool_time_ms": 346312, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.7324556000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1161553, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 283, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4416768, "tokens_cache_write": 0, "tokens_input": 1362834, "tokens_output": 67074, "tool_calls": 468, "tool_calls_by_class": { "harness_builtin": 467, "subagent": 1 }, "tool_result_tokens": 373882, "tool_time_ms": 344735, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.826735, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1213347, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 300, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4735680, "tokens_cache_write": 0, "tokens_input": 1436893, "tokens_output": 64461, "tool_calls": 510, "tool_calls_by_class": { "harness_builtin": 510 }, "tool_result_tokens": 338699, "tool_time_ms": 350558, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6404158999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1089175, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 268, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3982112, "tokens_cache_write": 0, "tokens_input": 1377317, "tokens_output": 62134, "tool_calls": 432, "tool_calls_by_class": { "harness_builtin": 432 }, "tool_result_tokens": 296240, "tool_time_ms": 242964, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6292456000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1143071, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 278, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4357568, "tokens_cache_write": 0, "tokens_input": 1205494, "tokens_output": 61994, "tool_calls": 465, "tool_calls_by_class": { "harness_builtin": 464, "subagent": 1 }, "tool_result_tokens": 374368, "tool_time_ms": 369950, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6510818999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1150646, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 280, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4317632, "tokens_cache_write": 0, "tokens_input": 1259966, "tokens_output": 63029, "tool_calls": 453, "tool_calls_by_class": { "harness_builtin": 452, "subagent": 1 }, "tool_result_tokens": 382142, "tool_time_ms": 278747, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6399571000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1120709, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 274, "model": "composer-2.5", "path": "runs/cursor/composer-2.5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|composer-2.5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4170848, "tokens_cache_write": 0, "tokens_input": 1289655, "tokens_output": 64384, "tool_calls": 453, "tool_calls_by_class": { "harness_builtin": 452, "subagent": 1 }, "tool_result_tokens": 341652, "tool_time_ms": 278980, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.20023666, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 376161, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 90, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7365988, "tokens_cache_write": 148058, "tokens_input": 270, "tokens_output": 13207, "tool_calls": 87, "tool_calls_by_class": { "harness_builtin": 87 }, "tool_result_tokens": 77509, "tool_time_ms": 52057, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.18880299, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 402459, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 67, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6598137, "tokens_cache_write": 170409, "tokens_input": 204, "tokens_output": 11831, "tool_calls": 81, "tool_calls_by_class": { "harness_builtin": 81 }, "tool_result_tokens": 71483, "tool_time_ms": 64066, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.23364254, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 463677, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 115, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9162897, "tokens_cache_write": 133616, "tokens_input": 345, "tokens_output": 14093, "tool_calls": 93, "tool_calls_by_class": { "harness_builtin": 93 }, "tool_result_tokens": 449994, "tool_time_ms": 61683, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.20152478000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 472929, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 87, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7363114, "tokens_cache_write": 140406, "tokens_input": 261, "tokens_output": 15924, "tool_calls": 83, "tool_calls_by_class": { "harness_builtin": 83 }, "tool_result_tokens": 152236, "tool_time_ms": 71451, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.19617787, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 459517, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 87, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7082166, "tokens_cache_write": 143455, "tokens_input": 264, "tokens_output": 15515, "tool_calls": 99, "tool_calls_by_class": { "harness_builtin": 99 }, "tool_result_tokens": 279874, "tool_time_ms": 99625, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.1778026, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 408223, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 90, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6577375, "tokens_cache_write": 121238, "tokens_input": 270, "tokens_output": 13243, "tool_calls": 97, "tool_calls_by_class": { "harness_builtin": 97 }, "tool_result_tokens": 201850, "tool_time_ms": 51788, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.21461482, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 466428, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 100, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7859731, "tokens_cache_write": 143612, "tokens_input": 300, "tokens_output": 17881, "tool_calls": 79, "tool_calls_by_class": { "harness_builtin": 79 }, "tool_result_tokens": 451132, "tool_time_ms": 104313, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.26957331, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 459986, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 109, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 10700918, "tokens_cache_write": 159875, "tokens_input": 327, "tokens_output": 12934, "tool_calls": 88, "tool_calls_by_class": { "harness_builtin": 88 }, "tool_result_tokens": 369948, "tool_time_ms": 69594, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.1650777, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 316102, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 72, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5851290, "tokens_cache_write": 132474, "tokens_input": 219, "tokens_output": 12408, "tool_calls": 93, "tool_calls_by_class": { "harness_builtin": 93 }, "tool_result_tokens": 70912, "tool_time_ms": 46233, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.20786891999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 444922, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 72, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7492181, "tokens_cache_write": 171758, "tokens_input": 219, "tokens_output": 12535, "tool_calls": 77, "tool_calls_by_class": { "harness_builtin": 77 }, "tool_result_tokens": 78960, "tool_time_ms": 123761, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.18867272, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 311723, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 73, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6819941, "tokens_cache_write": 158002, "tokens_input": 219, "tokens_output": 10608, "tool_calls": 89, "tool_calls_by_class": { "harness_builtin": 89 }, "tool_result_tokens": 54854, "tool_time_ms": 34287, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.14405949, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 341293, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 102, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5408607, "tokens_cache_write": 95535, "tokens_input": 306, "tokens_output": 9952, "tool_calls": 70, "tool_calls_by_class": { "harness_builtin": 70 }, "tool_result_tokens": 51947, "tool_time_ms": 29538, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.15339923, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 352017, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 102, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5872284, "tokens_cache_write": 98243, "tokens_input": 306, "tokens_output": 9443, "tool_calls": 80, "tool_calls_by_class": { "harness_builtin": 80 }, "tool_result_tokens": 47654, "tool_time_ms": 32302, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.18451053999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 344414, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 98, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6993037, "tokens_cache_write": 124076, "tokens_input": 294, "tokens_output": 11310, "tool_calls": 81, "tool_calls_by_class": { "harness_builtin": 81 }, "tool_result_tokens": 44238, "tool_time_ms": 30439, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.21772909000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 334043, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 107, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8662677, "tokens_cache_write": 132151, "tokens_input": 321, "tokens_output": 9478, "tool_calls": 84, "tool_calls_by_class": { "harness_builtin": 84 }, "tool_result_tokens": 49762, "tool_time_ms": 25260, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.14094378, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 329182, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 93, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5096129, "tokens_cache_write": 105476, "tokens_input": 279, "tokens_output": 10497, "tool_calls": 88, "tool_calls_by_class": { "harness_builtin": 88 }, "tool_result_tokens": 58268, "tool_time_ms": 27141, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.15913649, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 533824, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 105, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6019557, "tokens_cache_write": 104411, "tokens_input": 315, "tokens_output": 10483, "tool_calls": 90, "tool_calls_by_class": { "harness_builtin": 90 }, "tool_result_tokens": 53650, "tool_time_ms": 59268, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.2152833, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 283671, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 85, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7892750, "tokens_cache_write": 177146, "tokens_input": 255, "tokens_output": 10909, "tool_calls": 75, "tool_calls_by_class": { "harness_builtin": 75 }, "tool_result_tokens": 54867, "tool_time_ms": 23595, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.19540739999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 284804, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 87, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7369755, "tokens_cache_write": 145002, "tokens_input": 261, "tokens_output": 9758, "tool_calls": 78, "tool_calls_by_class": { "harness_builtin": 78 }, "tool_result_tokens": 51438, "tool_time_ms": 24072, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.20048052, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 330108, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 88, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7693891, "tokens_cache_write": 139630, "tokens_input": 264, "tokens_output": 9702, "tool_calls": 80, "tool_calls_by_class": { "harness_builtin": 80 }, "tool_result_tokens": 50776, "tool_time_ms": 22089, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.31877697, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1452666, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 225, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4589761, "tokens_cache_write": 719635, "tokens_input": 675, "tokens_output": 39115, "tool_calls": 295, "tool_calls_by_class": { "harness_builtin": 291, "subagent": 4 }, "tool_result_tokens": 883380, "tool_time_ms": 731679, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.31486996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1250940, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 217, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4395988, "tokens_cache_write": 732716, "tokens_input": 654, "tokens_output": 36367, "tool_calls": 287, "tool_calls_by_class": { "harness_builtin": 284, "subagent": 3 }, "tool_result_tokens": 657590, "tool_time_ms": 551048, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.36278439, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1223799, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 256, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5724712, "tokens_cache_write": 780787, "tokens_input": 771, "tokens_output": 44116, "tool_calls": 332, "tool_calls_by_class": { "harness_builtin": 331, "subagent": 1 }, "tool_result_tokens": 355269, "tool_time_ms": 304725, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.34029661, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1131178, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 213, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4971168, "tokens_cache_write": 771637, "tokens_input": 648, "tokens_output": 39862, "tool_calls": 313, "tool_calls_by_class": { "harness_builtin": 309, "subagent": 4 }, "tool_result_tokens": 396872, "tool_time_ms": 423246, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.39959906, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1149243, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 270, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6793138, "tokens_cache_write": 840990, "tokens_input": 810, "tokens_output": 44439, "tool_calls": 345, "tool_calls_by_class": { "harness_builtin": 344, "subagent": 1 }, "tool_result_tokens": 586667, "tool_time_ms": 266127, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.31072886, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1166448, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 217, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4176458, "tokens_cache_write": 734506, "tokens_input": 654, "tokens_output": 36202, "tool_calls": 289, "tool_calls_by_class": { "harness_builtin": 286, "subagent": 3 }, "tool_result_tokens": 384723, "tool_time_ms": 381597, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.34297953999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1052181, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 242, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5028987, "tokens_cache_write": 770524, "tokens_input": 732, "tokens_output": 41352, "tool_calls": 325, "tool_calls_by_class": { "harness_builtin": 323, "subagent": 2 }, "tool_result_tokens": 454679, "tool_time_ms": 286609, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.32524473, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1620214, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 225, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4728659, "tokens_cache_write": 753023, "tokens_input": 675, "tokens_output": 35234, "tool_calls": 287, "tool_calls_by_class": { "harness_builtin": 284, "subagent": 3 }, "tool_result_tokens": 668917, "tool_time_ms": 903074, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.37136027, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1122213, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 265, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5821121, "tokens_cache_write": 811309, "tokens_input": 801, "tokens_output": 43292, "tool_calls": 360, "tool_calls_by_class": { "harness_builtin": 359, "subagent": 1 }, "tool_result_tokens": 333139, "tool_time_ms": 257629, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.32863129, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 959142, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 231, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4824742, "tokens_cache_write": 746561, "tokens_input": 693, "tokens_output": 37798, "tool_calls": 302, "tool_calls_by_class": { "harness_builtin": 300, "subagent": 2 }, "tool_result_tokens": 444733, "tool_time_ms": 247198, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.43195046, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1065331, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 311, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5755978, "tokens_cache_write": 1073454, "tokens_input": 939, "tokens_output": 40233, "tool_calls": 367, "tool_calls_by_class": { "harness_builtin": 367 }, "tool_result_tokens": 319164, "tool_time_ms": 170419, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.41415169, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1127105, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 300, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5168777, "tokens_cache_write": 1051395, "tokens_input": 903, "tokens_output": 39789, "tool_calls": 354, "tool_calls_by_class": { "harness_builtin": 354 }, "tool_result_tokens": 328752, "tool_time_ms": 186824, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.38070253, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1132694, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 274, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4644924, "tokens_cache_write": 978445, "tokens_input": 822, "tokens_output": 35857, "tool_calls": 314, "tool_calls_by_class": { "harness_builtin": 311, "subagent": 3 }, "tool_result_tokens": 328337, "tool_time_ms": 362724, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.39640756, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1297047, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 276, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4841758, "tokens_cache_write": 1015076, "tokens_input": 831, "tokens_output": 38031, "tool_calls": 326, "tool_calls_by_class": { "harness_builtin": 322, "subagent": 4 }, "tool_result_tokens": 582433, "tool_time_ms": 404453, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.40980158, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1194633, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 312, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5644769, "tokens_cache_write": 990964, "tokens_input": 942, "tokens_output": 40814, "tool_calls": 358, "tool_calls_by_class": { "harness_builtin": 357, "subagent": 1 }, "tool_result_tokens": 574104, "tool_time_ms": 241678, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.38889418, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1219246, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 271, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4541629, "tokens_cache_write": 1017116, "tokens_input": 813, "tokens_output": 36350, "tool_calls": 331, "tool_calls_by_class": { "harness_builtin": 329, "subagent": 2 }, "tool_result_tokens": 582208, "tool_time_ms": 279125, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.42703126, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1270667, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 335, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6460823, "tokens_cache_write": 982344, "tokens_input": 1008, "tokens_output": 43356, "tool_calls": 385, "tool_calls_by_class": { "harness_builtin": 385 }, "tool_result_tokens": 475503, "tool_time_ms": 225553, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.38487727, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 945723, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 278, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4777401, "tokens_cache_write": 974461, "tokens_input": 834, "tokens_output": 37956, "tool_calls": 315, "tool_calls_by_class": { "harness_builtin": 315 }, "tool_result_tokens": 333165, "tool_time_ms": 148414, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.39502334, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1260162, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 308, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5140087, "tokens_cache_write": 973824, "tokens_input": 912, "tokens_output": 40486, "tool_calls": 346, "tool_calls_by_class": { "harness_builtin": 346 }, "tool_result_tokens": 274758, "tool_time_ms": 231753, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.41688668, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1178954, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 301, "model": "gpt-5.6-luna", "path": "runs/cursor/gpt-5.6-luna/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-luna|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5432429, "tokens_cache_write": 1039642, "tokens_input": 912, "tokens_output": 40121, "tool_calls": 340, "tool_calls_by_class": { "harness_builtin": 339, "subagent": 1 }, "tool_result_tokens": 336810, "tool_time_ms": 253053, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.9800906, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2355916, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 40, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1334539, "tokens_cache_write": 59103, "tokens_input": 120, "tokens_output": 7514, "tool_calls": 19, "tool_calls_by_class": { "subagent": 19 }, "tool_result_tokens": 1175831, "tool_time_ms": 2005089, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.082295, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1737944, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 44, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1548955, "tokens_cache_write": 61585, "tokens_input": 132, "tokens_output": 7713, "tool_calls": 24, "tool_calls_by_class": { "harness_builtin": 6, "subagent": 18 }, "tool_result_tokens": 953279, "tool_time_ms": 1424853, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.0218867999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1708597, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 43, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1391987, "tokens_cache_write": 60124, "tokens_input": 123, "tokens_output": 8199, "tool_calls": 22, "tool_calls_by_class": { "harness_builtin": 3, "subagent": 19 }, "tool_result_tokens": 948511, "tool_time_ms": 1459810, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9482196000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 483840, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 74, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5160254, "tokens_cache_write": 115182, "tokens_input": 222, "tokens_output": 15366, "tool_calls": 87, "tool_calls_by_class": { "harness_builtin": 87 }, "tool_result_tokens": 56395, "tool_time_ms": 51669, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.0003566, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1633312, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 43, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1409564, "tokens_cache_write": 55687, "tokens_input": 129, "tokens_output": 7879, "tool_calls": 22, "tool_calls_by_class": { "harness_builtin": 2, "subagent": 20 }, "tool_result_tokens": 892775, "tool_time_ms": 1369656, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8203924000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1590204, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 58, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2837416, "tokens_cache_write": 91290, "tokens_input": 174, "tokens_output": 11414, "tool_calls": 101, "tool_calls_by_class": { "harness_builtin": 85, "subagent": 16 }, "tool_result_tokens": 958086, "tool_time_ms": 1264645, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.892126, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1585085, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 40, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1216215, "tokens_cache_write": 51768, "tokens_input": 120, "tokens_output": 7316, "tool_calls": 18, "tool_calls_by_class": { "subagent": 18 }, "tool_result_tokens": 1222786, "tool_time_ms": 1310086, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.9448585999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1440985, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 41, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1314779, "tokens_cache_write": 53171, "tokens_input": 123, "tokens_output": 7630, "tool_calls": 20, "tool_calls_by_class": { "subagent": 20 }, "tool_result_tokens": 979868, "tool_time_ms": 1215522, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.7165996000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1357932, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 48, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2610084, "tokens_cache_write": 99450, "tokens_input": 144, "tokens_output": 8737, "tool_calls": 58, "tool_calls_by_class": { "harness_builtin": 43, "subagent": 15 }, "tool_result_tokens": 1052537, "tool_time_ms": 1344171, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.9430914000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1592283, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 42, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1284231, "tokens_cache_write": 52063, "tokens_input": 126, "tokens_output": 8429, "tool_calls": 21, "tool_calls_by_class": { "harness_builtin": 1, "subagent": 20 }, "tool_result_tokens": 1002228, "tool_time_ms": 1377846, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.3284578, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 567435, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 74, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3990162, "tokens_cache_write": 108085, "tokens_input": 222, "tokens_output": 9554, "tool_calls": 80, "tool_calls_by_class": { "harness_builtin": 77, "subagent": 3 }, "tool_result_tokens": 236906, "tool_time_ms": 191613, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.498898, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 624142, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 72, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4419560, "tokens_cache_write": 113162, "tokens_input": 216, "tokens_output": 8220, "tool_calls": 71, "tool_calls_by_class": { "harness_builtin": 68, "subagent": 3 }, "tool_result_tokens": 260168, "tool_time_ms": 252574, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.4939814, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 741990, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 68, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4206101, "tokens_cache_write": 125117, "tokens_input": 204, "tokens_output": 9257, "tool_calls": 78, "tool_calls_by_class": { "harness_builtin": 74, "subagent": 4 }, "tool_result_tokens": 317591, "tool_time_ms": 347800, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.6992298, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 549292, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 71, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4685432, "tokens_cache_write": 124505, "tokens_input": 213, "tokens_output": 10084, "tool_calls": 79, "tool_calls_by_class": { "harness_builtin": 79 }, "tool_result_tokens": 54131, "tool_time_ms": 63968, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.3715078000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 382571, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 68, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4039767, "tokens_cache_write": 114293, "tokens_input": 204, "tokens_output": 9166, "tool_calls": 76, "tool_calls_by_class": { "harness_builtin": 76 }, "tool_result_tokens": 66950, "tool_time_ms": 40159, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.497197, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 563194, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 83, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4464815, "tokens_cache_write": 101859, "tokens_input": 249, "tokens_output": 10049, "tool_calls": 91, "tool_calls_by_class": { "harness_builtin": 91 }, "tool_result_tokens": 52041, "tool_time_ms": 59872, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7064611999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 564096, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 85, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4820303, "tokens_cache_write": 110596, "tokens_input": 255, "tokens_output": 11217, "tool_calls": 91, "tool_calls_by_class": { "harness_builtin": 91 }, "tool_result_tokens": 61777, "tool_time_ms": 75303, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.0330596, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 590050, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 71, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3496369, "tokens_cache_write": 93840, "tokens_input": 213, "tokens_output": 8223, "tool_calls": 66, "tool_calls_by_class": { "harness_builtin": 62, "subagent": 4 }, "tool_result_tokens": 236351, "tool_time_ms": 242364, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.3219108, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 657248, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4047422, "tokens_cache_write": 103166, "tokens_input": 243, "tokens_output": 9307, "tool_calls": 81, "tool_calls_by_class": { "harness_builtin": 78, "subagent": 3 }, "tool_result_tokens": 232211, "tool_time_ms": 312163, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.5511964000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 841506, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 74, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4338271, "tokens_cache_write": 122572, "tokens_input": 222, "tokens_output": 10107, "tool_calls": 76, "tool_calls_by_class": { "harness_builtin": 72, "subagent": 4 }, "tool_result_tokens": 294462, "tool_time_ms": 444601, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9441968, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2852133, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 94, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1151087, "tokens_cache_write": 438854, "tokens_input": 258, "tokens_output": 14423, "tool_calls": 121, "tool_calls_by_class": { "harness_builtin": 102, "subagent": 19 }, "tool_result_tokens": 1005906, "tool_time_ms": 2412433, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.1034184, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1933872, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 108, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1433331, "tokens_cache_write": 422842, "tokens_input": 324, "tokens_output": 20729, "tool_calls": 135, "tool_calls_by_class": { "harness_builtin": 115, "subagent": 20 }, "tool_result_tokens": 995361, "tool_time_ms": 1650604, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.0005943999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2355831, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 104, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1376666, "tokens_cache_write": 408752, "tokens_input": 312, "tokens_output": 20246, "tool_calls": 123, "tool_calls_by_class": { "harness_builtin": 103, "subagent": 20 }, "tool_result_tokens": 982062, "tool_time_ms": 1801136, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9384056, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2081344, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 103, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1357549, "tokens_cache_write": 405110, "tokens_input": 309, "tokens_output": 18430, "tool_calls": 131, "tool_calls_by_class": { "harness_builtin": 111, "subagent": 20 }, "tool_result_tokens": 1033308, "tool_time_ms": 1560157, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.6187899999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2046890, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 86, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 985120, "tokens_cache_write": 385790, "tokens_input": 243, "tokens_output": 14741, "tool_calls": 107, "tool_calls_by_class": { "harness_builtin": 86, "subagent": 21 }, "tool_result_tokens": 1097241, "tool_time_ms": 1633573, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.1131404, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1883798, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 102, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1370291, "tokens_cache_write": 442136, "tokens_input": 306, "tokens_output": 17656, "tool_calls": 133, "tool_calls_by_class": { "harness_builtin": 111, "subagent": 22 }, "tool_result_tokens": 1038631, "tool_time_ms": 1508431, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7413146, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1977362, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 97, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1218284, "tokens_cache_write": 388637, "tokens_input": 279, "tokens_output": 15485, "tool_calls": 117, "tool_calls_by_class": { "harness_builtin": 97, "subagent": 20 }, "tool_result_tokens": 988463, "tool_time_ms": 1527330, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.0912334, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1942573, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 108, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1505631, "tokens_cache_write": 420325, "tokens_input": 324, "tokens_output": 19303, "tool_calls": 128, "tool_calls_by_class": { "harness_builtin": 110, "subagent": 18 }, "tool_result_tokens": 1011041, "tool_time_ms": 1346460, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.2889106, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1720378, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 102, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1382904, "tokens_cache_write": 465977, "tokens_input": 306, "tokens_output": 20232, "tool_calls": 152, "tool_calls_by_class": { "harness_builtin": 137, "subagent": 15 }, "tool_result_tokens": 917040, "tool_time_ms": 1205713, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.777335, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1805200, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 93, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1169310, "tokens_cache_write": 396247, "tokens_input": 279, "tokens_output": 16363, "tool_calls": 113, "tool_calls_by_class": { "harness_builtin": 95, "subagent": 18 }, "tool_result_tokens": 959153, "tool_time_ms": 1323057, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.297644399999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1577282, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 174, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2244466, "tokens_cache_write": 600822, "tokens_input": 522, "tokens_output": 19683, "tool_calls": 144, "tool_calls_by_class": { "harness_builtin": 126, "subagent": 18 }, "tool_result_tokens": 763365, "tool_time_ms": 845497, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.5447814, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1553677, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 186, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2559911, "tokens_cache_write": 618337, "tokens_input": 558, "tokens_output": 21345, "tool_calls": 156, "tool_calls_by_class": { "harness_builtin": 140, "subagent": 16 }, "tool_result_tokens": 705408, "tool_time_ms": 772048, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.1093782, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1762909, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 164, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2188513, "tokens_cache_write": 566733, "tokens_input": 492, "tokens_output": 19917, "tool_calls": 133, "tool_calls_by_class": { "harness_builtin": 112, "subagent": 21 }, "tool_result_tokens": 914553, "tool_time_ms": 964242, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.3078394, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1647360, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 177, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2253651, "tokens_cache_write": 596807, "tokens_input": 531, "tokens_output": 21011, "tool_calls": 148, "tool_calls_by_class": { "harness_builtin": 131, "subagent": 17 }, "tool_result_tokens": 759319, "tool_time_ms": 900050, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.3681622, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1788891, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 163, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2131853, "tokens_cache_write": 623229, "tokens_input": 489, "tokens_output": 19866, "tool_calls": 144, "tool_calls_by_class": { "harness_builtin": 127, "subagent": 17 }, "tool_result_tokens": 886328, "tool_time_ms": 936116, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 5.0961492, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1602298, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 199, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2812303, "tokens_cache_write": 699488, "tokens_input": 597, "tokens_output": 23570, "tool_calls": 186, "tool_calls_by_class": { "harness_builtin": 172, "subagent": 14 }, "tool_result_tokens": 621601, "tool_time_ms": 732875, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.0049154, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1765908, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 166, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2110546, "tokens_cache_write": 547349, "tokens_input": 498, "tokens_output": 21098, "tool_calls": 143, "tool_calls_by_class": { "harness_builtin": 121, "subagent": 22 }, "tool_result_tokens": 735864, "tool_time_ms": 991741, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.1015362, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1651902, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 165, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2052453, "tokens_cache_write": 580599, "tokens_input": 495, "tokens_output": 18779, "tool_calls": 116, "tool_calls_by_class": { "harness_builtin": 97, "subagent": 19 }, "tool_result_tokens": 788359, "tool_time_ms": 915394, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.5051418000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1664902, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 171, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2204037, "tokens_cache_write": 636055, "tokens_input": 513, "tokens_output": 22060, "tool_calls": 158, "tool_calls_by_class": { "harness_builtin": 140, "subagent": 18 }, "tool_result_tokens": 783360, "tool_time_ms": 935523, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 4.8528504, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1643942, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 193, "model": "gpt-5.6-sol", "path": "runs/cursor/gpt-5.6-sol/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-sol|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2637536, "tokens_cache_write": 666504, "tokens_input": 579, "tokens_output": 23150, "tool_calls": 179, "tool_calls_by_class": { "harness_builtin": 164, "subagent": 15 }, "tool_result_tokens": 658997, "tool_time_ms": 821383, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.4283634, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2608198, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 44, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1228847, "tokens_cache_write": 48068, "tokens_input": 132, "tokens_output": 5180, "tool_calls": 22, "tool_calls_by_class": { "harness_builtin": 3, "subagent": 19 }, "tool_result_tokens": 923860, "tool_time_ms": 2230358, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.3649195, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 632639, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 77, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4785795, "tokens_cache_write": 108077, "tokens_input": 234, "tokens_output": 11425, "tool_calls": 88, "tool_calls_by_class": { "harness_builtin": 88 }, "tool_result_tokens": 129607, "tool_time_ms": 112283, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.5196501, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1292195, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 63, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5254153, "tokens_cache_write": 146375, "tokens_input": 189, "tokens_output": 8542, "tool_calls": 69, "tool_calls_by_class": { "harness_builtin": 63, "subagent": 6 }, "tool_result_tokens": 379481, "tool_time_ms": 797642, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.3956462000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 571161, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 9, "family": "harness_bare", "llm_calls": 70, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4821176, "tokens_cache_write": 124574, "tokens_input": 210, "tokens_output": 9963, "tool_calls": 72, "tool_calls_by_class": { "harness_builtin": 72 }, "tool_result_tokens": 50234, "tool_time_ms": 59606, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6831572, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 452704, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5894751, "tokens_cache_write": 129714, "tokens_input": 243, "tokens_output": 14953, "tool_calls": 102, "tool_calls_by_class": { "harness_builtin": 102 }, "tool_result_tokens": 205457, "tool_time_ms": 69186, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.505493, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1716268, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 56, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4958455, "tokens_cache_write": 163204, "tokens_input": 168, "tokens_output": 8788, "tool_calls": 71, "tool_calls_by_class": { "harness_builtin": 65, "subagent": 6 }, "tool_result_tokens": 501836, "tool_time_ms": 1449512, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.4330497, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2326681, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 46, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1264756, "tokens_cache_write": 45337, "tokens_input": 138, "tokens_output": 5540, "tool_calls": 24, "tool_calls_by_class": { "harness_builtin": 3, "subagent": 21 }, "tool_result_tokens": 958744, "tool_time_ms": 2082420, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.4272952, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2036087, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 44, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1261766, "tokens_cache_write": 44268, "tokens_input": 132, "tokens_output": 5334, "tool_calls": 22, "tool_calls_by_class": { "harness_builtin": 2, "subagent": 20 }, "tool_result_tokens": 817745, "tool_time_ms": 1738019, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.4369323, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 486380, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 83, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 5288009, "tokens_cache_write": 98841, "tokens_input": 240, "tokens_output": 10979, "tool_calls": 95, "tool_calls_by_class": { "harness_builtin": 95 }, "tool_result_tokens": 59722, "tool_time_ms": 103063, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 0.4334582, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2286641, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 43, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1226976, "tokens_cache_write": 47690, "tokens_input": 129, "tokens_output": 5715, "tool_calls": 22, "tool_calls_by_class": { "harness_builtin": 2, "subagent": 20 }, "tool_result_tokens": 888809, "tool_time_ms": 2068217, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.4017198, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 597538, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 70, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4873954, "tokens_cache_write": 129890, "tokens_input": 210, "tokens_output": 8482, "tool_calls": 64, "tool_calls_by_class": { "harness_builtin": 58, "subagent": 6 }, "tool_result_tokens": 408323, "tool_time_ms": 339574, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.1437786, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 746748, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 74, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3936933, "tokens_cache_write": 101752, "tokens_input": 222, "tokens_output": 8464, "tool_calls": 70, "tool_calls_by_class": { "harness_builtin": 65, "subagent": 5 }, "tool_result_tokens": 296405, "tool_time_ms": 482555, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.2210923, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 743076, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 73, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4222614, "tokens_cache_write": 109547, "tokens_input": 219, "tokens_output": 8522, "tool_calls": 67, "tool_calls_by_class": { "harness_builtin": 63, "subagent": 4 }, "tool_result_tokens": 251207, "tool_time_ms": 481892, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.0504526, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1080817, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 68, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3602463, "tokens_cache_write": 94160, "tokens_input": 204, "tokens_output": 7846, "tool_calls": 63, "tool_calls_by_class": { "harness_builtin": 53, "subagent": 10 }, "tool_result_tokens": 436137, "tool_time_ms": 839014, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.3025324, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 951921, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 79, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4485697, "tokens_cache_write": 112998, "tokens_input": 237, "tokens_output": 10202, "tool_calls": 83, "tool_calls_by_class": { "harness_builtin": 76, "subagent": 7 }, "tool_result_tokens": 337305, "tool_time_ms": 673474, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.0281, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 823258, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 66, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3461495, "tokens_cache_write": 97154, "tokens_input": 198, "tokens_output": 7710, "tool_calls": 64, "tool_calls_by_class": { "harness_builtin": 59, "subagent": 5 }, "tool_result_tokens": 309650, "tool_time_ms": 591686, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.1640405, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 382933, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 81, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4011370, "tokens_cache_write": 98461, "tokens_input": 243, "tokens_output": 9594, "tool_calls": 76, "tool_calls_by_class": { "harness_builtin": 75, "subagent": 1 }, "tool_result_tokens": 93850, "tool_time_ms": 111428, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.2869313999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 434340, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 95, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4567132, "tokens_cache_write": 95534, "tokens_input": 285, "tokens_output": 11175, "tool_calls": 82, "tool_calls_by_class": { "harness_builtin": 81, "subagent": 1 }, "tool_result_tokens": 134475, "tool_time_ms": 134004, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.135399, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 265127, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 87, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4077735, "tokens_cache_write": 84196, "tokens_input": 261, "tokens_output": 9070, "tool_calls": 55, "tool_calls_by_class": { "harness_builtin": 55 }, "tool_result_tokens": 136455, "tool_time_ms": 18554, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.2019714000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 387224, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 77, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4036797, "tokens_cache_write": 119044, "tokens_input": 231, "tokens_output": 8045, "tool_calls": 78, "tool_calls_by_class": { "harness_builtin": 77, "subagent": 1 }, "tool_result_tokens": 126811, "tool_time_ms": 142325, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.0318308, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1998120, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 148, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2328804, "tokens_cache_write": 532888, "tokens_input": 447, "tokens_output": 19413, "tool_calls": 176, "tool_calls_by_class": { "harness_builtin": 168, "subagent": 8 }, "tool_result_tokens": 766719, "tool_time_ms": 1437521, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.1888875999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1788871, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 168, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2872103, "tokens_cache_write": 539366, "tokens_input": 516, "tokens_output": 22085, "tool_calls": 203, "tool_calls_by_class": { "harness_builtin": 198, "subagent": 5 }, "tool_result_tokens": 431702, "tool_time_ms": 1266093, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8996431, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2571193, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 132, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1995968, "tokens_cache_write": 514279, "tokens_input": 396, "tokens_output": 17830, "tool_calls": 155, "tool_calls_by_class": { "harness_builtin": 145, "subagent": 10 }, "tool_result_tokens": 1290421, "tool_time_ms": 2077614, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.790196, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2064923, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 132, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1980410, "tokens_cache_write": 475808, "tokens_input": 399, "tokens_output": 16983, "tool_calls": 151, "tool_calls_by_class": { "harness_builtin": 141, "subagent": 10 }, "tool_result_tokens": 630403, "tool_time_ms": 1660575, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8251640999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 3089563, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 145, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2136528, "tokens_cache_write": 470881, "tokens_input": 426, "tokens_output": 18317, "tool_calls": 152, "tool_calls_by_class": { "harness_builtin": 144, "subagent": 8 }, "tool_result_tokens": 622249, "tool_time_ms": 2573602, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.0365295000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1702970, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "harness_bare", "llm_calls": 142, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2177835, "tokens_cache_write": 547097, "tokens_input": 426, "tokens_output": 19364, "tool_calls": 177, "tool_calls_by_class": { "harness_builtin": 171, "subagent": 6 }, "tool_result_tokens": 545969, "tool_time_ms": 1239735, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.9374408, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1873437, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "harness_bare", "llm_calls": 147, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2280274, "tokens_cache_write": 508144, "tokens_input": 441, "tokens_output": 17512, "tool_calls": 157, "tool_calls_by_class": { "harness_builtin": 149, "subagent": 8 }, "tool_result_tokens": 834805, "tool_time_ms": 1409491, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.8317086, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 3148692, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 135, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1969248, "tokens_cache_write": 498778, "tokens_input": 393, "tokens_output": 15844, "tool_calls": 150, "tool_calls_by_class": { "harness_builtin": 141, "subagent": 9 }, "tool_result_tokens": 663160, "tool_time_ms": 2726349, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6999684, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2030706, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 131, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1828112, "tokens_cache_write": 451824, "tokens_input": 393, "tokens_output": 17000, "tool_calls": 134, "tool_calls_by_class": { "harness_builtin": 125, "subagent": 9 }, "tool_result_tokens": 541578, "tool_time_ms": 1584817, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 1.6003413000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2517990, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "harness_bare", "llm_calls": 124, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 1749579, "tokens_cache_write": 426231, "tokens_input": 372, "tokens_output": 15342, "tool_calls": 124, "tool_calls_by_class": { "harness_builtin": 114, "subagent": 10 }, "tool_result_tokens": 631811, "tool_time_ms": 2094576, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.0532344, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1203606, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 232, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3555637, "tokens_cache_write": 814234, "tokens_input": 711, "tokens_output": 25425, "tool_calls": 235, "tool_calls_by_class": { "harness_builtin": 232, "subagent": 3 }, "tool_result_tokens": 551395, "tool_time_ms": 455192, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.5934005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1347831, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 211, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 2965965, "tokens_cache_write": 700783, "tokens_input": 633, "tokens_output": 20582, "tool_calls": 185, "tool_calls_by_class": { "harness_builtin": 178, "subagent": 7 }, "tool_result_tokens": 424693, "tool_time_ms": 690398, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 3.024481, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1188672, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 242, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3626870, "tokens_cache_write": 803574, "tokens_input": 726, "tokens_output": 24060, "tool_calls": 242, "tool_calls_by_class": { "harness_builtin": 238, "subagent": 4 }, "tool_result_tokens": 349410, "tool_time_ms": 490575, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.9226985, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1560724, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 240, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3571795, "tokens_cache_write": 761039, "tokens_input": 729, "tokens_output": 25357, "tool_calls": 236, "tool_calls_by_class": { "harness_builtin": 232, "subagent": 4 }, "tool_result_tokens": 323522, "tool_time_ms": 742992, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.877878, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1489588, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 230, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3314255, "tokens_cache_write": 779302, "tokens_input": 690, "tokens_output": 22116, "tool_calls": 225, "tool_calls_by_class": { "harness_builtin": 219, "subagent": 6 }, "tool_result_tokens": 444069, "tool_time_ms": 792332, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7682786000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1216304, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 233, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3362853, "tokens_cache_write": 735724, "tokens_input": 699, "tokens_output": 21250, "tool_calls": 220, "tool_calls_by_class": { "harness_builtin": 216, "subagent": 4 }, "tool_result_tokens": 328938, "tool_time_ms": 460305, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.8541241, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1433765, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 213, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3251743, "tokens_cache_write": 776551, "tokens_input": 651, "tokens_output": 21758, "tool_calls": 219, "tool_calls_by_class": { "harness_builtin": 211, "subagent": 8 }, "tool_result_tokens": 350483, "tool_time_ms": 780173, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.6598025, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1313172, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 222, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3257560, "tokens_cache_write": 700673, "tokens_input": 666, "tokens_output": 21273, "tool_calls": 211, "tool_calls_by_class": { "harness_builtin": 204, "subagent": 7 }, "tool_result_tokens": 356141, "tool_time_ms": 706407, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7604853, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1108965, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 228, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3207029, "tokens_cache_write": 736771, "tokens_input": 684, "tokens_output": 22982, "tool_calls": 205, "tool_calls_by_class": { "harness_builtin": 201, "subagent": 4 }, "tool_result_tokens": 306084, "tool_time_ms": 409449, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 2.7008304, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1160710, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "harness_bare", "llm_calls": 233, "model": "gpt-5.6-terra", "path": "runs/cursor/gpt-5.6-terra/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|gpt-5.6-terra|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3314677, "tokens_cache_write": 708994, "tokens_input": 699, "tokens_output": 22001, "tool_calls": 206, "tool_calls_by_class": { "harness_builtin": 201, "subagent": 5 }, "tool_result_tokens": 294122, "tool_time_ms": 471958, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.22446, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 685539, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 91, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9093120, "tokens_cache_write": 0, "tokens_input": 181699, "tokens_output": 33712, "tool_calls": 130, "tool_calls_by_class": { "harness_builtin": 130 }, "tool_result_tokens": 87650, "tool_time_ms": 87144, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.296768, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 855567, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 90, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9118784, "tokens_cache_write": 0, "tokens_input": 184780, "tokens_output": 36572, "tool_calls": 130, "tool_calls_by_class": { "harness_builtin": 130 }, "tool_result_tokens": 113588, "tool_time_ms": 145949, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.374916, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 827685, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 94, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8305664, "tokens_cache_write": 0, "tokens_input": 163888, "tokens_output": 34475, "tool_calls": 123, "tool_calls_by_class": { "harness_builtin": 123 }, "tool_result_tokens": 91708, "tool_time_ms": 122047, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.35354, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 690362, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 89, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9166016, "tokens_cache_write": 0, "tokens_input": 189394, "tokens_output": 35829, "tool_calls": 119, "tool_calls_by_class": { "harness_builtin": 119 }, "tool_result_tokens": 106188, "tool_time_ms": 138041, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.435188, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 928236, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 94, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9322880, "tokens_cache_write": 0, "tokens_input": 174898, "tokens_output": 34393, "tool_calls": 124, "tool_calls_by_class": { "harness_builtin": 124 }, "tool_result_tokens": 97202, "tool_time_ms": 86868, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.401216, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 883128, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 96, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9292288, "tokens_cache_write": 0, "tokens_input": 173279, "tokens_output": 34651, "tool_calls": 128, "tool_calls_by_class": { "harness_builtin": 128 }, "tool_result_tokens": 99740, "tool_time_ms": 107286, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.372340000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 726802, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 87, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8273856, "tokens_cache_write": 0, "tokens_input": 167476, "tokens_output": 35715, "tool_calls": 121, "tool_calls_by_class": { "harness_builtin": 121 }, "tool_result_tokens": 95810, "tool_time_ms": 93832, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.354536, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 718452, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 101, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9217856, "tokens_cache_write": 0, "tokens_input": 169072, "tokens_output": 38366, "tool_calls": 142, "tool_calls_by_class": { "harness_builtin": 142 }, "tool_result_tokens": 93901, "tool_time_ms": 103954, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.337028, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 753568, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 95, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 8264192, "tokens_cache_write": 0, "tokens_input": 159324, "tokens_output": 36295, "tool_calls": 125, "tool_calls_by_class": { "harness_builtin": 125 }, "tool_result_tokens": 84651, "tool_time_ms": 102447, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.821984, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 759090, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "harness_bare", "llm_calls": 90, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7811904, "tokens_cache_write": 0, "tokens_input": 155920, "tokens_output": 32200, "tool_calls": 134, "tool_calls_by_class": { "harness_builtin": 134 }, "tool_result_tokens": 88285, "tool_time_ms": 123418, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.41394, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 376354, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 103, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7489856, "tokens_cache_write": 0, "tokens_input": 153777, "tokens_output": 25748, "tool_calls": 74, "tool_calls_by_class": { "harness_builtin": 74 }, "tool_result_tokens": 90173, "tool_time_ms": 34718, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.203732, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 429543, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 99, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7280128, "tokens_cache_write": 0, "tokens_input": 152004, "tokens_output": 26299, "tool_calls": 68, "tool_calls_by_class": { "harness_builtin": 68 }, "tool_result_tokens": 80599, "tool_time_ms": 37886, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.9386719999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 429330, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 98, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7016576, "tokens_cache_write": 0, "tokens_input": 151609, "tokens_output": 26305, "tool_calls": 67, "tool_calls_by_class": { "harness_builtin": 67 }, "tool_result_tokens": 84306, "tool_time_ms": 30855, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.082208, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 410839, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 95, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7150016, "tokens_cache_write": 0, "tokens_input": 160373, "tokens_output": 24225, "tool_calls": 113, "tool_calls_by_class": { "harness_builtin": 113 }, "tool_result_tokens": 90302, "tool_time_ms": 40526, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.149148, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 433595, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 108, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7287744, "tokens_cache_write": 0, "tokens_input": 141437, "tokens_output": 24638, "tool_calls": 77, "tool_calls_by_class": { "harness_builtin": 77 }, "tool_result_tokens": 78333, "tool_time_ms": 29682, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.867972, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 448753, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 104, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6971072, "tokens_cache_write": 0, "tokens_input": 146297, "tokens_output": 25976, "tool_calls": 73, "tool_calls_by_class": { "harness_builtin": 73 }, "tool_result_tokens": 84431, "tool_time_ms": 38729, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.5962, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 329441, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 96, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 7641024, "tokens_cache_write": 0, "tokens_input": 163395, "tokens_output": 25133, "tool_calls": 107, "tool_calls_by_class": { "harness_builtin": 107 }, "tool_result_tokens": 97365, "tool_time_ms": 34861, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.518332000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 355220, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 109, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6702912, "tokens_cache_write": 0, "tokens_input": 127295, "tokens_output": 25520, "tool_calls": 82, "tool_calls_by_class": { "harness_builtin": 82 }, "tool_result_tokens": 66225, "tool_time_ms": 32860, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 10.444564, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 522314, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 109, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 9442688, "tokens_cache_write": 0, "tokens_input": 166265, "tokens_output": 28068, "tool_calls": 143, "tool_calls_by_class": { "harness_builtin": 143 }, "tool_result_tokens": 95967, "tool_time_ms": 48931, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.744252, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 495395, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 103, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 6858816, "tokens_cache_write": 0, "tokens_input": 141133, "tokens_output": 26742, "tool_calls": 72, "tool_calls_by_class": { "harness_builtin": 72 }, "tool_result_tokens": 80178, "tool_time_ms": 38566, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.18192, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1389920, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 210, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4402048, "tokens_cache_write": 0, "tokens_input": 712522, "tokens_output": 77482, "tool_calls": 340, "tool_calls_by_class": { "harness_builtin": 340 }, "tool_result_tokens": 298551, "tool_time_ms": 274285, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.712164, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1437374, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 204, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4698688, "tokens_cache_write": 0, "tokens_input": 744409, "tokens_output": 86320, "tool_calls": 330, "tool_calls_by_class": { "harness_builtin": 330 }, "tool_result_tokens": 405671, "tool_time_ms": 255362, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.126972, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1414721, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "harness_bare", "llm_calls": 196, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4139456, "tokens_cache_write": 0, "tokens_input": 735186, "tokens_output": 87231, "tool_calls": 329, "tool_calls_by_class": { "harness_builtin": 329 }, "tool_result_tokens": 320749, "tool_time_ms": 206062, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.182952, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1557466, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 196, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4174848, "tokens_cache_write": 0, "tokens_input": 732269, "tokens_output": 89919, "tool_calls": 320, "tool_calls_by_class": { "harness_builtin": 320 }, "tool_result_tokens": 294643, "tool_time_ms": 265226, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.965236, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1627047, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 214, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4796992, "tokens_cache_write": 0, "tokens_input": 746792, "tokens_output": 98423, "tool_calls": 358, "tool_calls_by_class": { "harness_builtin": 358 }, "tool_result_tokens": 584630, "tool_time_ms": 319112, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.967928, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1501240, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 222, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4899456, "tokens_cache_write": 0, "tokens_input": 764974, "tokens_output": 84048, "tool_calls": 369, "tool_calls_by_class": { "harness_builtin": 369 }, "tool_result_tokens": 463120, "tool_time_ms": 235521, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.92462, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1396412, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 223, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4867520, "tokens_cache_write": 0, "tokens_input": 733214, "tokens_output": 93687, "tool_calls": 364, "tool_calls_by_class": { "harness_builtin": 364 }, "tool_result_tokens": 293527, "tool_time_ms": 257636, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.410252, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1280983, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 199, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4515456, "tokens_cache_write": 0, "tokens_input": 722596, "tokens_output": 83701, "tool_calls": 339, "tool_calls_by_class": { "harness_builtin": 339 }, "tool_result_tokens": 278732, "tool_time_ms": 271790, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.269864, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1557065, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "harness_bare", "llm_calls": 206, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 4390976, "tokens_cache_write": 0, "tokens_input": 709388, "tokens_output": 86778, "tool_calls": 327, "tool_calls_by_class": { "harness_builtin": 327 }, "tool_result_tokens": 292112, "tool_time_ms": 237785, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos-hard", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.546264, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1270219, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "harness_bare", "llm_calls": 190, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3791808, "tokens_cache_write": 0, "tokens_input": 688669, "tokens_output": 83315, "tool_calls": 319, "tool_calls_by_class": { "harness_builtin": 319 }, "tool_result_tokens": 287620, "tool_time_ms": 183035, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 9.191228, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1256513, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 225, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3774080, "tokens_cache_write": 0, "tokens_input": 1177515, "tokens_output": 58924, "tool_calls": 337, "tool_calls_by_class": { "harness_builtin": 337 }, "tool_result_tokens": 330312, "tool_time_ms": 162183, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.030252, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1059764, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 225, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3516800, "tokens_cache_write": 0, "tokens_input": 962796, "tokens_output": 55189, "tool_calls": 313, "tool_calls_by_class": { "harness_builtin": 313 }, "tool_result_tokens": 281539, "tool_time_ms": 199476, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.5345759999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1032258, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 215, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3435008, "tokens_cache_write": 0, "tokens_input": 854903, "tokens_output": 56663, "tool_calls": 293, "tool_calls_by_class": { "harness_builtin": 293 }, "tool_result_tokens": 289406, "tool_time_ms": 183085, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 7.760427999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 918667, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 219, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3745408, "tokens_cache_write": 0, "tokens_input": 841029, "tokens_output": 54242, "tool_calls": 281, "tool_calls_by_class": { "harness_builtin": 281 }, "tool_result_tokens": 296201, "tool_time_ms": 146043, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.335163999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1033795, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 222, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3898816, "tokens_cache_write": 0, "tokens_input": 943346, "tokens_output": 55247, "tool_calls": 306, "tool_calls_by_class": { "harness_builtin": 306 }, "tool_result_tokens": 305860, "tool_time_ms": 165800, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.476288, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 860937, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 222, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3470144, "tokens_cache_write": 0, "tokens_input": 1091507, "tokens_output": 53343, "tool_calls": 333, "tool_calls_by_class": { "harness_builtin": 333 }, "tool_result_tokens": 342749, "tool_time_ms": 167841, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.354364, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 944072, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 216, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3392896, "tokens_cache_write": 0, "tokens_input": 1065485, "tokens_output": 58294, "tool_calls": 304, "tool_calls_by_class": { "harness_builtin": 304 }, "tool_result_tokens": 276192, "tool_time_ms": 169982, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.001092, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 932406, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 228, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3940416, "tokens_cache_write": 0, "tokens_input": 848318, "tokens_output": 55617, "tool_calls": 312, "tool_calls_by_class": { "harness_builtin": 312 }, "tool_result_tokens": 302829, "tool_time_ms": 142185, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.07196, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 940058, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 219, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3564160, "tokens_cache_write": 0, "tokens_input": 961638, "tokens_output": 55104, "tool_calls": 325, "tool_calls_by_class": { "harness_builtin": 325 }, "tool_result_tokens": 343966, "tool_time_ms": 184336, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "cursor", "arm_label": "Cursor Agent (bare)", "battery": "memos", "cost_basis": "estimated:cursor-published-0826", "cost_reported_usd": null, "cost_usd": 8.747008, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 947417, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "harness_bare", "llm_calls": 223, "model": "grok-4.6", "path": "runs/cursor/grok-4.6/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|cursor|grok-4.6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "cursor_reported_derived", "tokens_cache_read": 3673216, "tokens_cache_write": 0, "tokens_input": 1095600, "tokens_output": 57616, "tool_calls": 336, "tool_calls_by_class": { "harness_builtin": 336 }, "tool_result_tokens": 309953, "tool_time_ms": 175994, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8193624999999995, "cost_usd": 4.8193624999999995, "denominator": 20, "duration_basis": "run", "duration_ms": 249609, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 52, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2195026, "tokens_cache_write": 183424, "tokens_input": 330, "tokens_output": 20487, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 79, "subagent": 4, "vendor_mcp": 3 }, "tool_result_tokens": 47562, "tool_time_ms": 30566, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.524586999999999, "cost_usd": 7.524586999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 322699, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 86, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3866337, "tokens_cache_write": 308387, "tokens_input": 170, "tokens_output": 40817, "tool_calls": 151, "tool_calls_by_class": { "cc_builtin": 138, "subagent": 10, "vendor_mcp": 3 }, "tool_result_tokens": 106155, "tool_time_ms": 19457, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.189777500000003, "cost_usd": 7.189777500000003, "denominator": 20, "duration_basis": "run", "duration_ms": 282978, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 60, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2343205, "tokens_cache_write": 190059, "tokens_input": 1229, "tokens_output": 26824, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 99, "subagent": 8, "vendor_mcp": 4 }, "tool_result_tokens": 49232, "tool_time_ms": 21654, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4966444999999995, "cost_usd": 3.4966444999999995, "denominator": 20, "duration_basis": "run", "duration_ms": 394709, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 37, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2598971, "tokens_cache_write": 98022, "tokens_input": 70, "tokens_output": 25939, "tool_calls": 40, "tool_calls_by_class": { "cc_builtin": 35, "vendor_mcp": 5 }, "tool_result_tokens": 23023, "tool_time_ms": 13399, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.5153205000000005, "cost_usd": 6.5153205000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 319890, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 71, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3206809, "tokens_cache_write": 249461, "tokens_input": 140, "tokens_output": 33201, "tool_calls": 134, "tool_calls_by_class": { "cc_builtin": 119, "subagent": 7, "vendor_mcp": 8 }, "tool_result_tokens": 86298, "tool_time_ms": 15921, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.184550500000002, "cost_usd": 12.184550500000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 484865, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 82, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7046257, "tokens_cache_write": 212580, "tokens_input": 156, "tokens_output": 24747, "tool_calls": 68, "tool_calls_by_class": { "cc_builtin": 65, "vendor_mcp": 3 }, "tool_result_tokens": 26819, "tool_time_ms": 21106, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.596374, "cost_usd": 11.596374, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 519336, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 86, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7770454, "tokens_cache_write": 120497, "tokens_input": 163, "tokens_output": 28287, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 66, "vendor_mcp": 4 }, "tool_result_tokens": 28322, "tool_time_ms": 21205, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.531615, "cost_usd": 10.531615, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 467843, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 81, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7001185, "tokens_cache_write": 109200, "tokens_input": 153, "tokens_output": 26898, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 61, "vendor_mcp": 5 }, "tool_result_tokens": 25297, "tool_time_ms": 20997, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.457586, "cost_usd": 9.457586, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 511232, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 77, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6199796, "tokens_cache_write": 90769, "tokens_input": 146, "tokens_output": 28819, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 58, "vendor_mcp": 1 }, "tool_result_tokens": 19266, "tool_time_ms": 19730, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.1771865, "cost_usd": 10.1771865, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 705827, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 93, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7158302, "tokens_cache_write": 98183, "tokens_input": 177, "tokens_output": 27596, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 70, "vendor_mcp": 4 }, "tool_result_tokens": 16919, "tool_time_ms": 26951, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.017197, "cost_usd": 7.017197, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 373093, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 78, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5138807, "tokens_cache_write": 50155, "tokens_input": 149, "tokens_output": 17476, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 48 }, "tool_result_tokens": 12738, "tool_time_ms": 20319, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.127834, "cost_usd": 8.127834, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 394395, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5917614, "tokens_cache_write": 62186, "tokens_input": 160, "tokens_output": 19298, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 54, "vendor_mcp": 2 }, "tool_result_tokens": 19731, "tool_time_ms": 21512, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.433685, "cost_usd": 8.433685, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 424234, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 85, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6014235, "tokens_cache_write": 66894, "tokens_input": 162, "tokens_output": 21599, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 55, "vendor_mcp": 2 }, "tool_result_tokens": 22058, "tool_time_ms": 21483, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.742422000000001, "cost_usd": 8.742422000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 430535, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 91, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6499182, "tokens_cache_write": 60123, "tokens_input": 173, "tokens_output": 20781, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 61, "vendor_mcp": 1 }, "tool_result_tokens": 17116, "tool_time_ms": 27181, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.234184, "cost_usd": 8.234184, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 403968, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 89, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6081564, "tokens_cache_write": 57231, "tokens_input": 170, "tokens_output": 20126, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 15405, "tool_time_ms": 22429, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.448804250000002, "cost_usd": 10.448804250000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 701895, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 98, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4634014, "tokens_cache_write": 277759, "tokens_input": 690, "tokens_output": 35974, "tool_calls": 104, "tool_calls_by_class": { "cc_builtin": 89, "subagent": 1, "vendor_mcp": 14 }, "tool_result_tokens": 58681, "tool_time_ms": 20854, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.257053500000001, "cost_usd": 10.257053500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 705477, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 100, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5071547, "tokens_cache_write": 211744, "tokens_input": 430, "tokens_output": 40937, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 89, "vendor_mcp": 22 }, "tool_result_tokens": 71673, "tool_time_ms": 21031, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.5458175, "cost_usd": 8.5458175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 670812, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 88, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4390212, "tokens_cache_write": 164231, "tokens_input": 174, "tokens_output": 37102, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 76, "vendor_mcp": 14 }, "tool_result_tokens": 59722, "tool_time_ms": 19796, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.588646499999998, "cost_usd": 9.588646499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 684375, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 99, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4983784, "tokens_cache_write": 181614, "tokens_input": 2488, "tokens_output": 41047, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 86, "vendor_mcp": 17 }, "tool_result_tokens": 68263, "tool_time_ms": 23744, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.9004145, "cost_usd": 8.9004145, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 657977, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 97, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4849407, "tokens_cache_write": 154442, "tokens_input": 261, "tokens_output": 38704, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 75, "vendor_mcp": 23 }, "tool_result_tokens": 59354, "tool_time_ms": 20034, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.647775, "cost_usd": 2.647775, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 72614, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 18, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 772915, "tokens_cache_write": 85135, "tokens_input": 36, "tokens_output": 3436, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "vendor_mcp": 2 }, "tool_result_tokens": 5080, "tool_time_ms": 2863, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9156659999999999, "cost_usd": 1.9156659999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 77612, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 18, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 815696, "tokens_cache_write": 45648, "tokens_input": 36, "tokens_output": 3733, "tool_calls": 13, "tool_calls_by_class": { "cc_builtin": 11, "vendor_mcp": 2 }, "tool_result_tokens": 4565, "tool_time_ms": 3748, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.876942, "cost_usd": 1.876942, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 78850, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 19, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 860702, "tokens_cache_write": 42393, "tokens_input": 38, "tokens_output": 3360, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 11, "vendor_mcp": 1 }, "tool_result_tokens": 2881, "tool_time_ms": 4109, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.591879, "cost_usd": 1.591879, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 69331, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 19, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 878349, "tokens_cache_write": 27055, "tokens_input": 38, "tokens_output": 3441, "tool_calls": 13, "tool_calls_by_class": { "cc_builtin": 12, "vendor_mcp": 1 }, "tool_result_tokens": 4475, "tool_time_ms": 3592, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.498584, "cost_usd": 1.498584, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 89711, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 19, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 881624, "tokens_cache_write": 22289, "tokens_input": 38, "tokens_output": 3416, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 11, "vendor_mcp": 1 }, "tool_result_tokens": 3487, "tool_time_ms": 3938, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.954160000000002, "cost_usd": 10.954160000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 609835, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 111, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5284920, "tokens_cache_write": 203848, "tokens_input": 593, "tokens_output": 31727, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 82, "vendor_mcp": 8 }, "tool_result_tokens": 44850, "tool_time_ms": 25999, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.793626, "cost_usd": 10.793626, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 655418, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 110, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5258056, "tokens_cache_write": 190045, "tokens_input": 3452, "tokens_output": 34003, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 87, "vendor_mcp": 11 }, "tool_result_tokens": 45661, "tool_time_ms": 24974, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.825385999999998, "cost_usd": 10.825385999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 626115, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 107, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5117206, "tokens_cache_write": 199218, "tokens_input": 3112, "tokens_output": 33854, "tool_calls": 92, "tool_calls_by_class": { "cc_builtin": 79, "vendor_mcp": 13 }, "tool_result_tokens": 50587, "tool_time_ms": 26607, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.297209, "cost_usd": 10.297209, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 640232, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 114, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5497209, "tokens_cache_write": 154665, "tokens_input": 2050, "tokens_output": 33724, "tool_calls": 95, "tool_calls_by_class": { "cc_builtin": 86, "vendor_mcp": 9 }, "tool_result_tokens": 44610, "tool_time_ms": 28130, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.438771000000001, "cost_usd": 9.438771000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 598278, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 107, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5167941, "tokens_cache_write": 135792, "tokens_input": 214, "tokens_output": 31057, "tool_calls": 92, "tool_calls_by_class": { "cc_builtin": 83, "vendor_mcp": 9 }, "tool_result_tokens": 47518, "tool_time_ms": 25121, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 13.632814, "cost_usd": 13.632814, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 626252, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 117, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5459804, "tokens_cache_write": 320635, "tokens_input": 386, "tokens_output": 35129, "tool_calls": 102, "tool_calls_by_class": { "cc_builtin": 92, "vendor_mcp": 10 }, "tool_result_tokens": 46508, "tool_time_ms": 31162, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 13.132416, "cost_usd": 13.132416, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 617756, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 113, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|12", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5325256, "tokens_cache_write": 301338, "tokens_input": 225, "tokens_output": 35563, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 86, "vendor_mcp": 12 }, "tool_result_tokens": 56541, "tool_time_ms": 26044, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.818522, "cost_usd": 11.818522, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 587439, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 113, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|13", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5378242, "tokens_cache_write": 239569, "tokens_input": 225, "tokens_output": 32933, "tool_calls": 96, "tool_calls_by_class": { "cc_builtin": 87, "vendor_mcp": 9 }, "tool_result_tokens": 46105, "tool_time_ms": 29464, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.858771, "cost_usd": 11.858771, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 556321, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 110, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|14", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5198781, "tokens_cache_write": 249567, "tokens_input": 220, "tokens_output": 33329, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 81, "vendor_mcp": 9 }, "tool_result_tokens": 49840, "tool_time_ms": 25242, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 12.566540999999997, "cost_usd": 12.566540999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 597243, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 115, "model": "claude-fable-5", "path": "runs/gitnexus/claude-fable-5/ZeroShotExam/memos/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|15", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5483871, "tokens_cache_write": 261848, "tokens_input": 2311, "tokens_output": 36452, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 15 }, "tool_result_tokens": 60453, "tool_time_ms": 25336, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.855136400000001, "cost_usd": 6.855136400000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1105552, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 263, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 31156634, "tokens_cache_write": 1774152, "tokens_input": 1599, "tokens_output": 37914, "tool_calls": 243, "tool_calls_by_class": { "cc_builtin": 233, "harness_builtin": 2, "vendor_mcp": 8 }, "tool_result_tokens": 73391, "tool_time_ms": 68557, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.708901200000001, "cost_usd": 7.708901200000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1011431, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 239, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 28730162, "tokens_cache_write": 2328907, "tokens_input": 1476, "tokens_output": 35319, "tool_calls": 219, "tool_calls_by_class": { "cc_builtin": 195, "harness_builtin": 3, "vendor_mcp": 21 }, "tool_result_tokens": 82173, "tool_time_ms": 40103, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.335222499999998, "cost_usd": 7.335222499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 946745, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 236, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 28759355, "tokens_cache_write": 2141868, "tokens_input": 2341, "tokens_output": 34642, "tool_calls": 241, "tool_calls_by_class": { "cc_builtin": 238, "vendor_mcp": 3 }, "tool_result_tokens": 78548, "tool_time_ms": 50608, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.801349650000001, "cost_usd": 5.801349650000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 837113, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 205, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 24125283, "tokens_cache_write": 1601031, "tokens_input": 1249, "tokens_output": 29932, "tool_calls": 185, "tool_calls_by_class": { "cc_builtin": 163, "harness_builtin": 1, "vendor_mcp": 21 }, "tool_result_tokens": 95203, "tool_time_ms": 26795, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7471314000000016, "cost_usd": 4.7471314000000016, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 855089, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 224, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 23280434, "tokens_cache_write": 1135860, "tokens_input": 1373, "tokens_output": 29199, "tool_calls": 204, "tool_calls_by_class": { "cc_builtin": 190, "vendor_mcp": 14 }, "tool_result_tokens": 64912, "tool_time_ms": 38498, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.6824290500000005, "cost_usd": 6.6824290500000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 857733, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 198, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 25488770, "tokens_cache_write": 1972492, "tokens_input": 1215, "tokens_output": 30388, "tool_calls": 179, "tool_calls_by_class": { "cc_builtin": 160, "harness_builtin": 2, "vendor_mcp": 17 }, "tool_result_tokens": 91071, "tool_time_ms": 31660, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.941000600000001, "cost_usd": 6.941000600000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1023118, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 242, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 28262071, "tokens_cache_write": 1947056, "tokens_input": 1436, "tokens_output": 36364, "tool_calls": 222, "tool_calls_by_class": { "cc_builtin": 207, "harness_builtin": 1, "vendor_mcp": 14 }, "tool_result_tokens": 83387, "tool_time_ms": 34378, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.9889521000000006, "cost_usd": 6.9889521000000006, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1074889, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 260, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 30375167, "tokens_cache_write": 1865547, "tokens_input": 1579, "tokens_output": 36730, "tool_calls": 240, "tool_calls_by_class": { "cc_builtin": 225, "harness_builtin": 2, "vendor_mcp": 13 }, "tool_result_tokens": 83508, "tool_time_ms": 38779, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2363474, "cost_usd": 2.2363474, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 455446, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 136, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12731724, "tokens_cache_write": 430638, "tokens_input": 949, "tokens_output": 20190, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 106 }, "tool_result_tokens": 55644, "tool_time_ms": 23958, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2428742999999995, "cost_usd": 2.2428742999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 535562, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 154, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13415063, "tokens_cache_write": 403472, "tokens_input": 1052, "tokens_output": 23195, "tool_calls": 124, "tool_calls_by_class": { "cc_builtin": 118, "subagent": 1, "vendor_mcp": 5 }, "tool_result_tokens": 61878, "tool_time_ms": 27804, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9723385, "cost_usd": 2.9723385, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1338035, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 331, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12496720, "tokens_cache_write": 1011589, "tokens_input": 2333, "tokens_output": 30161, "tool_calls": 403, "tool_calls_by_class": { "cc_builtin": 374, "subagent": 29 }, "tool_result_tokens": 349859, "tool_time_ms": 75574, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9569422, "cost_usd": 2.9569422, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 539147, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 156, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14676582, "tokens_cache_write": 695068, "tokens_input": 1048, "tokens_output": 19620, "tool_calls": 126, "tool_calls_by_class": { "cc_builtin": 125, "vendor_mcp": 1 }, "tool_result_tokens": 52319, "tool_time_ms": 36385, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.10766755, "cost_usd": 5.10766755, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 665051, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 392, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 20259316, "tokens_cache_write": 1619805, "tokens_input": 2770, "tokens_output": 41390, "tool_calls": 458, "tool_calls_by_class": { "cc_builtin": 398, "subagent": 59, "vendor_mcp": 1 }, "tool_result_tokens": 526785, "tool_time_ms": 70182, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2226866000000003, "cost_usd": 3.2226866000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 585082, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 167, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16250466, "tokens_cache_write": 743196, "tokens_input": 1143, "tokens_output": 22021, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 136, "vendor_mcp": 1 }, "tool_result_tokens": 55210, "tool_time_ms": 35854, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2974031, "cost_usd": 2.2974031, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 480793, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 136, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13610681, "tokens_cache_write": 413592, "tokens_input": 1006, "tokens_output": 21629, "tool_calls": 107, "tool_calls_by_class": { "cc_builtin": 102, "harness_builtin": 1, "vendor_mcp": 4 }, "tool_result_tokens": 57968, "tool_time_ms": 21883, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.918543100000001, "cost_usd": 2.918543100000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1326497, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 331, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12830181, "tokens_cache_write": 925194, "tokens_input": 8705, "tokens_output": 34124, "tool_calls": 465, "tool_calls_by_class": { "cc_builtin": 427, "skill": 1, "subagent": 30, "vendor_mcp": 7 }, "tool_result_tokens": 339910, "tool_time_ms": 83948, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4696372999999996, "cost_usd": 3.4696372999999996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 821479, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 312, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15802788, "tokens_cache_write": 872158, "tokens_input": 6089, "tokens_output": 48378, "tool_calls": 389, "tool_calls_by_class": { "cc_builtin": 295, "harness_builtin": 2, "subagent": 4, "vendor_mcp": 88 }, "tool_result_tokens": 264825, "tool_time_ms": 31934, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.546801549999999, "cost_usd": 3.546801549999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 611917, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 297, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13835968, "tokens_cache_write": 1090135, "tokens_input": 1975, "tokens_output": 40575, "tool_calls": 395, "tool_calls_by_class": { "cc_builtin": 323, "subagent": 8, "vendor_mcp": 64 }, "tool_result_tokens": 307298, "tool_time_ms": 44167, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.401253799999999, "cost_usd": 3.401253799999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 831160, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 303, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14983428, "tokens_cache_write": 923216, "tokens_input": 3974, "tokens_output": 46726, "tool_calls": 403, "tool_calls_by_class": { "cc_builtin": 309, "subagent": 7, "vendor_mcp": 87 }, "tool_result_tokens": 260606, "tool_time_ms": 42816, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7301102999999998, "cost_usd": 3.7301102999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 823135, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 311, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16377563, "tokens_cache_write": 987974, "tokens_input": 3144, "tokens_output": 49834, "tool_calls": 410, "tool_calls_by_class": { "cc_builtin": 309, "harness_builtin": 3, "subagent": 4, "vendor_mcp": 94 }, "tool_result_tokens": 286602, "tool_time_ms": 40167, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.81467635, "cost_usd": 2.81467635, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 830568, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 266, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13409676, "tokens_cache_write": 669125, "tokens_input": 1997, "tokens_output": 39589, "tool_calls": 317, "tool_calls_by_class": { "cc_builtin": 241, "harness_builtin": 1, "subagent": 4, "vendor_mcp": 71 }, "tool_result_tokens": 225931, "tool_time_ms": 27527, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9402470000000003, "cost_usd": 2.9402470000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 833385, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 276, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13621800, "tokens_cache_write": 748942, "tokens_input": 3147, "tokens_output": 38626, "tool_calls": 336, "tool_calls_by_class": { "cc_builtin": 272, "subagent": 3, "vendor_mcp": 61 }, "tool_result_tokens": 243345, "tool_time_ms": 44028, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.83865245, "cost_usd": 2.83865245, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 846509, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 275, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14150097, "tokens_cache_write": 623079, "tokens_input": 2747, "tokens_output": 41813, "tool_calls": 328, "tool_calls_by_class": { "cc_builtin": 249, "harness_builtin": 2, "subagent": 3, "vendor_mcp": 74 }, "tool_result_tokens": 217344, "tool_time_ms": 28104, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6698397, "cost_usd": 3.6698397, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 710909, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 346, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17010572, "tokens_cache_write": 924079, "tokens_input": 4934, "tokens_output": 47710, "tool_calls": 423, "tool_calls_by_class": { "cc_builtin": 355, "subagent": 8, "vendor_mcp": 60 }, "tool_result_tokens": 325396, "tool_time_ms": 37643, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.26614185, "cost_usd": 3.26614185, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 644135, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 297, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14370601, "tokens_cache_write": 866775, "tokens_input": 5424, "tokens_output": 46754, "tool_calls": 334, "tool_calls_by_class": { "cc_builtin": 269, "harness_builtin": 2, "subagent": 7, "vendor_mcp": 56 }, "tool_result_tokens": 264768, "tool_time_ms": 32072, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.44613565, "cost_usd": 3.44613565, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 643301, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 336, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15579959, "tokens_cache_write": 909917, "tokens_input": 4732, "tokens_output": 43645, "tool_calls": 381, "tool_calls_by_class": { "cc_builtin": 323, "harness_builtin": 1, "subagent": 7, "vendor_mcp": 50 }, "tool_result_tokens": 298842, "tool_time_ms": 28745, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.75153825, "cost_usd": 2.75153825, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 645955, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 238, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10623405, "tokens_cache_write": 816591, "tokens_input": 3883, "tokens_output": 32735, "tool_calls": 283, "tool_calls_by_class": { "cc_builtin": 221, "subagent": 5, "vendor_mcp": 57 }, "tool_result_tokens": 238295, "tool_time_ms": 25981, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7646031000000004, "cost_usd": 2.7646031000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 523294, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 232, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9707416, "tokens_cache_write": 902873, "tokens_input": 4725, "tokens_output": 31072, "tool_calls": 278, "tool_calls_by_class": { "cc_builtin": 225, "harness_builtin": 1, "skill": 1, "subagent": 9, "vendor_mcp": 42 }, "tool_result_tokens": 234425, "tool_time_ms": 31978, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1467512, "cost_usd": 3.1467512, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 580738, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 275, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 25, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12287642, "tokens_cache_write": 955608, "tokens_input": 3741, "tokens_output": 39434, "tool_calls": 327, "tool_calls_by_class": { "cc_builtin": 257, "subagent": 11, "vendor_mcp": 59 }, "tool_result_tokens": 297170, "tool_time_ms": 32174, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9203631000000003, "cost_usd": 2.9203631000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 503022, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 259, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 23, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11180141, "tokens_cache_write": 886103, "tokens_input": 1771, "tokens_output": 36222, "tool_calls": 275, "tool_calls_by_class": { "cc_builtin": 212, "subagent": 7, "vendor_mcp": 56 }, "tool_result_tokens": 265085, "tool_time_ms": 28206, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0564147000000004, "cost_usd": 2.0564147000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 553868, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 186, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8664952, "tokens_cache_write": 532546, "tokens_input": 1234, "tokens_output": 28936, "tool_calls": 191, "tool_calls_by_class": { "cc_builtin": 144, "harness_builtin": 2, "skill": 1, "subagent": 1, "vendor_mcp": 43 }, "tool_result_tokens": 177561, "tool_time_ms": 17039, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2878846499999996, "cost_usd": 2.2878846499999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 560032, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 204, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9527604, "tokens_cache_write": 608617, "tokens_input": 3391, "tokens_output": 30976, "tool_calls": 227, "tool_calls_by_class": { "cc_builtin": 167, "harness_builtin": 1, "subagent": 5, "vendor_mcp": 54 }, "tool_result_tokens": 228164, "tool_time_ms": 21309, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1145334, "cost_usd": 2.1145334, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 542364, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 200, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9075054, "tokens_cache_write": 570717, "tokens_input": 3245, "tokens_output": 28862, "tool_calls": 215, "tool_calls_by_class": { "cc_builtin": 168, "harness_builtin": 1, "subagent": 4, "vendor_mcp": 42 }, "tool_result_tokens": 206469, "tool_time_ms": 21444, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5478627, "cost_usd": 2.5478627, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 628931, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 252, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11842427, "tokens_cache_write": 643572, "tokens_input": 1655, "tokens_output": 34236, "tool_calls": 271, "tool_calls_by_class": { "cc_builtin": 207, "harness_builtin": 3, "subagent": 4, "vendor_mcp": 57 }, "tool_result_tokens": 229937, "tool_time_ms": 22556, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4991986, "cost_usd": 2.4991986, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 576444, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 230, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10694446, "tokens_cache_write": 657653, "tokens_input": 2134, "tokens_output": 33425, "tool_calls": 274, "tool_calls_by_class": { "cc_builtin": 200, "harness_builtin": 2, "subagent": 6, "vendor_mcp": 66 }, "tool_result_tokens": 261824, "tool_time_ms": 26531, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7878128, "cost_usd": 2.7878128, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 489127, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 250, "model": "claude-haiku-4-5", "path": "runs/gitnexus/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11352978, "tokens_cache_write": 828044, "tokens_input": 3694, "tokens_output": 35104, "tool_calls": 302, "tool_calls_by_class": { "cc_builtin": 247, "harness_builtin": 1, "subagent": 6, "vendor_mcp": 48 }, "tool_result_tokens": 284896, "tool_time_ms": 33146, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.098681999999997, "cost_usd": 7.098681999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 443239, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 132, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5533214, "tokens_cache_write": 420635, "tokens_input": 4101, "tokens_output": 28630, "tool_calls": 253, "tool_calls_by_class": { "cc_builtin": 237, "subagent": 15, "vendor_mcp": 1 }, "tool_result_tokens": 175056, "tool_time_ms": 15517, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.032093500000004, "cost_usd": 9.032093500000004, "denominator": 20, "duration_basis": "run", "duration_ms": 441462, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 162, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9146612, "tokens_cache_write": 391162, "tokens_input": 194, "tokens_output": 34202, "tool_calls": 340, "tool_calls_by_class": { "cc_builtin": 311, "subagent": 11, "vendor_mcp": 18 }, "tool_result_tokens": 186117, "tool_time_ms": 20480, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.7777272500000025, "cost_usd": 6.7777272500000025, "denominator": 20, "duration_basis": "run", "duration_ms": 448018, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 150, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6127337, "tokens_cache_write": 337701, "tokens_input": 1772, "tokens_output": 26193, "tool_calls": 281, "tool_calls_by_class": { "cc_builtin": 250, "subagent": 14, "vendor_mcp": 17 }, "tool_result_tokens": 158759, "tool_time_ms": 24267, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.184261250000002, "cost_usd": 6.184261250000002, "denominator": 20, "duration_basis": "run", "duration_ms": 466305, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 121, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5420405, "tokens_cache_write": 305708, "tokens_input": 2645, "tokens_output": 21764, "tool_calls": 253, "tool_calls_by_class": { "cc_builtin": 245, "subagent": 8 }, "tool_result_tokens": 134745, "tool_time_ms": 69506, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.838960499999997, "cost_usd": 9.838960499999997, "denominator": 20, "duration_basis": "run", "duration_ms": 473770, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 186, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9036041, "tokens_cache_write": 455725, "tokens_input": 3454, "tokens_output": 41793, "tool_calls": 390, "tool_calls_by_class": { "cc_builtin": 371, "subagent": 17, "vendor_mcp": 2 }, "tool_result_tokens": 199802, "tool_time_ms": 26801, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.7281024999999985, "cost_usd": 6.7281024999999985, "denominator": 20, "duration_basis": "run", "duration_ms": 348878, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5257145, "tokens_cache_write": 388459, "tokens_input": 179, "tokens_output": 22093, "tool_calls": 238, "tool_calls_by_class": { "cc_builtin": 215, "subagent": 14, "vendor_mcp": 9 }, "tool_result_tokens": 174208, "tool_time_ms": 16420, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.188151250000004, "cost_usd": 11.188151250000004, "denominator": 20, "duration_basis": "run", "duration_ms": 590854, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 180, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10250435, "tokens_cache_write": 520559, "tokens_input": 7065, "tokens_output": 41680, "tool_calls": 347, "tool_calls_by_class": { "cc_builtin": 328, "subagent": 13, "vendor_mcp": 6 }, "tool_result_tokens": 190951, "tool_time_ms": 84237, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.854016499999999, "cost_usd": 5.854016499999999, "denominator": 20, "duration_basis": "run", "duration_ms": 530692, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 113, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5112798, "tokens_cache_write": 293168, "tokens_input": 1926, "tokens_output": 17665, "tool_calls": 266, "tool_calls_by_class": { "cc_builtin": 256, "subagent": 8, "vendor_mcp": 2 }, "tool_result_tokens": 131287, "tool_time_ms": 21140, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.208214999999999, "cost_usd": 7.208214999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 525221, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 176, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7287320, "tokens_cache_write": 318980, "tokens_input": 222, "tokens_output": 31183, "tool_calls": 272, "tool_calls_by_class": { "cc_builtin": 247, "subagent": 12, "vendor_mcp": 13 }, "tool_result_tokens": 137058, "tool_time_ms": 20720, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.767031500000002, "cost_usd": 6.767031500000002, "denominator": 20, "duration_basis": "run", "duration_ms": 562553, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 159, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7188738, "tokens_cache_write": 277111, "tokens_input": 982, "tokens_output": 25279, "tool_calls": 253, "tool_calls_by_class": { "cc_builtin": 242, "subagent": 8, "vendor_mcp": 3 }, "tool_result_tokens": 143937, "tool_time_ms": 29730, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.818173249999999, "cost_usd": 6.818173249999999, "denominator": 30, "duration_basis": "run", "duration_ms": 354376, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 105, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4557269, "tokens_cache_write": 444878, "tokens_input": 3966, "tokens_output": 31835, "tool_calls": 306, "tool_calls_by_class": { "cc_builtin": 293, "subagent": 12, "vendor_mcp": 1 }, "tool_result_tokens": 196577, "tool_time_ms": 48826, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.687842500000002, "cost_usd": 6.687842500000002, "denominator": 30, "duration_basis": "run", "duration_ms": 294982, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4824215, "tokens_cache_write": 404695, "tokens_input": 4712, "tokens_output": 29114, "tool_calls": 274, "tool_calls_by_class": { "cc_builtin": 263, "subagent": 11 }, "tool_result_tokens": 175192, "tool_time_ms": 71413, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.218373499999998, "cost_usd": 6.218373499999998, "denominator": 30, "duration_basis": "run", "duration_ms": 302742, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5544297, "tokens_cache_write": 345817, "tokens_input": 114, "tokens_output": 21217, "tool_calls": 249, "tool_calls_by_class": { "cc_builtin": 236, "subagent": 6, "vendor_mcp": 7 }, "tool_result_tokens": 179851, "tool_time_ms": 17876, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.26309475, "cost_usd": 7.26309475, "denominator": 30, "duration_basis": "run", "duration_ms": 291685, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 143, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5535082, "tokens_cache_write": 437583, "tokens_input": 1519, "tokens_output": 28177, "tool_calls": 315, "tool_calls_by_class": { "cc_builtin": 294, "subagent": 18, "vendor_mcp": 3 }, "tool_result_tokens": 186312, "tool_time_ms": 34711, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.127665250000001, "cost_usd": 5.127665250000001, "denominator": 30, "duration_basis": "run", "duration_ms": 280404, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 72, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3810708, "tokens_cache_write": 321916, "tokens_input": 5803, "tokens_output": 19209, "tool_calls": 237, "tool_calls_by_class": { "cc_builtin": 231, "subagent": 6 }, "tool_result_tokens": 118320, "tool_time_ms": 78074, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4519892499999996, "cost_usd": 3.4519892499999996, "denominator": 30, "duration_basis": "run", "duration_ms": 374598, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 32, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1634741, "tokens_cache_write": 284299, "tokens_input": 748, "tokens_output": 17452, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 134, "subagent": 3 }, "tool_result_tokens": 99927, "tool_time_ms": 30001, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5338895, "cost_usd": 5.5338895, "denominator": 30, "duration_basis": "run", "duration_ms": 578460, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 135, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5862634, "tokens_cache_write": 237992, "tokens_input": 2718, "tokens_output": 25190, "tool_calls": 216, "tool_calls_by_class": { "cc_builtin": 210, "subagent": 6 }, "tool_result_tokens": 102265, "tool_time_ms": 45608, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.672599, "cost_usd": 6.672599, "denominator": 30, "duration_basis": "run", "duration_ms": 457264, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 142, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5871853, "tokens_cache_write": 349996, "tokens_input": 4425, "tokens_output": 31303, "tool_calls": 293, "tool_calls_by_class": { "cc_builtin": 281, "subagent": 12 }, "tool_result_tokens": 165998, "tool_time_ms": 44397, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.35112675, "cost_usd": 8.35112675, "denominator": 30, "duration_basis": "run", "duration_ms": 424427, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 127, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6586756, "tokens_cache_write": 469943, "tokens_input": 2235, "tokens_output": 31644, "tool_calls": 354, "tool_calls_by_class": { "cc_builtin": 345, "subagent": 9 }, "tool_result_tokens": 184272, "tool_time_ms": 76689, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.585263999999998, "cost_usd": 7.585263999999998, "denominator": 30, "duration_basis": "run", "duration_ms": 329182, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 126, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6364738, "tokens_cache_write": 442786, "tokens_input": 3622, "tokens_output": 27948, "tool_calls": 340, "tool_calls_by_class": { "cc_builtin": 316, "subagent": 11, "vendor_mcp": 13 }, "tool_result_tokens": 205503, "tool_time_ms": 21209, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.795649000000001, "cost_usd": 7.795649000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 822733, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11066038, "tokens_cache_write": 173509, "tokens_input": 183, "tokens_output": 21065, "tool_calls": 114, "tool_calls_by_class": { "cc_builtin": 107, "vendor_mcp": 7 }, "tool_result_tokens": 43072, "tool_time_ms": 26451, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.753751500000001, "cost_usd": 6.753751500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 635483, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 105, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10223383, "tokens_cache_write": 122048, "tokens_input": 161, "tokens_output": 16831, "tool_calls": 100, "tool_calls_by_class": { "cc_builtin": 94, "vendor_mcp": 6 }, "tool_result_tokens": 47884, "tool_time_ms": 21061, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.421833, "cost_usd": 6.421833, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 658227, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 105, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9996976, "tokens_cache_write": 101097, "tokens_input": 165, "tokens_output": 16462, "tool_calls": 96, "tool_calls_by_class": { "cc_builtin": 88, "vendor_mcp": 8 }, "tool_result_tokens": 49413, "tool_time_ms": 21774, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.261747000000002, "cost_usd": 9.261747000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 950921, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12160414, "tokens_cache_write": 265700, "tokens_input": 543, "tokens_output": 20873, "tool_calls": 120, "tool_calls_by_class": { "cc_builtin": 118, "vendor_mcp": 2 }, "tool_result_tokens": 49167, "tool_time_ms": 36758, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.8852400000000005, "cost_usd": 7.8852400000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 813651, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 117, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10798600, "tokens_cache_write": 198558, "tokens_input": 527, "tokens_output": 19909, "tool_calls": 113, "tool_calls_by_class": { "cc_builtin": 111, "vendor_mcp": 2 }, "tool_result_tokens": 46811, "tool_time_ms": 32152, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.5405765, "cost_usd": 7.5405765, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 800974, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11757663, "tokens_cache_write": 115660, "tokens_input": 514, "tokens_output": 20103, "tool_calls": 119, "tool_calls_by_class": { "cc_builtin": 112, "vendor_mcp": 7 }, "tool_result_tokens": 41398, "tool_time_ms": 25520, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.263809999999999, "cost_usd": 7.263809999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 897018, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 115, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11295550, "tokens_cache_write": 114270, "tokens_input": 222, "tokens_output": 18889, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 105, "vendor_mcp": 4 }, "tool_result_tokens": 46732, "tool_time_ms": 23917, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.118268, "cost_usd": 8.118268, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 845982, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 120, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12560276, "tokens_cache_write": 127046, "tokens_input": 524, "tokens_output": 22602, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 106, "vendor_mcp": 5 }, "tool_result_tokens": 57294, "tool_time_ms": 24302, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.070389499999999, "cost_usd": 6.070389499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 627829, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8263239, "tokens_cache_write": 152793, "tokens_input": 148, "tokens_output": 16404, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 77, "vendor_mcp": 6 }, "tool_result_tokens": 42256, "tool_time_ms": 17901, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.206922499999999, "cost_usd": 6.206922499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 811287, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 115, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9964785, "tokens_cache_write": 75239, "tokens_input": 1483, "tokens_output": 18589, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 98, "vendor_mcp": 8 }, "tool_result_tokens": 35685, "tool_time_ms": 21078, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.40739, "cost_usd": 4.40739, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 644143, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6332830, "tokens_cache_write": 94548, "tokens_input": 184, "tokens_output": 11783, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 85 }, "tool_result_tokens": 17253, "tool_time_ms": 34755, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.095785000000001, "cost_usd": 4.095785000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 589693, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 109, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6536600, "tokens_cache_write": 48984, "tokens_input": 179, "tokens_output": 13470, "tool_calls": 94, "tool_calls_by_class": { "cc_builtin": 94 }, "tool_result_tokens": 14303, "tool_time_ms": 35362, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.765322500000001, "cost_usd": 3.765322500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 503694, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5965005, "tokens_cache_write": 52003, "tokens_input": 173, "tokens_output": 10477, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 84 }, "tool_result_tokens": 16296, "tool_time_ms": 31430, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9465824999999994, "cost_usd": 3.9465824999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 523761, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 98, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6079445, "tokens_cache_write": 62560, "tokens_input": 172, "tokens_output": 11216, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 83 }, "tool_result_tokens": 20221, "tool_time_ms": 29588, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.982377, "cost_usd": 3.982377, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 539198, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6382094, "tokens_cache_write": 50998, "tokens_input": 170, "tokens_output": 11220, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 85 }, "tool_result_tokens": 19130, "tool_time_ms": 28961, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.136266999999998, "cost_usd": 4.136266999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 622665, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6535164, "tokens_cache_write": 57028, "tokens_input": 216, "tokens_output": 11893, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 89 }, "tool_result_tokens": 21849, "tool_time_ms": 33996, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7275995, "cost_usd": 3.7275995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 628928, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 100, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6004319, "tokens_cache_write": 46174, "tokens_input": 170, "tokens_output": 10514, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 76 }, "tool_result_tokens": 18156, "tool_time_ms": 27272, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9282605, "cost_usd": 3.9282605, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 481696, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6246151, "tokens_cache_write": 50648, "tokens_input": 171, "tokens_output": 11914, "tool_calls": 91, "tool_calls_by_class": { "cc_builtin": 91 }, "tool_result_tokens": 18468, "tool_time_ms": 34882, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.611047000000001, "cost_usd": 4.611047000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 560063, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 24, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6681314, "tokens_cache_write": 96680, "tokens_input": 268, "tokens_output": 12090, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 83 }, "tool_result_tokens": 24625, "tool_time_ms": 26573, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7792605000000004, "cost_usd": 3.7792605000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 548989, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5953261, "tokens_cache_write": 51379, "tokens_input": 163, "tokens_output": 11521, "tool_calls": 87, "tool_calls_by_class": { "cc_builtin": 87 }, "tool_result_tokens": 20103, "tool_time_ms": 32398, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.328272250000001, "cost_usd": 9.328272250000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1347608, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 146, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6662202, "tokens_cache_write": 574081, "tokens_input": 11583, "tokens_output": 33653, "tool_calls": 323, "tool_calls_by_class": { "cc_builtin": 290, "subagent": 5, "vendor_mcp": 28 }, "tool_result_tokens": 214495, "tool_time_ms": 36185, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.12613875, "cost_usd": 8.12613875, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1101863, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 129, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6010695, "tokens_cache_write": 476230, "tokens_input": 633, "tokens_output": 28755, "tool_calls": 264, "tool_calls_by_class": { "cc_builtin": 229, "subagent": 3, "vendor_mcp": 32 }, "tool_result_tokens": 175471, "tool_time_ms": 27899, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.551491499999999, "cost_usd": 7.551491499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1129930, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 132, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6107323, "tokens_cache_write": 404358, "tokens_input": 5343, "tokens_output": 30054, "tool_calls": 252, "tool_calls_by_class": { "cc_builtin": 221, "subagent": 3, "vendor_mcp": 28 }, "tool_result_tokens": 151244, "tool_time_ms": 23481, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.193844499999999, "cost_usd": 9.193844499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1479319, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 166, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7589169, "tokens_cache_write": 481645, "tokens_input": 4398, "tokens_output": 35180, "tool_calls": 332, "tool_calls_by_class": { "cc_builtin": 294, "subagent": 4, "vendor_mcp": 34 }, "tool_result_tokens": 173848, "tool_time_ms": 34414, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.74688375, "cost_usd": 6.74688375, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1037338, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 121, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5586860, "tokens_cache_write": 338487, "tokens_input": 244, "tokens_output": 30673, "tool_calls": 223, "tool_calls_by_class": { "cc_builtin": 189, "subagent": 2, "vendor_mcp": 32 }, "tool_result_tokens": 131258, "tool_time_ms": 21079, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.4032355, "cost_usd": 7.4032355, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1073583, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 121, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5620036, "tokens_cache_write": 425177, "tokens_input": 4397, "tokens_output": 28084, "tool_calls": 235, "tool_calls_by_class": { "cc_builtin": 203, "subagent": 3, "vendor_mcp": 29 }, "tool_result_tokens": 177724, "tool_time_ms": 22042, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.557412250000002, "cost_usd": 7.557412250000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1147792, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 131, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6013407, "tokens_cache_write": 415048, "tokens_input": 3517, "tokens_output": 30133, "tool_calls": 272, "tool_calls_by_class": { "cc_builtin": 245, "subagent": 3, "vendor_mcp": 24 }, "tool_result_tokens": 165246, "tool_time_ms": 28852, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.339547999999999, "cost_usd": 8.339547999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1232489, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 136, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6649546, "tokens_cache_write": 485538, "tokens_input": 1683, "tokens_output": 32070, "tool_calls": 253, "tool_calls_by_class": { "cc_builtin": 220, "subagent": 4, "vendor_mcp": 29 }, "tool_result_tokens": 195528, "tool_time_ms": 33243, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.640737250000001, "cost_usd": 7.640737250000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1130275, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5829617, "tokens_cache_write": 439448, "tokens_input": 254, "tokens_output": 27442, "tool_calls": 247, "tool_calls_by_class": { "cc_builtin": 209, "subagent": 3, "vendor_mcp": 35 }, "tool_result_tokens": 185175, "tool_time_ms": 20676, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.8881795, "cost_usd": 7.8881795, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1250621, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 137, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6393589, "tokens_cache_write": 423423, "tokens_input": 4427, "tokens_output": 34444, "tool_calls": 273, "tool_calls_by_class": { "cc_builtin": 237, "subagent": 3, "vendor_mcp": 33 }, "tool_result_tokens": 171576, "tool_time_ms": 25567, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.6341234999999985, "cost_usd": 7.6341234999999985, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 909177, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 130, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5725917, "tokens_cache_write": 420317, "tokens_input": 3468, "tokens_output": 25558, "tool_calls": 189, "tool_calls_by_class": { "cc_builtin": 161, "subagent": 1, "vendor_mcp": 27 }, "tool_result_tokens": 127527, "tool_time_ms": 53983, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.190562250000001, "cost_usd": 8.190562250000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1048449, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 142, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6029372, "tokens_cache_write": 491586, "tokens_input": 2966, "tokens_output": 26309, "tool_calls": 222, "tool_calls_by_class": { "cc_builtin": 194, "subagent": 4, "vendor_mcp": 24 }, "tool_result_tokens": 157939, "tool_time_ms": 45814, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.242806750000001, "cost_usd": 7.242806750000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 988602, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 135, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5676481, "tokens_cache_write": 422878, "tokens_input": 1159, "tokens_output": 25419, "tool_calls": 215, "tool_calls_by_class": { "cc_builtin": 192, "subagent": 4, "vendor_mcp": 19 }, "tool_result_tokens": 124132, "tool_time_ms": 54316, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.43182475, "cost_usd": 7.43182475, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 964966, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 134, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5846602, "tokens_cache_write": 419212, "tokens_input": 2188, "tokens_output": 25748, "tool_calls": 210, "tool_calls_by_class": { "cc_builtin": 182, "subagent": 3, "vendor_mcp": 25 }, "tool_result_tokens": 151743, "tool_time_ms": 49584, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.034665500000001, "cost_usd": 8.034665500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1049481, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 139, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5910066, "tokens_cache_write": 488406, "tokens_input": 944, "tokens_output": 27234, "tool_calls": 240, "tool_calls_by_class": { "cc_builtin": 215, "subagent": 4, "vendor_mcp": 21 }, "tool_result_tokens": 168215, "tool_time_ms": 50734, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.2685794999999995, "cost_usd": 7.2685794999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 955329, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 127, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5461354, "tokens_cache_write": 420230, "tokens_input": 4290, "tokens_output": 25873, "tool_calls": 204, "tool_calls_by_class": { "cc_builtin": 174, "subagent": 3, "vendor_mcp": 27 }, "tool_result_tokens": 139965, "tool_time_ms": 45205, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.499937499999998, "cost_usd": 7.499937499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 966525, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 143, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6474885, "tokens_cache_write": 378324, "tokens_input": 1975, "tokens_output": 27557, "tool_calls": 219, "tool_calls_by_class": { "cc_builtin": 189, "subagent": 2, "vendor_mcp": 28 }, "tool_result_tokens": 143981, "tool_time_ms": 36809, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.8298705, "cost_usd": 7.8298705, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 982367, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 132, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5681351, "tokens_cache_write": 469437, "tokens_input": 2093, "tokens_output": 25585, "tool_calls": 220, "tool_calls_by_class": { "cc_builtin": 191, "subagent": 3, "vendor_mcp": 26 }, "tool_result_tokens": 166154, "tool_time_ms": 34793, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.214662750000001, "cost_usd": 6.214662750000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 791386, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 119, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5247748, "tokens_cache_write": 306448, "tokens_input": 1417, "tokens_output": 24241, "tool_calls": 176, "tool_calls_by_class": { "cc_builtin": 155, "subagent": 1, "vendor_mcp": 20 }, "tool_result_tokens": 112961, "tool_time_ms": 33115, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.614705, "cost_usd": 6.614705, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 963922, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 125, "model": "claude-opus-4-6", "path": "runs/gitnexus/claude-opus-4-6/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5475190, "tokens_cache_write": 350822, "tokens_input": 442, "tokens_output": 25373, "tool_calls": 204, "tool_calls_by_class": { "cc_builtin": 177, "subagent": 2, "vendor_mcp": 25 }, "tool_result_tokens": 130478, "tool_time_ms": 30523, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.74084825, "cost_usd": 6.74084825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 696105, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8067808, "tokens_cache_write": 198996, "tokens_input": 177, "tokens_output": 36969, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 77, "subagent": 4, "vendor_mcp": 4 }, "tool_result_tokens": 35689, "tool_time_ms": 26522, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.3899057500000005, "cost_usd": 6.3899057500000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 981738, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6269304, "tokens_cache_write": 230101, "tokens_input": 194, "tokens_output": 37529, "tool_calls": 114, "tool_calls_by_class": { "cc_builtin": 104, "subagent": 9, "vendor_mcp": 1 }, "tool_result_tokens": 62626, "tool_time_ms": 37057, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.921902, "cost_usd": 5.921902, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 661047, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 77, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7585404, "tokens_cache_write": 115067, "tokens_input": 141, "tokens_output": 39113, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 58, "vendor_mcp": 9 }, "tool_result_tokens": 30076, "tool_time_ms": 21396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.267488249999999, "cost_usd": 7.267488249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 782360, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9047084, "tokens_cache_write": 179811, "tokens_input": 178, "tokens_output": 40653, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 71, "subagent": 2, "vendor_mcp": 1 }, "tool_result_tokens": 37211, "tool_time_ms": 26392, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.947931500000001, "cost_usd": 4.947931500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 543505, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 71, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6275183, "tokens_cache_write": 109819, "tokens_input": 135, "tokens_output": 28459, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 50, "vendor_mcp": 6 }, "tool_result_tokens": 30855, "tool_time_ms": 20268, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.837919, "cost_usd": 5.837919, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 888959, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5768818, "tokens_cache_write": 208134, "tokens_input": 2039, "tokens_output": 34826, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 100, "subagent": 9, "vendor_mcp": 2 }, "tool_result_tokens": 51862, "tool_time_ms": 35344, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.727939749999999, "cost_usd": 4.727939749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 502985, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 83, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6183127, "tokens_cache_write": 100123, "tokens_input": 159, "tokens_output": 29939, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 56, "subagent": 1, "vendor_mcp": 6 }, "tool_result_tokens": 21247, "tool_time_ms": 25210, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.955733000000002, "cost_usd": 7.955733000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1422526, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 120, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7834266, "tokens_cache_write": 320763, "tokens_input": 3027, "tokens_output": 37852, "tool_calls": 132, "tool_calls_by_class": { "cc_builtin": 112, "subagent": 9, "vendor_mcp": 11 }, "tool_result_tokens": 90450, "tool_time_ms": 33795, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.335481250000001, "cost_usd": 7.335481250000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 656143, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 76, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7467455, "tokens_cache_write": 274799, "tokens_input": 143, "tokens_output": 37688, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 56, "subagent": 2, "vendor_mcp": 5 }, "tool_result_tokens": 41121, "tool_time_ms": 28810, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.744141249999999, "cost_usd": 6.744141249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1029154, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 115, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6954835, "tokens_cache_write": 221167, "tokens_input": 646, "tokens_output": 40384, "tool_calls": 126, "tool_calls_by_class": { "cc_builtin": 116, "subagent": 9, "vendor_mcp": 1 }, "tool_result_tokens": 61798, "tool_time_ms": 41389, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1450724999999995, "cost_usd": 4.1450724999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 401002, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5566305, "tokens_cache_write": 90525, "tokens_input": 169, "tokens_output": 18233, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 11261, "tool_time_ms": 24802, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7208760000000005, "cost_usd": 3.7208760000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 377934, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5504422, "tokens_cache_write": 54189, "tokens_input": 160, "tokens_output": 17039, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 12998, "tool_time_ms": 21850, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.872410999999999, "cost_usd": 3.872410999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 541721, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5619392, "tokens_cache_write": 55256, "tokens_input": 166, "tokens_output": 20373, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 12335, "tool_time_ms": 22633, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5736455, "cost_usd": 3.5736455, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 384651, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5266911, "tokens_cache_write": 50929, "tokens_input": 160, "tokens_output": 17204, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 10893, "tool_time_ms": 25071, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.089173, "cost_usd": 4.089173, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 456698, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5925936, "tokens_cache_write": 60966, "tokens_input": 164, "tokens_output": 20629, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 14873, "tool_time_ms": 23537, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9146490000000003, "cost_usd": 3.9146490000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 427057, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5623748, "tokens_cache_write": 57355, "tokens_input": 170, "tokens_output": 21135, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 10695, "tool_time_ms": 25522, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.203307499999999, "cost_usd": 4.203307499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 456504, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6245145, "tokens_cache_write": 55036, "tokens_input": 185, "tokens_output": 21178, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 12527, "tool_time_ms": 31701, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.73704, "cost_usd": 3.73704, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 414431, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5504700, "tokens_cache_write": 51176, "tokens_input": 156, "tokens_output": 18886, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 14322, "tool_time_ms": 23831, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1873649999999993, "cost_usd": 3.1873649999999993, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 363194, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 78, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4735020, "tokens_cache_write": 40271, "tokens_input": 149, "tokens_output": 16656, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 48 }, "tool_result_tokens": 10444, "tool_time_ms": 20770, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.3521065000000005, "cost_usd": 4.3521065000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 453883, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6232013, "tokens_cache_write": 68157, "tokens_input": 166, "tokens_output": 22148, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 15358, "tool_time_ms": 27024, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.90605525, "cost_usd": 0.90605525, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 42385, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 13, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 483638, "tokens_cache_write": 64572, "tokens_input": 26, "tokens_output": 5219, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 12, "subagent": 1, "vendor_mcp": 1 }, "tool_result_tokens": 11301, "tool_time_ms": 13370, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.828246, "cost_usd": 0.828246, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 85318, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 365877, "tokens_cache_write": 69083, "tokens_input": 20, "tokens_output": 2923, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "subagent": 1, "vendor_mcp": 1 }, "tool_result_tokens": 7811, "tool_time_ms": 11681, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.503139, "cost_usd": 0.503139, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 50077, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 8, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 343968, "tokens_cache_write": 26730, "tokens_input": 16, "tokens_output": 2551, "tool_calls": 8, "tool_calls_by_class": { "cc_builtin": 7, "vendor_mcp": 1 }, "tool_result_tokens": 5981, "tool_time_ms": 2965, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 1.16981975, "cost_usd": 1.16981975, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 55371, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 19, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 674842, "tokens_cache_write": 92300, "tokens_input": 38, "tokens_output": 4140, "tool_calls": 18, "tool_calls_by_class": { "cc_builtin": 16, "subagent": 2 }, "tool_result_tokens": 8478, "tool_time_ms": 5970, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.75804525, "cost_usd": 0.75804525, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 89001, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 405758, "tokens_cache_write": 57992, "tokens_input": 22, "tokens_output": 4139, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 9, "subagent": 1 }, "tool_result_tokens": 5683, "tool_time_ms": 3042, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.81131325, "cost_usd": 0.81131325, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 75217, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 418749, "tokens_cache_write": 63956, "tokens_input": 1674, "tokens_output": 3956, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 8, "subagent": 1, "vendor_mcp": 1 }, "tool_result_tokens": 7936, "tool_time_ms": 2406, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.6606825, "cost_usd": 0.6606825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 62092, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 9, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 340495, "tokens_cache_write": 53000, "tokens_input": 18, "tokens_output": 3050, "tool_calls": 11, "tool_calls_by_class": { "cc_builtin": 8, "subagent": 1, "vendor_mcp": 2 }, "tool_result_tokens": 9403, "tool_time_ms": 2592, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.5142880000000001, "cost_usd": 0.5142880000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 44872, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 271226, "tokens_cache_write": 31584, "tokens_input": 12, "tokens_output": 2511, "tool_calls": 8, "tool_calls_by_class": { "cc_builtin": 5, "vendor_mcp": 3 }, "tool_result_tokens": 9907, "tool_time_ms": 1549, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.7293260000000001, "cost_usd": 0.7293260000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 80284, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 388007, "tokens_cache_write": 54457, "tokens_input": 20, "tokens_output": 2906, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 10, "subagent": 1, "vendor_mcp": 1 }, "tool_result_tokens": 9095, "tool_time_ms": 1539, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 1.12724175, "cost_usd": 1.12724175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 5982, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 16, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 552641, "tokens_cache_write": 90096, "tokens_input": 32, "tokens_output": 4447, "tool_calls": 18, "tool_calls_by_class": { "cc_builtin": 14, "subagent": 2, "vendor_mcp": 2 }, "tool_result_tokens": 14087, "tool_time_ms": 2001, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.761536, "cost_usd": 8.761536, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 983483, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 134, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|11", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5665862, "tokens_cache_write": 549080, "tokens_input": 2892, "tokens_output": 41357, "tool_calls": 155, "tool_calls_by_class": { "cc_builtin": 136, "other": 1, "subagent": 7, "vendor_mcp": 11 }, "tool_result_tokens": 89804, "tool_time_ms": 128539, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.683344500000002, "cost_usd": 9.683344500000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 996433, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 159, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|12", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7124514, "tokens_cache_write": 504860, "tokens_input": 6373, "tokens_output": 60436, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 161, "subagent": 6, "vendor_mcp": 17 }, "tool_result_tokens": 99071, "tool_time_ms": 90297, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.5388485, "cost_usd": 9.5388485, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 748585, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 160, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|13", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6839832, "tokens_cache_write": 501390, "tokens_input": 1678, "tokens_output": 53755, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 165, "other": 1, "subagent": 8, "vendor_mcp": 8 }, "tool_result_tokens": 98657, "tool_time_ms": 84851, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.955891, "cost_usd": 8.955891, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 865092, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 140, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|14", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6215232, "tokens_cache_write": 471255, "tokens_input": 5068, "tokens_output": 49038, "tool_calls": 170, "tool_calls_by_class": { "cc_builtin": 147, "subagent": 4, "vendor_mcp": 19 }, "tool_result_tokens": 93743, "tool_time_ms": 64229, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.5884675, "cost_usd": 8.5884675, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 744848, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 134, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|15", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5729200, "tokens_cache_write": 494791, "tokens_input": 696, "tokens_output": 50488, "tool_calls": 162, "tool_calls_by_class": { "cc_builtin": 138, "subagent": 6, "vendor_mcp": 18 }, "tool_result_tokens": 94231, "tool_time_ms": 41133, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.083117499999997, "cost_usd": 10.083117499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 933618, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 163, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep16", "query_calls": 0, "rep": 16, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|16", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6620178, "tokens_cache_write": 634656, "tokens_input": 1610, "tokens_output": 55767, "tool_calls": 212, "tool_calls_by_class": { "cc_builtin": 169, "subagent": 14, "vendor_mcp": 29 }, "tool_result_tokens": 106643, "tool_time_ms": 68065, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.53197925, "cost_usd": 9.53197925, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1112399, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 139, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep17", "query_calls": 0, "rep": 17, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|17", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5992321, "tokens_cache_write": 597065, "tokens_input": 4811, "tokens_output": 52324, "tool_calls": 164, "tool_calls_by_class": { "cc_builtin": 127, "other": 1, "subagent": 11, "vendor_mcp": 25 }, "tool_result_tokens": 112841, "tool_time_ms": 52215, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.167288, "cost_usd": 9.167288, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1057128, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 138, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep18", "query_calls": 0, "rep": 18, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|18", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5841451, "tokens_cache_write": 579274, "tokens_input": 336, "tokens_output": 50650, "tool_calls": 173, "tool_calls_by_class": { "cc_builtin": 145, "subagent": 8, "vendor_mcp": 20 }, "tool_result_tokens": 106841, "tool_time_ms": 62015, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.583364499999998, "cost_usd": 9.583364499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1040235, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 145, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep19", "query_calls": 0, "rep": 19, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|19", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6337214, "tokens_cache_write": 578317, "tokens_input": 9981, "tokens_output": 53051, "tool_calls": 183, "tool_calls_by_class": { "cc_builtin": 146, "subagent": 9, "vendor_mcp": 28 }, "tool_result_tokens": 113487, "tool_time_ms": 51016, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.95732775, "cost_usd": 8.95732775, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1067127, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 140, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos-hard/rep20", "query_calls": 0, "rep": 20, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|20", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6074098, "tokens_cache_write": 523908, "tokens_input": 759, "tokens_output": 46690, "tool_calls": 179, "tool_calls_by_class": { "cc_builtin": 144, "other": 1, "subagent": 8, "vendor_mcp": 26 }, "tool_result_tokens": 101432, "tool_time_ms": 45151, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.9739935, "cost_usd": 0.9739935, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 48877, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 9, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 264442, "tokens_cache_write": 92031, "tokens_input": 87, "tokens_output": 1418, "tool_calls": 7, "tool_calls_by_class": { "cc_builtin": 6, "subagent": 1 }, "tool_result_tokens": 903, "tool_time_ms": 11318, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.34211400000000003, "cost_usd": 0.34211400000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 34054, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 243368, "tokens_cache_write": 19752, "tokens_input": 12, "tokens_output": 914, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 878, "tool_time_ms": 5199, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.332392, "cost_usd": 0.332392, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 36976, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 243294, "tokens_cache_write": 18961, "tokens_input": 12, "tokens_output": 843, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 652, "tool_time_ms": 9492, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.3939895, "cost_usd": 0.3939895, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 29002, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 188699, "tokens_cache_write": 27904, "tokens_input": 10, "tokens_output": 822, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 866, "tool_time_ms": 10639, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.3295055, "cost_usd": 0.3295055, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 36544, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 243281, "tokens_cache_write": 18713, "tokens_input": 12, "tokens_output": 827, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 478, "tool_time_ms": 12837, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.3764035, "cost_usd": 0.3764035, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 39234, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 7, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 287447, "tokens_cache_write": 21056, "tokens_input": 14, "tokens_output": 882, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 774, "tool_time_ms": 14021, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2869005, "cost_usd": 0.2869005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 37651, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 247821, "tokens_cache_write": 14288, "tokens_input": 12, "tokens_output": 802, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 568, "tool_time_ms": 11926, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.6776825, "cost_usd": 0.6776825, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 29078, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 7, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 270755, "tokens_cache_write": 52021, "tokens_input": 14, "tokens_output": 881, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 450, "tool_time_ms": 790, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.270785, "cost_usd": 0.270785, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 26963, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 7, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 314750, "tokens_cache_write": 9119, "tokens_input": 14, "tokens_output": 886, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 889, "tool_time_ms": 907, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.31406, "cost_usd": 0.31406, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 26159, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 7, "model": "claude-opus-4-8", "path": "runs/gitnexus/claude-opus-4-8/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 294080, "tokens_cache_write": 14565, "tokens_input": 14, "tokens_output": 852, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 515, "tool_time_ms": 5276, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.011035000000001, "cost_usd": 4.011035000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 331833, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 44, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4236970, "tokens_cache_write": 136544, "tokens_input": 572, "tokens_output": 20970, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 60, "vendor_mcp": 6 }, "tool_result_tokens": 41785, "tool_time_ms": 8692, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9046469999999998, "cost_usd": 2.9046469999999998, "denominator": 20, "duration_basis": "run", "duration_ms": 304812, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 36, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3137504, "tokens_cache_write": 79461, "tokens_input": 6057, "tokens_output": 20440, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 50, "vendor_mcp": 4 }, "tool_result_tokens": 31740, "tool_time_ms": 11993, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.030036, "cost_usd": 3.030036, "denominator": 20, "duration_basis": "run", "duration_ms": 334190, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 35, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3152782, "tokens_cache_write": 88605, "tokens_input": 444, "tokens_output": 22615, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 49, "vendor_mcp": 7 }, "tool_result_tokens": 36184, "tool_time_ms": 11841, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0075324999999995, "cost_usd": 3.0075324999999995, "denominator": 20, "duration_basis": "run", "duration_ms": 333298, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 37, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3258155, "tokens_cache_write": 83787, "tokens_input": 447, "tokens_output": 21534, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 53, "vendor_mcp": 7 }, "tool_result_tokens": 33150, "tool_time_ms": 10653, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4462150000000014, "cost_usd": 3.4462150000000014, "denominator": 20, "duration_basis": "run", "duration_ms": 344104, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 43, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3988670, "tokens_cache_write": 88553, "tokens_input": 815, "tokens_output": 22491, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 52, "vendor_mcp": 6 }, "tool_result_tokens": 35432, "tool_time_ms": 11805, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4096615000000003, "cost_usd": 2.4096615000000003, "denominator": 20, "duration_basis": "run", "duration_ms": 278399, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 28, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2420183, "tokens_cache_write": 68427, "tokens_input": 7945, "tokens_output": 19023, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 41, "vendor_mcp": 6 }, "tool_result_tokens": 32781, "tool_time_ms": 13154, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6811875000000005, "cost_usd": 2.6811875000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 295361, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 29, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2738635, "tokens_cache_write": 80483, "tokens_input": 53, "tokens_output": 20271, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 44, "vendor_mcp": 6 }, "tool_result_tokens": 37928, "tool_time_ms": 13539, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3367364999999993, "cost_usd": 3.3367364999999993, "denominator": 20, "duration_basis": "run", "duration_ms": 345220, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 38, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3529053, "tokens_cache_write": 93635, "tokens_input": 10387, "tokens_output": 23357, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 56, "vendor_mcp": 6 }, "tool_result_tokens": 38142, "tool_time_ms": 14483, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9743005, "cost_usd": 2.9743005, "denominator": 20, "duration_basis": "run", "duration_ms": 304181, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3055431, "tokens_cache_write": 91432, "tokens_input": 1353, "tokens_output": 21020, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 57, "vendor_mcp": 8 }, "tool_result_tokens": 37481, "tool_time_ms": 15164, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.947663, "cost_usd": 2.947663, "denominator": 20, "duration_basis": "run", "duration_ms": 339575, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3166866, "tokens_cache_write": 79963, "tokens_input": 1050, "tokens_output": 22374, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 46, "vendor_mcp": 7 }, "tool_result_tokens": 35859, "tool_time_ms": 13840, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5428155, "cost_usd": 2.5428155, "denominator": 30, "duration_basis": "run", "duration_ms": 232369, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1953751, "tokens_cache_write": 116502, "tokens_input": 44, "tokens_output": 16028, "tool_calls": 44, "tool_calls_by_class": { "cc_builtin": 44 }, "tool_result_tokens": 30545, "tool_time_ms": 14093, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.7839285, "cost_usd": 1.7839285, "denominator": 30, "duration_basis": "run", "duration_ms": 233201, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 23, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1612577, "tokens_cache_write": 58132, "tokens_input": 2654, "tokens_output": 15322, "tool_calls": 43, "tool_calls_by_class": { "cc_builtin": 42, "vendor_mcp": 1 }, "tool_result_tokens": 20277, "tool_time_ms": 18278, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9605425, "cost_usd": 1.9605425, "denominator": 30, "duration_basis": "run", "duration_ms": 212834, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 21, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1687135, "tokens_cache_write": 73311, "tokens_input": 38, "tokens_output": 15347, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 28855, "tool_time_ms": 14644, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2452154999999996, "cost_usd": 2.2452154999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 253693, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 26, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2142091, "tokens_cache_write": 74363, "tokens_input": 1138, "tokens_output": 16994, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 48 }, "tool_result_tokens": 28449, "tool_time_ms": 21865, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5665964999999984, "cost_usd": 3.5665964999999984, "denominator": 30, "duration_basis": "run", "duration_ms": 326199, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 42, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4062863, "tokens_cache_write": 101833, "tokens_input": 3412, "tokens_output": 19991, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 65, "vendor_mcp": 1 }, "tool_result_tokens": 41151, "tool_time_ms": 25872, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.192516000000001, "cost_usd": 2.192516000000001, "denominator": 30, "duration_basis": "run", "duration_ms": 264888, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 27, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2104632, "tokens_cache_write": 70347, "tokens_input": 706, "tokens_output": 17328, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 26004, "tool_time_ms": 19272, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6583715000000003, "cost_usd": 1.6583715000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 217885, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 15, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1151563, "tokens_cache_write": 67940, "tokens_input": 683, "tokens_output": 15991, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 25774, "tool_time_ms": 15201, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.752671, "cost_usd": 1.752671, "denominator": 30, "duration_basis": "run", "duration_ms": 209736, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 19, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1418732, "tokens_cache_write": 65902, "tokens_input": 1142, "tokens_output": 15143, "tool_calls": 34, "tool_calls_by_class": { "cc_builtin": 34 }, "tool_result_tokens": 25336, "tool_time_ms": 15187, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9406780000000003, "cost_usd": 1.9406780000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 227923, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1786306, "tokens_cache_write": 64371, "tokens_input": 1228, "tokens_output": 15907, "tool_calls": 43, "tool_calls_by_class": { "cc_builtin": 43 }, "tool_result_tokens": 24099, "tool_time_ms": 16348, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2524529999999996, "cost_usd": 2.2524529999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 236974, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 27, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2206366, "tokens_cache_write": 75485, "tokens_input": 49, "tokens_output": 15767, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 29141, "tool_time_ms": 28229, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.088248, "cost_usd": 12.088248, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 884910, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 112, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14138746, "tokens_cache_write": 386313, "tokens_input": 334, "tokens_output": 46163, "tool_calls": 99, "tool_calls_by_class": { "cc_builtin": 96, "vendor_mcp": 3 }, "tool_result_tokens": 60060, "tool_time_ms": 27928, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.1264865, "cost_usd": 7.1264865, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 710514, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 91, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9318833, "tokens_cache_write": 150731, "tokens_input": 172, "tokens_output": 38356, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 83 }, "tool_result_tokens": 35500, "tool_time_ms": 36937, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.08151, "cost_usd": 9.08151, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 933348, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 100, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12077520, "tokens_cache_write": 202285, "tokens_input": 190, "tokens_output": 40758, "tool_calls": 87, "tool_calls_by_class": { "cc_builtin": 82, "vendor_mcp": 5 }, "tool_result_tokens": 53416, "tool_time_ms": 22157, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.723871, "cost_usd": 7.723871, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 751522, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10210912, "tokens_cache_write": 158919, "tokens_input": 295, "tokens_output": 41110, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 79, "vendor_mcp": 4 }, "tool_result_tokens": 44139, "tool_time_ms": 22905, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.929729, "cost_usd": 8.929729, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 806086, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11738808, "tokens_cache_write": 198180, "tokens_input": 190, "tokens_output": 43103, "tool_calls": 105, "tool_calls_by_class": { "cc_builtin": 93, "vendor_mcp": 12 }, "tool_result_tokens": 46737, "tool_time_ms": 25872, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.089046, "cost_usd": 8.089046, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 763142, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10897592, "tokens_cache_write": 155989, "tokens_input": 1692, "tokens_output": 42876, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 91, "vendor_mcp": 7 }, "tool_result_tokens": 41303, "tool_time_ms": 25637, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.2089895, "cost_usd": 8.2089895, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 784734, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11211489, "tokens_cache_write": 153811, "tokens_input": 192, "tokens_output": 42567, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 85, "vendor_mcp": 5 }, "tool_result_tokens": 40977, "tool_time_ms": 24930, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.0361585, "cost_usd": 8.0361585, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 754357, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10415467, "tokens_cache_write": 174886, "tokens_input": 3973, "tokens_output": 42388, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 86, "vendor_mcp": 4 }, "tool_result_tokens": 45556, "tool_time_ms": 23861, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.3657375, "cost_usd": 7.3657375, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 728061, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9189025, "tokens_cache_write": 171012, "tokens_input": 986, "tokens_output": 42247, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 90, "vendor_mcp": 8 }, "tool_result_tokens": 42342, "tool_time_ms": 24607, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.588341999999999, "cost_usd": 7.588341999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 691693, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10122814, "tokens_cache_write": 152968, "tokens_input": 3971, "tokens_output": 39096, "tool_calls": 95, "tool_calls_by_class": { "cc_builtin": 90, "vendor_mcp": 5 }, "tool_result_tokens": 42652, "tool_time_ms": 23655, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.471890999999999, "cost_usd": 6.471890999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 597328, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 108, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8775142, "tokens_cache_write": 143109, "tokens_input": 206, "tokens_output": 26088, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 78 }, "tool_result_tokens": 29049, "tool_time_ms": 31366, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.694572999999999, "cost_usd": 5.694572999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 564120, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 105, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8199946, "tokens_cache_write": 88315, "tokens_input": 200, "tokens_output": 28418, "tool_calls": 75, "tool_calls_by_class": { "cc_builtin": 74, "vendor_mcp": 1 }, "tool_result_tokens": 23858, "tool_time_ms": 26914, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.042554000000001, "cost_usd": 6.042554000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 560712, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8498908, "tokens_cache_write": 110019, "tokens_input": 192, "tokens_output": 27678, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 75, "vendor_mcp": 1 }, "tool_result_tokens": 31458, "tool_time_ms": 27728, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.958259000000001, "cost_usd": 4.958259000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 513610, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6832028, "tokens_cache_write": 92261, "tokens_input": 167, "tokens_output": 24752, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 69 }, "tool_result_tokens": 25275, "tool_time_ms": 25654, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.887457499999999, "cost_usd": 5.887457499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 578754, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 100, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8211785, "tokens_cache_write": 103470, "tokens_input": 753, "tokens_output": 29724, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 77 }, "tool_result_tokens": 29072, "tool_time_ms": 26341, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5954705, "cost_usd": 5.5954705, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 583930, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7887431, "tokens_cache_write": 97004, "tokens_input": 188, "tokens_output": 27231, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 70, "vendor_mcp": 1 }, "tool_result_tokens": 24217, "tool_time_ms": 29259, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.249545500000001, "cost_usd": 5.249545500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 556970, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6978801, "tokens_cache_write": 110716, "tokens_input": 167, "tokens_output": 26086, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 29156, "tool_time_ms": 23606, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8853539999999995, "cost_usd": 5.8853539999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 714950, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7856178, "tokens_cache_write": 108136, "tokens_input": 176, "tokens_output": 35001, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 74, "vendor_mcp": 2 }, "tool_result_tokens": 30403, "tool_time_ms": 29125, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.0646255, "cost_usd": 4.0646255, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 446019, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 81, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5612351, "tokens_cache_write": 73808, "tokens_input": 154, "tokens_output": 20784, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 19051, "tool_time_ms": 20958, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8036925, "cost_usd": 4.8036925, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 487276, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7051845, "tokens_cache_write": 71090, "tokens_input": 179, "tokens_output": 22639, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 22917, "tool_time_ms": 27048, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.61536, "cost_usd": 8.61536, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 964925, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 132, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6722810, "tokens_cache_write": 369657, "tokens_input": 2952, "tokens_output": 61705, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 170, "vendor_mcp": 12 }, "tool_result_tokens": 111484, "tool_time_ms": 35514, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.858429999999998, "cost_usd": 8.858429999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 906671, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 138, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7231580, "tokens_cache_write": 379550, "tokens_input": 643, "tokens_output": 57757, "tool_calls": 198, "tool_calls_by_class": { "cc_builtin": 180, "vendor_mcp": 18 }, "tool_result_tokens": 121394, "tool_time_ms": 27824, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.8869445, "cost_usd": 7.8869445, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 871068, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 131, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6684839, "tokens_cache_write": 316286, "tokens_input": 1353, "tokens_output": 54996, "tool_calls": 178, "tool_calls_by_class": { "cc_builtin": 163, "vendor_mcp": 15 }, "tool_result_tokens": 103834, "tool_time_ms": 29618, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.174399, "cost_usd": 8.174399, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 932466, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 133, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6881998, "tokens_cache_write": 331517, "tokens_input": 1981, "tokens_output": 56333, "tool_calls": 190, "tool_calls_by_class": { "cc_builtin": 177, "vendor_mcp": 13 }, "tool_result_tokens": 111190, "tool_time_ms": 30975, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.879406, "cost_usd": 7.879406, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 901834, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 126, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6488332, "tokens_cache_write": 322597, "tokens_input": 1734, "tokens_output": 56024, "tool_calls": 173, "tool_calls_by_class": { "cc_builtin": 164, "vendor_mcp": 9 }, "tool_result_tokens": 107960, "tool_time_ms": 28611, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.969896, "cost_usd": 7.969896, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 914810, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 129, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6623102, "tokens_cache_write": 322401, "tokens_input": 2022, "tokens_output": 56969, "tool_calls": 181, "tool_calls_by_class": { "cc_builtin": 166, "vendor_mcp": 15 }, "tool_result_tokens": 107686, "tool_time_ms": 25265, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.637714, "cost_usd": 7.637714, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 860486, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 122, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6204768, "tokens_cache_write": 317861, "tokens_input": 799, "tokens_output": 54109, "tool_calls": 164, "tool_calls_by_class": { "cc_builtin": 152, "vendor_mcp": 12 }, "tool_result_tokens": 108388, "tool_time_ms": 24852, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.9936715, "cost_usd": 7.9936715, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 856523, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 123, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6417043, "tokens_cache_write": 342839, "tokens_input": 2332, "tokens_output": 53804, "tool_calls": 176, "tool_calls_by_class": { "cc_builtin": 161, "vendor_mcp": 15 }, "tool_result_tokens": 121573, "tool_time_ms": 24636, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.1397115, "cost_usd": 8.1397115, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 887215, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 129, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6723943, "tokens_cache_write": 342643, "tokens_input": 1062, "tokens_output": 53840, "tool_calls": 180, "tool_calls_by_class": { "cc_builtin": 170, "vendor_mcp": 10 }, "tool_result_tokens": 119616, "tool_time_ms": 28926, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.5019425, "cost_usd": 8.5019425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 924165, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 132, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6844355, "tokens_cache_write": 357226, "tokens_input": 4811, "tokens_output": 59338, "tool_calls": 188, "tool_calls_by_class": { "cc_builtin": 172, "vendor_mcp": 16 }, "tool_result_tokens": 125546, "tool_time_ms": 31887, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.694585000000002, "cost_usd": 8.694585000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 788474, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 143, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6945300, "tokens_cache_write": 405496, "tokens_input": 9545, "tokens_output": 44770, "tool_calls": 181, "tool_calls_by_class": { "cc_builtin": 166, "vendor_mcp": 15 }, "tool_result_tokens": 111924, "tool_time_ms": 59984, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.042476, "cost_usd": 9.042476, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 836325, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 148, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7153282, "tokens_cache_write": 423099, "tokens_input": 1409, "tokens_output": 49112, "tool_calls": 189, "tool_calls_by_class": { "cc_builtin": 180, "vendor_mcp": 9 }, "tool_result_tokens": 98890, "tool_time_ms": 67529, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.176509999999999, "cost_usd": 8.176509999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 807310, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 139, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6776100, "tokens_cache_write": 358649, "tokens_input": 5994, "tokens_output": 46880, "tool_calls": 171, "tool_calls_by_class": { "cc_builtin": 161, "vendor_mcp": 10 }, "tool_result_tokens": 106739, "tool_time_ms": 42791, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.969178000000001, "cost_usd": 7.969178000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 736719, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 138, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6758946, "tokens_cache_write": 352635, "tokens_input": 781, "tokens_output": 42378, "tool_calls": 166, "tool_calls_by_class": { "cc_builtin": 152, "vendor_mcp": 14 }, "tool_result_tokens": 105473, "tool_time_ms": 44367, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.3921895, "cost_usd": 8.3921895, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 816086, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 146, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7213339, "tokens_cache_write": 357411, "tokens_input": 3147, "tokens_output": 47827, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 171, "vendor_mcp": 13 }, "tool_result_tokens": 106651, "tool_time_ms": 50437, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.895790000000001, "cost_usd": 7.895790000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 731643, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 134, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6561180, "tokens_cache_write": 353225, "tokens_input": 845, "tokens_output": 43149, "tool_calls": 164, "tool_calls_by_class": { "cc_builtin": 149, "vendor_mcp": 15 }, "tool_result_tokens": 106339, "tool_time_ms": 43125, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.808729999999999, "cost_usd": 8.808729999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 862545, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 148, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7183920, "tokens_cache_write": 398898, "tokens_input": 2253, "tokens_output": 48661, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 171, "vendor_mcp": 11 }, "tool_result_tokens": 111898, "tool_time_ms": 75098, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.0519435, "cost_usd": 8.0519435, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 827930, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 141, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6837787, "tokens_cache_write": 345538, "tokens_input": 7709, "tokens_output": 45565, "tool_calls": 176, "tool_calls_by_class": { "cc_builtin": 168, "vendor_mcp": 8 }, "tool_result_tokens": 103691, "tool_time_ms": 84765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.6232985, "cost_usd": 8.6232985, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 832545, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 150, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7376717, "tokens_cache_write": 372550, "tokens_input": 4353, "tokens_output": 47507, "tool_calls": 190, "tool_calls_by_class": { "cc_builtin": 174, "vendor_mcp": 16 }, "tool_result_tokens": 111498, "tool_time_ms": 56811, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.908833999999999, "cost_usd": 7.908833999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 775399, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 138, "model": "claude-opus-5", "path": "runs/gitnexus/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6720548, "tokens_cache_write": 340097, "tokens_input": 6143, "tokens_output": 44675, "tool_calls": 177, "tool_calls_by_class": { "cc_builtin": 162, "vendor_mcp": 15 }, "tool_result_tokens": 102361, "tool_time_ms": 54106, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.152644, "cost_usd": 5.152644, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 492324, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 108, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12370700, "tokens_cache_write": 177681, "tokens_input": 216, "tokens_output": 24980, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 79, "vendor_mcp": 10 }, "tool_result_tokens": 45201, "tool_time_ms": 19285, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.227014, "cost_usd": 5.227014, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 516500, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 99, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12613390, "tokens_cache_write": 170951, "tokens_input": 477, "tokens_output": 27724, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 100, "vendor_mcp": 9 }, "tool_result_tokens": 54425, "tool_time_ms": 23592, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.4479069, "cost_usd": 6.4479069, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 574991, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 117, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14757503, "tokens_cache_write": 261684, "tokens_input": 234, "tokens_output": 29990, "tool_calls": 97, "tool_calls_by_class": { "cc_builtin": 85, "vendor_mcp": 12 }, "tool_result_tokens": 53158, "tool_time_ms": 21324, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.910939899999999, "cost_usd": 5.910939899999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 570185, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 127, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14032483, "tokens_cache_write": 217588, "tokens_input": 254, "tokens_output": 26327, "tool_calls": 107, "tool_calls_by_class": { "cc_builtin": 100, "vendor_mcp": 7 }, "tool_result_tokens": 38960, "tool_time_ms": 26353, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.709197799999999, "cost_usd": 4.709197799999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 519912, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 102, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11647916, "tokens_cache_write": 134691, "tokens_input": 204, "tokens_output": 27071, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 71, "vendor_mcp": 11 }, "tool_result_tokens": 42131, "tool_time_ms": 18438, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.4856603, "cost_usd": 4.4856603, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 464001, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 93, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10972241, "tokens_cache_write": 138995, "tokens_input": 186, "tokens_output": 23964, "tool_calls": 80, "tool_calls_by_class": { "cc_builtin": 69, "vendor_mcp": 11 }, "tool_result_tokens": 44992, "tool_time_ms": 18066, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.175664499999999, "cost_usd": 5.175664499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 516187, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 109, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11351465, "tokens_cache_write": 230451, "tokens_input": 218, "tokens_output": 25791, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 83, "vendor_mcp": 6 }, "tool_result_tokens": 34342, "tool_time_ms": 23170, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.339300199999999, "cost_usd": 4.339300199999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 481580, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 92, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10869504, "tokens_cache_write": 111232, "tokens_input": 184, "tokens_output": 27367, "tool_calls": 91, "tool_calls_by_class": { "cc_builtin": 80, "vendor_mcp": 11 }, "tool_result_tokens": 42723, "tool_time_ms": 19117, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.9647699, "cost_usd": 4.9647699, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 530106, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 95, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12182523, "tokens_cache_write": 144006, "tokens_input": 3969, "tokens_output": 28938, "tool_calls": 105, "tool_calls_by_class": { "cc_builtin": 84, "vendor_mcp": 21 }, "tool_result_tokens": 50870, "tool_time_ms": 17054, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8167359000000003, "cost_usd": 3.8167359000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 456695, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 90, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9599233, "tokens_cache_write": 89867, "tokens_input": 2183, "tokens_output": 26081, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 65, "vendor_mcp": 9 }, "tool_result_tokens": 32432, "tool_time_ms": 17540, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4145559, "cost_usd": 3.4145559, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 361838, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 99, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8075113, "tokens_cache_write": 124808, "tokens_input": 198, "tokens_output": 16172, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 67, "vendor_mcp": 2 }, "tool_result_tokens": 19103, "tool_time_ms": 26189, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8586361, "cost_usd": 2.8586361, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 324230, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 94, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7362477, "tokens_cache_write": 68159, "tokens_input": 188, "tokens_output": 16025, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 63, "vendor_mcp": 1 }, "tool_result_tokens": 14219, "tool_time_ms": 23766, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0662262, "cost_usd": 3.0662262, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 342083, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 98, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8128694, "tokens_cache_write": 65990, "tokens_input": 196, "tokens_output": 15406, "tool_calls": 68, "tool_calls_by_class": { "cc_builtin": 65, "vendor_mcp": 3 }, "tool_result_tokens": 18572, "tool_time_ms": 23822, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9642705999999994, "cost_usd": 2.9642705999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 360367, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 98, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7873042, "tokens_cache_write": 60620, "tokens_input": 196, "tokens_output": 15870, "tool_calls": 68, "tool_calls_by_class": { "cc_builtin": 68 }, "tool_result_tokens": 14362, "tool_time_ms": 27556, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.395063999999999, "cost_usd": 3.395063999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 395688, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 109, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8771630, "tokens_cache_write": 78281, "tokens_input": 218, "tokens_output": 19549, "tool_calls": 79, "tool_calls_by_class": { "cc_builtin": 77, "vendor_mcp": 2 }, "tool_result_tokens": 18240, "tool_time_ms": 27192, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6688513, "cost_usd": 2.6688513, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 323159, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6869141, "tokens_cache_write": 62540, "tokens_input": 178, "tokens_output": 15489, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 58, "vendor_mcp": 1 }, "tool_result_tokens": 13965, "tool_time_ms": 20588, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4856040999999993, "cost_usd": 2.4856040999999993, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 379883, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 86, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6627817, "tokens_cache_write": 49293, "tokens_input": 172, "tokens_output": 13399, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 13108, "tool_time_ms": 24624, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8554531, "cost_usd": 2.8554531, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 337076, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 92, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7159067, "tokens_cache_write": 72801, "tokens_input": 184, "tokens_output": 18025, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 59, "vendor_mcp": 3 }, "tool_result_tokens": 14766, "tool_time_ms": 24157, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2141874000000006, "cost_usd": 3.2141874000000006, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 344481, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 99, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8343308, "tokens_cache_write": 77376, "tokens_input": 198, "tokens_output": 16423, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 66, "vendor_mcp": 3 }, "tool_result_tokens": 20598, "tool_time_ms": 24555, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4379149, "cost_usd": 2.4379149, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 391761, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 84, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6432583, "tokens_cache_write": 55361, "tokens_input": 168, "tokens_output": 11698, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 51, "vendor_mcp": 3 }, "tool_result_tokens": 15028, "tool_time_ms": 20186, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.518669050000001, "cost_usd": 5.518669050000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 679576, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 153, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9208286, "tokens_cache_write": 375248, "tokens_input": 377, "tokens_output": 38611, "tool_calls": 149, "tool_calls_by_class": { "cc_builtin": 128, "subagent": 1, "vendor_mcp": 20 }, "tool_result_tokens": 98250, "tool_time_ms": 24650, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.69827535, "cost_usd": 6.69827535, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 827461, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 182, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10802542, "tokens_cache_write": 488670, "tokens_input": 1204, "tokens_output": 50867, "tool_calls": 196, "tool_calls_by_class": { "cc_builtin": 143, "subagent": 4, "vendor_mcp": 49 }, "tool_result_tokens": 104320, "tool_time_ms": 24208, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.571862499999999, "cost_usd": 5.571862499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 735692, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 156, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9629265, "tokens_cache_write": 347240, "tokens_input": 382, "tokens_output": 44854, "tool_calls": 171, "tool_calls_by_class": { "cc_builtin": 137, "subagent": 1, "vendor_mcp": 33 }, "tool_result_tokens": 102829, "tool_time_ms": 23507, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.0297826, "cost_usd": 5.0297826, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 664158, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 146, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9101032, "tokens_cache_write": 287557, "tokens_input": 292, "tokens_output": 38217, "tool_calls": 143, "tool_calls_by_class": { "cc_builtin": 108, "vendor_mcp": 35 }, "tool_result_tokens": 93125, "tool_time_ms": 19066, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.532488399999999, "cost_usd": 5.532488399999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 698467, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 151, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9314073, "tokens_cache_write": 361415, "tokens_input": 376, "tokens_output": 42204, "tool_calls": 167, "tool_calls_by_class": { "cc_builtin": 118, "subagent": 1, "vendor_mcp": 48 }, "tool_result_tokens": 114190, "tool_time_ms": 18747, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.48107275, "cost_usd": 5.48107275, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 681714, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 155, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9504250, "tokens_cache_write": 343646, "tokens_input": 310, "tokens_output": 42610, "tool_calls": 152, "tool_calls_by_class": { "cc_builtin": 123, "subagent": 1, "vendor_mcp": 28 }, "tool_result_tokens": 105457, "tool_time_ms": 20578, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.839970399999999, "cost_usd": 5.839970399999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 682771, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 179, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10644933, "tokens_cache_write": 341449, "tokens_input": 649, "tokens_output": 41023, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 141, "subagent": 1, "vendor_mcp": 40 }, "tool_result_tokens": 99417, "tool_time_ms": 29086, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.865725399999999, "cost_usd": 5.865725399999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 698761, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 158, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9576923, "tokens_cache_write": 393372, "tokens_input": 2365, "tokens_output": 44464, "tool_calls": 173, "tool_calls_by_class": { "cc_builtin": 138, "subagent": 1, "vendor_mcp": 34 }, "tool_result_tokens": 95664, "tool_time_ms": 21619, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.176107000000002, "cost_usd": 5.176107000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 706676, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 142, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8948640, "tokens_cache_write": 311013, "tokens_input": 354, "tokens_output": 41625, "tool_calls": 150, "tool_calls_by_class": { "cc_builtin": 105, "vendor_mcp": 45 }, "tool_result_tokens": 102837, "tool_time_ms": 19878, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.301531299999999, "cost_usd": 5.301531299999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 697992, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 140, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8990241, "tokens_cache_write": 335616, "tokens_input": 1296, "tokens_output": 39125, "tool_calls": 143, "tool_calls_by_class": { "cc_builtin": 111, "vendor_mcp": 32 }, "tool_result_tokens": 116758, "tool_time_ms": 19526, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.8192016, "cost_usd": 6.8192016, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 670623, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 168, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9600902, "tokens_cache_write": 592985, "tokens_input": 3580, "tokens_output": 33641, "tool_calls": 169, "tool_calls_by_class": { "cc_builtin": 136, "subagent": 5, "vendor_mcp": 28 }, "tool_result_tokens": 104246, "tool_time_ms": 70920, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.499287099999999, "cost_usd": 5.499287099999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 618411, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 140, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8247997, "tokens_cache_write": 423947, "tokens_input": 2237, "tokens_output": 31633, "tool_calls": 131, "tool_calls_by_class": { "cc_builtin": 107, "skill": 1, "vendor_mcp": 23 }, "tool_result_tokens": 87528, "tool_time_ms": 53304, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.502786149999999, "cost_usd": 5.502786149999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 744877, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 161, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9329088, "tokens_cache_write": 371745, "tokens_input": 3872, "tokens_output": 33335, "tool_calls": 144, "tool_calls_by_class": { "cc_builtin": 109, "subagent": 2, "vendor_mcp": 33 }, "tool_result_tokens": 97862, "tool_time_ms": 111073, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.268750899999999, "cost_usd": 5.268750899999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 688180, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 155, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9068533, "tokens_cache_write": 358203, "tokens_input": 310, "tokens_output": 32179, "tool_calls": 147, "tool_calls_by_class": { "cc_builtin": 126, "subagent": 1, "vendor_mcp": 20 }, "tool_result_tokens": 89289, "tool_time_ms": 39250, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.067155999999999, "cost_usd": 5.067155999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 553774, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 137, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8173020, "tokens_cache_write": 363727, "tokens_input": 2376, "tokens_output": 28384, "tool_calls": 120, "tool_calls_by_class": { "cc_builtin": 93, "vendor_mcp": 27 }, "tool_result_tokens": 85602, "tool_time_ms": 33039, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.7984487, "cost_usd": 4.7984487, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 447282, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 143, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8077939, "tokens_cache_write": 337722, "tokens_input": 286, "tokens_output": 28114, "tool_calls": 119, "tool_calls_by_class": { "cc_builtin": 94, "subagent": 1, "vendor_mcp": 24 }, "tool_result_tokens": 83324, "tool_time_ms": 32512, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.45968065, "cost_usd": 5.45968065, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 642970, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 162, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9571060, "tokens_cache_write": 362923, "tokens_input": 2281, "tokens_output": 32666, "tool_calls": 161, "tool_calls_by_class": { "cc_builtin": 136, "subagent": 2, "vendor_mcp": 23 }, "tool_result_tokens": 103606, "tool_time_ms": 45982, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.543779649999999, "cost_usd": 5.543779649999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 566032, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 160, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8878853, "tokens_cache_write": 413876, "tokens_input": 4461, "tokens_output": 34987, "tool_calls": 147, "tool_calls_by_class": { "cc_builtin": 117, "subagent": 3, "vendor_mcp": 27 }, "tool_result_tokens": 96392, "tool_time_ms": 51954, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.410930049999999, "cost_usd": 5.410930049999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 585300, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 153, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8352551, "tokens_cache_write": 445236, "tokens_input": 365, "tokens_output": 27842, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 118, "subagent": 3, "vendor_mcp": 19 }, "tool_result_tokens": 93505, "tool_time_ms": 62678, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "gitnexus", "arm_label": "CC+GitNexus", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.0121527, "cost_usd": 6.0121527, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 668532, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 173, "model": "claude-sonnet-5", "path": "runs/gitnexus/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|gitnexus|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9661159, "tokens_cache_write": 462857, "tokens_input": 453, "tokens_output": 34187, "tool_calls": 161, "tool_calls_by_class": { "cc_builtin": 141, "subagent": 4, "vendor_mcp": 16 }, "tool_result_tokens": 89687, "tool_time_ms": 63645, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.672380500000002, "cost_usd": 6.672380500000002, "denominator": 20, "duration_basis": "run", "duration_ms": 338771, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 65, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3225344, "tokens_cache_write": 170256, "tokens_input": 125, "tokens_output": 28987, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 99, "skill": 1, "subagent": 3 }, "tool_result_tokens": 50898, "tool_time_ms": 3944, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8188879999999994, "cost_usd": 3.8188879999999994, "denominator": 20, "duration_basis": "run", "duration_ms": 349537, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 44, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2797646, "tokens_cache_write": 114014, "tokens_input": 84, "tokens_output": 22595, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 44, "skill": 1 }, "tool_result_tokens": 21396, "tool_time_ms": 2147, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.351653750000001, "cost_usd": 6.351653750000001, "denominator": 20, "duration_basis": "run", "duration_ms": 266980, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 90, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3387908, "tokens_cache_write": 203117, "tokens_input": 177, "tokens_output": 29258, "tool_calls": 132, "tool_calls_by_class": { "cc_builtin": 125, "skill": 1, "subagent": 6 }, "tool_result_tokens": 68592, "tool_time_ms": 6595, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.230561, "cost_usd": 6.230561, "denominator": 20, "duration_basis": "run", "duration_ms": 340080, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 41, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3452581, "tokens_cache_write": 83388, "tokens_input": 77, "tokens_output": 22189, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 52, "skill": 1 }, "tool_result_tokens": 27794, "tool_time_ms": 2471, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.392303000000001, "cost_usd": 5.392303000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 365272, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 36, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2673633, "tokens_cache_write": 73265, "tokens_input": 67, "tokens_output": 25054, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 51, "skill": 1 }, "tool_result_tokens": 17228, "tool_time_ms": 2158, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.052164000000003, "cost_usd": 11.052164000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 534844, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 91, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7752334, "tokens_cache_write": 92150, "tokens_input": 173, "tokens_output": 29102, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 70, "skill": 1 }, "tool_result_tokens": 22947, "tool_time_ms": 8493, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.847658000000001, "cost_usd": 10.847658000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 549081, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 87, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7491658, "tokens_cache_write": 93032, "tokens_input": 166, "tokens_output": 29874, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 66, "skill": 1 }, "tool_result_tokens": 23520, "tool_time_ms": 4910, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.309531000000002, "cost_usd": 10.309531000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 499096, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 83, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6879667, "tokens_cache_write": 138763, "tokens_input": 158, "tokens_output": 25811, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 66, "skill": 1 }, "tool_result_tokens": 24214, "tool_time_ms": 3335, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 16.1563985, "cost_usd": 16.1563985, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 601643, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 86, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11923774, "tokens_cache_write": 169376, "tokens_input": 164, "tokens_output": 32208, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 61, "skill": 5 }, "tool_result_tokens": 23228, "tool_time_ms": 6011, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.092725999999999, "cost_usd": 10.092725999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 490924, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 80, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6988576, "tokens_cache_write": 89454, "tokens_input": 152, "tokens_output": 26271, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 60, "skill": 1 }, "tool_result_tokens": 25078, "tool_time_ms": 4031, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.634439, "cost_usd": 6.634439, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 363091, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 87, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4562289, "tokens_cache_write": 53677, "tokens_input": 166, "tokens_output": 19939, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 16360, "tool_time_ms": 2388, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.982244000000001, "cost_usd": 5.982244000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 445950, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 77, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4042764, "tokens_cache_write": 48853, "tokens_input": 147, "tokens_output": 19219, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 47 }, "tool_result_tokens": 14326, "tool_time_ms": 2289, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.093774000000002, "cost_usd": 9.093774000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 418028, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 90, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6558384, "tokens_cache_write": 72909, "tokens_input": 171, "tokens_output": 21510, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 64, "skill": 1 }, "tool_result_tokens": 17012, "tool_time_ms": 3795, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.343186, "cost_usd": 6.343186, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 405379, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4324536, "tokens_cache_write": 51485, "tokens_input": 160, "tokens_output": 19747, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 15338, "tool_time_ms": 2569, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.984426999999998, "cost_usd": 8.984426999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 509307, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6488077, "tokens_cache_write": 70905, "tokens_input": 175, "tokens_output": 21530, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 61, "skill": 1 }, "tool_result_tokens": 15881, "tool_time_ms": 3378, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 17.892662, "cost_usd": 17.892662, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 891343, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 145, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6453910, "tokens_cache_write": 498820, "tokens_input": 284, "tokens_output": 50654, "tool_calls": 139, "tool_calls_by_class": { "cc_builtin": 122, "skill": 17 }, "tool_result_tokens": 74062, "tool_time_ms": 11753, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 16.778755500000003, "cost_usd": 16.778755500000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 933627, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 144, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6342683, "tokens_cache_write": 441250, "tokens_input": 282, "tokens_output": 52134, "tool_calls": 135, "tool_calls_by_class": { "cc_builtin": 120, "skill": 15 }, "tool_result_tokens": 68574, "tool_time_ms": 10194, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 16.565979999999996, "cost_usd": 16.565979999999996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 918223, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 144, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6451244, "tokens_cache_write": 455247, "tokens_input": 294, "tokens_output": 48985, "tool_calls": 136, "tool_calls_by_class": { "cc_builtin": 120, "skill": 16 }, "tool_result_tokens": 75106, "tool_time_ms": 12150, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 18.258654999999997, "cost_usd": 18.258654999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 879560, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 145, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6673179, "tokens_cache_write": 500015, "tokens_input": 284, "tokens_output": 52625, "tool_calls": 138, "tool_calls_by_class": { "cc_builtin": 119, "skill": 19 }, "tool_result_tokens": 81286, "tool_time_ms": 13190, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 16.936661, "cost_usd": 16.936661, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 914455, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 148, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6727649, "tokens_cache_write": 453480, "tokens_input": 288, "tokens_output": 50848, "tool_calls": 139, "tool_calls_by_class": { "cc_builtin": 123, "skill": 16 }, "tool_result_tokens": 81112, "tool_time_ms": 11023, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.773576, "cost_usd": 4.773576, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 162362, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 34, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1377686, "tokens_cache_write": 151903, "tokens_input": 68, "tokens_output": 7143, "tool_calls": 27, "tool_calls_by_class": { "cc_builtin": 21, "skill": 6 }, "tool_result_tokens": 12341, "tool_time_ms": 3809, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.471109, "cost_usd": 3.471109, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 128401, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 26, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 954789, "tokens_cache_write": 110365, "tokens_input": 52, "tokens_output": 6170, "tool_calls": 20, "tool_calls_by_class": { "cc_builtin": 16, "skill": 4 }, "tool_result_tokens": 10006, "tool_time_ms": 2226, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.834434, "cost_usd": 4.834434, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 177353, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 36, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1466704, "tokens_cache_write": 146243, "tokens_input": 72, "tokens_output": 8843, "tool_calls": 30, "tool_calls_by_class": { "cc_builtin": 24, "skill": 6 }, "tool_result_tokens": 13483, "tool_time_ms": 3813, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.21472, "cost_usd": 3.21472, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 106719, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 27, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1034980, "tokens_cache_write": 95365, "tokens_input": 54, "tokens_output": 5438, "tool_calls": 21, "tool_calls_by_class": { "cc_builtin": 17, "skill": 4 }, "tool_result_tokens": 9773, "tool_time_ms": 1731, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.379008, "cost_usd": 4.379008, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 138477, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 33, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1348438, "tokens_cache_write": 132648, "tokens_input": 66, "tokens_output": 7539, "tool_calls": 26, "tool_calls_by_class": { "cc_builtin": 20, "skill": 6 }, "tool_result_tokens": 12576, "tool_time_ms": 3470, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 22.330274000000003, "cost_usd": 22.330274000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 950637, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 165, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6893028, "tokens_cache_write": 677831, "tokens_input": 328, "tokens_output": 47072, "tool_calls": 142, "tool_calls_by_class": { "cc_builtin": 115, "skill": 27 }, "tool_result_tokens": 70044, "tool_time_ms": 15771, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 22.227538999999997, "cost_usd": 22.227538999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 850530, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 159, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6706759, "tokens_cache_write": 659650, "tokens_input": 318, "tokens_output": 46492, "tool_calls": 135, "tool_calls_by_class": { "cc_builtin": 108, "skill": 27 }, "tool_result_tokens": 69809, "tool_time_ms": 15331, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 22.15041, "cost_usd": 22.15041, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 862416, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 159, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6715500, "tokens_cache_write": 657712, "tokens_input": 317, "tokens_output": 45550, "tool_calls": 139, "tool_calls_by_class": { "cc_builtin": 112, "skill": 27 }, "tool_result_tokens": 67077, "tool_time_ms": 15547, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 21.863380999999993, "cost_usd": 21.863380999999993, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 883316, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 162, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6914721, "tokens_cache_write": 633103, "tokens_input": 320, "tokens_output": 45668, "tool_calls": 143, "tool_calls_by_class": { "cc_builtin": 116, "skill": 27 }, "tool_result_tokens": 67263, "tool_time_ms": 15388, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 21.020251000000002, "cost_usd": 21.020251000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 859609, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 162, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6824771, "tokens_cache_write": 593065, "tokens_input": 323, "tokens_output": 46619, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 112, "skill": 25 }, "tool_result_tokens": 70082, "tool_time_ms": 15012, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 23.928705999999995, "cost_usd": 23.928705999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 854133, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 172, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7248156, "tokens_cache_write": 708555, "tokens_input": 340, "tokens_output": 50121, "tool_calls": 149, "tool_calls_by_class": { "cc_builtin": 123, "skill": 26 }, "tool_result_tokens": 73962, "tool_time_ms": 19194, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 22.500194000000004, "cost_usd": 22.500194000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 877139, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 170, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|12", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7020614, "tokens_cache_write": 650447, "tokens_input": 339, "tokens_output": 49345, "tool_calls": 145, "tool_calls_by_class": { "cc_builtin": 121, "skill": 24 }, "tool_result_tokens": 72047, "tool_time_ms": 16724, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 23.810474000000006, "cost_usd": 23.810474000000006, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 872015, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 168, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|13", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7037674, "tokens_cache_write": 714970, "tokens_input": 335, "tokens_output": 49401, "tool_calls": 144, "tool_calls_by_class": { "cc_builtin": 116, "skill": 28 }, "tool_result_tokens": 73203, "tool_time_ms": 17765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 24.115233999999997, "cost_usd": 24.115233999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 854007, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 170, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|14", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7232364, "tokens_cache_write": 719444, "tokens_input": 339, "tokens_output": 49812, "tool_calls": 148, "tool_calls_by_class": { "cc_builtin": 120, "skill": 28 }, "tool_result_tokens": 77393, "tool_time_ms": 18728, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 22.471378, "cost_usd": 22.471378, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 798101, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 166, "model": "claude-fable-5", "path": "runs/graphify/claude-fable-5/ZeroShotExam/memos/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|15", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6860098, "tokens_cache_write": 662671, "tokens_input": 331, "tokens_output": 47091, "tool_calls": 147, "tool_calls_by_class": { "cc_builtin": 122, "skill": 25 }, "tool_result_tokens": 73735, "tool_time_ms": 15656, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8568976000000004, "cost_usd": 2.8568976000000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 138461, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 363, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11181431, "tokens_cache_write": 794322, "tokens_input": 4151, "tokens_output": 62550, "tool_calls": 565, "tool_calls_by_class": { "cc_builtin": 545, "subagent": 20 }, "tool_result_tokens": 360842, "tool_time_ms": 16307, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.4366948, "cost_usd": 5.4366948, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 749454, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 262, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 26312703, "tokens_cache_write": 1326390, "tokens_input": 1600, "tokens_output": 31409, "tool_calls": 253, "tool_calls_by_class": { "cc_builtin": 252, "subagent": 1 }, "tool_result_tokens": 87517, "tool_time_ms": 7608, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.79162885, "cost_usd": 4.79162885, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 746199, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 273, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 24493236, "tokens_cache_write": 1086251, "tokens_input": 1638, "tokens_output": 37425, "tool_calls": 260, "tool_calls_by_class": { "cc_builtin": 259, "subagent": 1 }, "tool_result_tokens": 85593, "tool_time_ms": 6824, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5802664, "cost_usd": 5.5802664, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 611536, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 427, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 25313362, "tokens_cache_write": 1413545, "tokens_input": 2709, "tokens_output": 41658, "tool_calls": 540, "tool_calls_by_class": { "cc_builtin": 515, "subagent": 25 }, "tool_result_tokens": 359824, "tool_time_ms": 17071, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.50756575, "cost_usd": 5.50756575, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 529453, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 8, "family": "cc_competitor", "llm_calls": 330, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 24452702, "tokens_cache_write": 1433929, "tokens_input": 2129, "tokens_output": 38306, "tool_calls": 445, "tool_calls_by_class": { "cc_builtin": 429, "subagent": 16 }, "tool_result_tokens": 296425, "tool_time_ms": 13999, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.024008800000001, "cost_usd": 5.024008800000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 561718, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 286, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 24391683, "tokens_cache_write": 1202835, "tokens_input": 1805, "tokens_output": 35742, "tool_calls": 332, "tool_calls_by_class": { "cc_builtin": 328, "harness_builtin": 1, "subagent": 3 }, "tool_result_tokens": 131172, "tool_time_ms": 11361, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.64944525, "cost_usd": 5.64944525, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 795531, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 319, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 27694720, "tokens_cache_write": 1365549, "tokens_input": 1930, "tokens_output": 37859, "tool_calls": 324, "tool_calls_by_class": { "cc_builtin": 320, "skill": 1, "subagent": 3 }, "tool_result_tokens": 125988, "tool_time_ms": 9271, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3733583000000005, "cost_usd": 2.3733583000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 114529, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 347, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9945103, "tokens_cache_write": 648716, "tokens_input": 2442, "tokens_output": 38614, "tool_calls": 521, "tool_calls_by_class": { "cc_builtin": 501, "subagent": 20 }, "tool_result_tokens": 300911, "tool_time_ms": 19177, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3002116, "cost_usd": 2.3002116, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 532419, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 166, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12747916, "tokens_cache_write": 455494, "tokens_input": 1072, "tokens_output": 22672, "tool_calls": 136, "tool_calls_by_class": { "cc_builtin": 136 }, "tool_result_tokens": 50775, "tool_time_ms": 3887, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9407942999999999, "cost_usd": 1.9407942999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 465721, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 141, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11412883, "tokens_cache_write": 349539, "tokens_input": 933, "tokens_output": 19899, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 111 }, "tool_result_tokens": 61372, "tool_time_ms": 2667, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9066665, "cost_usd": 2.9066665, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 596258, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 178, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14704325, "tokens_cache_write": 656120, "tokens_input": 1165, "tokens_output": 27927, "tool_calls": 160, "tool_calls_by_class": { "cc_builtin": 159, "subagent": 1 }, "tool_result_tokens": 73139, "tool_time_ms": 3981, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.20729795, "cost_usd": 4.20729795, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 809558, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 375, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16547167, "tokens_cache_write": 1251357, "tokens_input": 3384, "tokens_output": 39073, "tool_calls": 476, "tool_calls_by_class": { "cc_builtin": 415, "harness_builtin": 1, "subagent": 60 }, "tool_result_tokens": 471909, "tool_time_ms": 10487, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1414642, "cost_usd": 3.1414642, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 605130, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 184, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15706072, "tokens_cache_write": 721843, "tokens_input": 1161, "tokens_output": 25202, "tool_calls": 154, "tool_calls_by_class": { "cc_builtin": 154 }, "tool_result_tokens": 58672, "tool_time_ms": 3844, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.01056675, "cost_usd": 4.01056675, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 584255, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 238, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 21015860, "tokens_cache_write": 888774, "tokens_input": 1556, "tokens_output": 34502, "tool_calls": 232, "tool_calls_by_class": { "cc_builtin": 228, "subagent": 4 }, "tool_result_tokens": 122853, "tool_time_ms": 6126, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5661070500000007, "cost_usd": 3.5661070500000007, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 631799, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 191, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17607773, "tokens_cache_write": 841864, "tokens_input": 1227, "tokens_output": 27030, "tool_calls": 160, "tool_calls_by_class": { "cc_builtin": 159, "subagent": 1 }, "tool_result_tokens": 72349, "tool_time_ms": 3736, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3191854000000003, "cost_usd": 2.3191854000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 513109, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 156, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12488824, "tokens_cache_write": 480518, "tokens_input": 1017, "tokens_output": 21650, "tool_calls": 126, "tool_calls_by_class": { "cc_builtin": 126 }, "tool_result_tokens": 53609, "tool_time_ms": 3687, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5403445500000004, "cost_usd": 3.5403445500000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 537018, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 438, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14632173, "tokens_cache_write": 980777, "tokens_input": 2812, "tokens_output": 53529, "tool_calls": 580, "tool_calls_by_class": { "cc_builtin": 563, "harness_builtin": 1, "skill": 1, "subagent": 15 }, "tool_result_tokens": 344118, "tool_time_ms": 11316, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.172116700000001, "cost_usd": 4.172116700000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 598138, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 492, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16894922, "tokens_cache_write": 1203114, "tokens_input": 3916, "tokens_output": 59354, "tool_calls": 624, "tool_calls_by_class": { "cc_builtin": 606, "skill": 1, "subagent": 17 }, "tool_result_tokens": 449735, "tool_time_ms": 11921, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.00222695, "cost_usd": 3.00222695, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 498205, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 383, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12456647, "tokens_cache_write": 839409, "tokens_input": 2754, "tokens_output": 45994, "tool_calls": 484, "tool_calls_by_class": { "cc_builtin": 471, "skill": 1, "subagent": 12 }, "tool_result_tokens": 324599, "tool_time_ms": 9406, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.07876235, "cost_usd": 4.07876235, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 618607, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 11, "family": "cc_competitor", "llm_calls": 460, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 9, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15339571, "tokens_cache_write": 1212791, "tokens_input": 3018, "tokens_output": 58796, "tool_calls": 658, "tool_calls_by_class": { "cc_builtin": 633, "skill": 2, "subagent": 23 }, "tool_result_tokens": 451154, "tool_time_ms": 12378, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4522665000000003, "cost_usd": 3.4522665000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 667515, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 408, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13054515, "tokens_cache_write": 1057192, "tokens_input": 2707, "tokens_output": 49805, "tool_calls": 596, "tool_calls_by_class": { "cc_builtin": 577, "skill": 2, "subagent": 17 }, "tool_result_tokens": 409269, "tool_time_ms": 12351, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9531383499999997, "cost_usd": 3.9531383499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 427902, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 10, "family": "cc_competitor", "llm_calls": 470, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15919751, "tokens_cache_write": 1160203, "tokens_input": 3008, "tokens_output": 59590, "tool_calls": 590, "tool_calls_by_class": { "cc_builtin": 571, "skill": 2, "subagent": 17 }, "tool_result_tokens": 406675, "tool_time_ms": 13210, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1202201, "cost_usd": 4.1202201, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 732538, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 509, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16667506, "tokens_cache_write": 1191851, "tokens_input": 3212, "tokens_output": 55930, "tool_calls": 621, "tool_calls_by_class": { "cc_builtin": 600, "skill": 1, "subagent": 20 }, "tool_result_tokens": 454824, "tool_time_ms": 15349, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.559102249999999, "cost_usd": 3.559102249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 917394, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "cc_competitor", "llm_calls": 416, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14858935, "tokens_cache_write": 989460, "tokens_input": 2674, "tokens_output": 59503, "tool_calls": 566, "tool_calls_by_class": { "cc_builtin": 548, "skill": 3, "subagent": 15 }, "tool_result_tokens": 365547, "tool_time_ms": 11064, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.2319756, "cost_usd": 4.2319756, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 555657, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 12, "family": "cc_competitor", "llm_calls": 476, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 8, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16255856, "tokens_cache_write": 1258998, "tokens_input": 3103, "tokens_output": 64493, "tool_calls": 661, "tool_calls_by_class": { "cc_builtin": 645, "skill": 2, "subagent": 14 }, "tool_result_tokens": 442686, "tool_time_ms": 12240, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5980436, "cost_usd": 3.5980436, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 589922, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 11, "family": "cc_competitor", "llm_calls": 398, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 9, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14021251, "tokens_cache_write": 1033121, "tokens_input": 3251, "tokens_output": 53542, "tool_calls": 553, "tool_calls_by_class": { "cc_builtin": 536, "skill": 2, "subagent": 15 }, "tool_result_tokens": 352565, "tool_time_ms": 9645, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7012356000000004, "cost_usd": 3.7012356000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 752735, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 407, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11884011, "tokens_cache_write": 1240152, "tokens_input": 2739, "tokens_output": 53640, "tool_calls": 553, "tool_calls_by_class": { "cc_builtin": 525, "subagent": 28 }, "tool_result_tokens": 495605, "tool_time_ms": 129324, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.86744105, "cost_usd": 3.86744105, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 454211, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 431, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13884353, "tokens_cache_write": 1181601, "tokens_input": 3633, "tokens_output": 58079, "tool_calls": 551, "tool_calls_by_class": { "cc_builtin": 530, "skill": 1, "subagent": 20 }, "tool_result_tokens": 461465, "tool_time_ms": 9789, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.51534615, "cost_usd": 3.51534615, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 426963, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 416, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13787319, "tokens_cache_write": 996249, "tokens_input": 2667, "tokens_output": 54985, "tool_calls": 485, "tool_calls_by_class": { "cc_builtin": 471, "skill": 1, "subagent": 13 }, "tool_result_tokens": 370281, "tool_time_ms": 8660, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.2002446, "cost_usd": 4.2002446, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 595559, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 11, "family": "cc_competitor", "llm_calls": 485, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15371531, "tokens_cache_write": 1307236, "tokens_input": 3169, "tokens_output": 61404, "tool_calls": 580, "tool_calls_by_class": { "cc_builtin": 555, "skill": 1, "subagent": 24 }, "tool_result_tokens": 516519, "tool_time_ms": 10316, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3116493000000005, "cost_usd": 3.3116493000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 603789, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 371, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11695193, "tokens_cache_write": 1030951, "tokens_input": 2464, "tokens_output": 47518, "tool_calls": 441, "tool_calls_by_class": { "cc_builtin": 415, "skill": 1, "subagent": 25 }, "tool_result_tokens": 390674, "tool_time_ms": 7950, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0325342000000006, "cost_usd": 3.0325342000000006, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 611415, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 407, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12333812, "tokens_cache_write": 885797, "tokens_input": 3427, "tokens_output": 49635, "tool_calls": 504, "tool_calls_by_class": { "cc_builtin": 484, "subagent": 20 }, "tool_result_tokens": 381414, "tool_time_ms": 9662, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5189774, "cost_usd": 3.5189774, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 708043, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 429, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 24, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12743289, "tokens_cache_write": 1079427, "tokens_input": 2858, "tokens_output": 55493, "tool_calls": 522, "tool_calls_by_class": { "cc_builtin": 492, "skill": 2, "subagent": 28 }, "tool_result_tokens": 414893, "tool_time_ms": 9250, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8793978999999994, "cost_usd": 2.8793978999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 586278, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 355, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11110739, "tokens_cache_write": 837465, "tokens_input": 2607, "tokens_output": 42744, "tool_calls": 482, "tool_calls_by_class": { "cc_builtin": 462, "subagent": 20 }, "tool_result_tokens": 400250, "tool_time_ms": 8149, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.95336655, "cost_usd": 2.95336655, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 705888, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 392, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12424063, "tokens_cache_write": 791028, "tokens_input": 2581, "tokens_output": 52215, "tool_calls": 486, "tool_calls_by_class": { "cc_builtin": 464, "skill": 1, "subagent": 21 }, "tool_result_tokens": 343881, "tool_time_ms": 8553, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6173557499999998, "cost_usd": 3.6173557499999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 546989, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 463, "model": "claude-haiku-4-5", "path": "runs/graphify/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14979390, "tokens_cache_write": 1021246, "tokens_input": 2991, "tokens_output": 53321, "tool_calls": 530, "tool_calls_by_class": { "cc_builtin": 507, "subagent": 23 }, "tool_result_tokens": 426262, "tool_time_ms": 9388, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.9970525, "cost_usd": 5.9970525, "denominator": 20, "duration_basis": "run", "duration_ms": 344682, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 128, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4078620, "tokens_cache_write": 347812, "tokens_input": 3860, "tokens_output": 27045, "tool_calls": 293, "tool_calls_by_class": { "cc_builtin": 281, "subagent": 12 }, "tool_result_tokens": 153956, "tool_time_ms": 5883, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.023920750000002, "cost_usd": 8.023920750000002, "denominator": 20, "duration_basis": "run", "duration_ms": 360370, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 181, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7024699, "tokens_cache_write": 418997, "tokens_input": 215, "tokens_output": 42747, "tool_calls": 310, "tool_calls_by_class": { "cc_builtin": 297, "subagent": 13 }, "tool_result_tokens": 222693, "tool_time_ms": 6893, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.025918749999997, "cost_usd": 10.025918749999997, "denominator": 20, "duration_basis": "run", "duration_ms": 571675, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 194, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9898055, "tokens_cache_write": 425523, "tokens_input": 222, "tokens_output": 41886, "tool_calls": 395, "tool_calls_by_class": { "cc_builtin": 384, "subagent": 11 }, "tool_result_tokens": 216724, "tool_time_ms": 11199, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.867660749999996, "cost_usd": 7.867660749999996, "denominator": 20, "duration_basis": "run", "duration_ms": 605299, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 174, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6575464, "tokens_cache_write": 409104, "tokens_input": 3027, "tokens_output": 37993, "tool_calls": 373, "tool_calls_by_class": { "cc_builtin": 362, "subagent": 11 }, "tool_result_tokens": 200390, "tool_time_ms": 7640, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.93069225, "cost_usd": 8.93069225, "denominator": 20, "duration_basis": "run", "duration_ms": 564003, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 187, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9443562, "tokens_cache_write": 344022, "tokens_input": 217, "tokens_output": 40701, "tool_calls": 360, "tool_calls_by_class": { "cc_builtin": 348, "subagent": 12 }, "tool_result_tokens": 155158, "tool_time_ms": 8114, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.846268499999999, "cost_usd": 4.846268499999999, "denominator": 20, "duration_basis": "run", "duration_ms": 472664, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 100, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2967567, "tokens_cache_write": 290826, "tokens_input": 3036, "tokens_output": 19732, "tool_calls": 230, "tool_calls_by_class": { "cc_builtin": 219, "subagent": 11 }, "tool_result_tokens": 134448, "tool_time_ms": 4969, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.116222999999999, "cost_usd": 6.116222999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 381364, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 153, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4237256, "tokens_cache_write": 341703, "tokens_input": 225, "tokens_output": 38676, "tool_calls": 288, "tool_calls_by_class": { "cc_builtin": 272, "subagent": 16 }, "tool_result_tokens": 140796, "tool_time_ms": 11474, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.569000250000001, "cost_usd": 6.569000250000001, "denominator": 20, "duration_basis": "run", "duration_ms": 310185, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 173, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4249573, "tokens_cache_write": 402384, "tokens_input": 239, "tokens_output": 27391, "tool_calls": 336, "tool_calls_by_class": { "cc_builtin": 315, "subagent": 21 }, "tool_result_tokens": 168307, "tool_time_ms": 7174, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.308133250000004, "cost_usd": 7.308133250000004, "denominator": 20, "duration_basis": "run", "duration_ms": 382015, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 169, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5249889, "tokens_cache_write": 399575, "tokens_input": 225, "tokens_output": 25699, "tool_calls": 383, "tool_calls_by_class": { "cc_builtin": 368, "subagent": 15 }, "tool_result_tokens": 197019, "tool_time_ms": 9759, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.981607750000003, "cost_usd": 6.981607750000003, "denominator": 20, "duration_basis": "run", "duration_ms": 406310, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 161, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6642728, "tokens_cache_write": 318658, "tokens_input": 3037, "tokens_output": 29882, "tool_calls": 339, "tool_calls_by_class": { "cc_builtin": 329, "subagent": 10 }, "tool_result_tokens": 172205, "tool_time_ms": 7211, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.0910945, "cost_usd": 5.0910945, "denominator": 30, "duration_basis": "run", "duration_ms": 224891, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3066769, "tokens_cache_write": 351613, "tokens_input": 1875, "tokens_output": 24473, "tool_calls": 236, "tool_calls_by_class": { "cc_builtin": 223, "subagent": 13 }, "tool_result_tokens": 185223, "tool_time_ms": 4567, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.055421250000001, "cost_usd": 5.055421250000001, "denominator": 30, "duration_basis": "run", "duration_ms": 333044, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 72, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4150520, "tokens_cache_write": 293549, "tokens_input": 1056, "tokens_output": 20367, "tool_calls": 192, "tool_calls_by_class": { "cc_builtin": 187, "subagent": 5 }, "tool_result_tokens": 175051, "tool_time_ms": 3888, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.559506249999999, "cost_usd": 4.559506249999999, "denominator": 30, "duration_basis": "run", "duration_ms": 290496, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 93, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2730175, "tokens_cache_write": 308456, "tokens_input": 141, "tokens_output": 21962, "tool_calls": 232, "tool_calls_by_class": { "cc_builtin": 220, "subagent": 12 }, "tool_result_tokens": 151443, "tool_time_ms": 4567, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.385693750000003, "cost_usd": 6.385693750000003, "denominator": 30, "duration_basis": "run", "duration_ms": 291662, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 144, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4185445, "tokens_cache_write": 408863, "tokens_input": 192, "tokens_output": 34095, "tool_calls": 327, "tool_calls_by_class": { "cc_builtin": 310, "subagent": 17 }, "tool_result_tokens": 209058, "tool_time_ms": 5623, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.911135999999996, "cost_usd": 6.911135999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 258697, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 162, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4232733, "tokens_cache_write": 445450, "tokens_input": 1022, "tokens_output": 33284, "tool_calls": 352, "tool_calls_by_class": { "cc_builtin": 328, "subagent": 24 }, "tool_result_tokens": 198345, "tool_time_ms": 5143, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.763349249999999, "cost_usd": 5.763349249999999, "denominator": 30, "duration_basis": "run", "duration_ms": 247543, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3825361, "tokens_cache_write": 374909, "tokens_input": 901, "tokens_output": 27844, "tool_calls": 238, "tool_calls_by_class": { "cc_builtin": 229, "subagent": 9 }, "tool_result_tokens": 204485, "tool_time_ms": 4225, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.561566249999999, "cost_usd": 6.561566249999999, "denominator": 30, "duration_basis": "run", "duration_ms": 356156, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 162, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4402710, "tokens_cache_write": 411184, "tokens_input": 228, "tokens_output": 35563, "tool_calls": 322, "tool_calls_by_class": { "cc_builtin": 304, "subagent": 18 }, "tool_result_tokens": 202514, "tool_time_ms": 5513, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.243813749999999, "cost_usd": 4.243813749999999, "denominator": 30, "duration_basis": "run", "duration_ms": 277569, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 56, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3119360, "tokens_cache_write": 268380, "tokens_input": 66, "tokens_output": 18154, "tool_calls": 175, "tool_calls_by_class": { "cc_builtin": 171, "subagent": 4 }, "tool_result_tokens": 153588, "tool_time_ms": 3670, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.1007955, "cost_usd": 6.1007955, "denominator": 30, "duration_basis": "run", "duration_ms": 297295, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 126, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3457421, "tokens_cache_write": 415841, "tokens_input": 3245, "tokens_output": 28557, "tool_calls": 347, "tool_calls_by_class": { "cc_builtin": 329, "subagent": 18 }, "tool_result_tokens": 206781, "tool_time_ms": 5818, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2533172499999985, "cost_usd": 6.2533172499999985, "denominator": 30, "duration_basis": "run", "duration_ms": 334919, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 108, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4153132, "tokens_cache_write": 403261, "tokens_input": 477, "tokens_output": 35344, "tool_calls": 293, "tool_calls_by_class": { "cc_builtin": 282, "subagent": 11 }, "tool_result_tokens": 216588, "tool_time_ms": 5727, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.579876749999999, "cost_usd": 8.579876749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1116488, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 152, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10508316, "tokens_cache_write": 290158, "tokens_input": 200, "tokens_output": 32340, "tool_calls": 219, "tool_calls_by_class": { "cc_builtin": 215, "subagent": 4 }, "tool_result_tokens": 140634, "tool_time_ms": 9208, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.21788175, "cost_usd": 5.21788175, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 941974, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 118, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6480766, "tokens_cache_write": 167065, "tokens_input": 256, "tokens_output": 23650, "tool_calls": 142, "tool_calls_by_class": { "cc_builtin": 139, "subagent": 3 }, "tool_result_tokens": 88423, "tool_time_ms": 4662, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.7339215, "cost_usd": 9.7339215, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1418433, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 166, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11660833, "tokens_cache_write": 352526, "tokens_input": 5556, "tokens_output": 34494, "tool_calls": 260, "tool_calls_by_class": { "cc_builtin": 254, "subagent": 6 }, "tool_result_tokens": 201110, "tool_time_ms": 7917, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.132560750000003, "cost_usd": 9.132560750000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1600367, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 177, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9353874, "tokens_cache_write": 410949, "tokens_input": 9285, "tokens_output": 35515, "tool_calls": 363, "tool_calls_by_class": { "cc_builtin": 355, "subagent": 8 }, "tool_result_tokens": 233054, "tool_time_ms": 9826, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.6353455, "cost_usd": 6.6353455, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1186176, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 144, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7833191, "tokens_cache_write": 233182, "tokens_input": 284, "tokens_output": 24704, "tool_calls": 224, "tool_calls_by_class": { "cc_builtin": 220, "subagent": 4 }, "tool_result_tokens": 118908, "tool_time_ms": 6582, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.289147250000001, "cost_usd": 9.289147250000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1235221, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 170, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12636452, "tokens_cache_write": 247183, "tokens_input": 218, "tokens_output": 29691, "tool_calls": 219, "tool_calls_by_class": { "cc_builtin": 215, "subagent": 4 }, "tool_result_tokens": 135665, "tool_time_ms": 7016, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.242752500000001, "cost_usd": 7.242752500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1021406, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 132, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9879825, "tokens_cache_write": 186295, "tokens_input": 5371, "tokens_output": 26348, "tool_calls": 155, "tool_calls_by_class": { "cc_builtin": 152, "subagent": 3 }, "tool_result_tokens": 101059, "tool_time_ms": 8096, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.85562475, "cost_usd": 7.85562475, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1383339, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 160, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9083302, "tokens_cache_write": 280399, "tokens_input": 7506, "tokens_output": 32027, "tool_calls": 252, "tool_calls_by_class": { "cc_builtin": 247, "subagent": 5 }, "tool_result_tokens": 144314, "tool_time_ms": 10089, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.864674999999999, "cost_usd": 7.864674999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1270090, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 166, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11081180, "tokens_cache_write": 178121, "tokens_input": 304, "tokens_output": 32875, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 181, "subagent": 3 }, "tool_result_tokens": 89216, "tool_time_ms": 8283, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.561453999999999, "cost_usd": 7.561453999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 867837, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 142, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11297123, "tokens_cache_write": 140826, "tokens_input": 278, "tokens_output": 24318, "tool_calls": 147, "tool_calls_by_class": { "cc_builtin": 145, "subagent": 2 }, "tool_result_tokens": 66308, "tool_time_ms": 6878, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.826307499999999, "cost_usd": 3.826307499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 576211, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 112, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5514355, "tokens_cache_write": 73727, "tokens_input": 172, "tokens_output": 13240, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 82 }, "tool_result_tokens": 18556, "tool_time_ms": 2727, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2413994999999995, "cost_usd": 3.2413994999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 591091, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4967409, "tokens_cache_write": 45028, "tokens_input": 163, "tokens_output": 12264, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 73 }, "tool_result_tokens": 16763, "tool_time_ms": 2775, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0213465, "cost_usd": 3.0213465, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 522131, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4575753, "tokens_cache_write": 43992, "tokens_input": 525, "tokens_output": 11637, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 76 }, "tool_result_tokens": 16222, "tool_time_ms": 2608, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6017305000000004, "cost_usd": 3.6017305000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 638244, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 111, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5552861, "tokens_cache_write": 49687, "tokens_input": 171, "tokens_output": 13103, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 82 }, "tool_result_tokens": 19257, "tool_time_ms": 2747, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4043669999999997, "cost_usd": 3.4043669999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 555675, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5169364, "tokens_cache_write": 49339, "tokens_input": 534, "tokens_output": 12945, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 83 }, "tool_result_tokens": 19276, "tool_time_ms": 2964, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1546800000000004, "cost_usd": 3.1546800000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 530471, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4847030, "tokens_cache_write": 45596, "tokens_input": 161, "tokens_output": 10976, "tool_calls": 80, "tool_calls_by_class": { "cc_builtin": 80 }, "tool_result_tokens": 17680, "tool_time_ms": 2801, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5680435, "cost_usd": 3.5680435, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 567881, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 113, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5616027, "tokens_cache_write": 43294, "tokens_input": 173, "tokens_output": 13049, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 83 }, "tool_result_tokens": 19995, "tool_time_ms": 2687, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8727499999999995, "cost_usd": 3.8727499999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 628909, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 117, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6049080, "tokens_cache_write": 48220, "tokens_input": 177, "tokens_output": 14605, "tool_calls": 91, "tool_calls_by_class": { "cc_builtin": 91 }, "tool_result_tokens": 21591, "tool_time_ms": 3272, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9402485, "cost_usd": 2.9402485, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 564383, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 91, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4443617, "tokens_cache_write": 42976, "tokens_input": 151, "tokens_output": 11517, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 69 }, "tool_result_tokens": 20664, "tool_time_ms": 2794, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4567219999999996, "cost_usd": 3.4567219999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 624058, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5294454, "tokens_cache_write": 48305, "tokens_input": 164, "tokens_output": 13025, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 85 }, "tool_result_tokens": 18204, "tool_time_ms": 3061, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.50540775, "cost_usd": 12.50540775, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2296390, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 221, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8119048, "tokens_cache_write": 892381, "tokens_input": 1725, "tokens_output": 48436, "tool_calls": 532, "tool_calls_by_class": { "cc_builtin": 518, "subagent": 14 }, "tool_result_tokens": 412476, "tool_time_ms": 10898, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.166969499999999, "cost_usd": 13.166969499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2368254, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 235, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7422784, "tokens_cache_write": 991761, "tokens_input": 5546, "tokens_output": 53427, "tool_calls": 512, "tool_calls_by_class": { "cc_builtin": 494, "subagent": 18 }, "tool_result_tokens": 420124, "tool_time_ms": 12694, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.270259249999999, "cost_usd": 13.270259249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2057551, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 237, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8220396, "tokens_cache_write": 998564, "tokens_input": 4102, "tokens_output": 53501, "tool_calls": 556, "tool_calls_by_class": { "cc_builtin": 539, "subagent": 17 }, "tool_result_tokens": 470706, "tool_time_ms": 9226, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.322483249999998, "cost_usd": 13.322483249999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2496819, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 253, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9190029, "tokens_cache_write": 907249, "tokens_input": 4057, "tokens_output": 62107, "tool_calls": 536, "tool_calls_by_class": { "cc_builtin": 520, "subagent": 16 }, "tool_result_tokens": 404214, "tool_time_ms": 15352, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.399109750000001, "cost_usd": 13.399109750000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2384761, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 253, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8819952, "tokens_cache_write": 951232, "tokens_input": 1683, "tokens_output": 51520, "tool_calls": 543, "tool_calls_by_class": { "cc_builtin": 527, "subagent": 16 }, "tool_result_tokens": 441388, "tool_time_ms": 11767, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.077637999999997, "cost_usd": 13.077637999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2170230, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 242, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7800441, "tokens_cache_write": 979292, "tokens_input": 1601, "tokens_output": 55215, "tool_calls": 556, "tool_calls_by_class": { "cc_builtin": 538, "subagent": 18 }, "tool_result_tokens": 437759, "tool_time_ms": 9143, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.681431000000002, "cost_usd": 13.681431000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2440521, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 250, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8774412, "tokens_cache_write": 988429, "tokens_input": 3874, "tokens_output": 55600, "tool_calls": 554, "tool_calls_by_class": { "cc_builtin": 537, "subagent": 17 }, "tool_result_tokens": 462563, "tool_time_ms": 11746, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.18511075, "cost_usd": 13.18511075, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2558428, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 222, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7540089, "tokens_cache_write": 975888, "tokens_input": 597, "tokens_output": 55636, "tool_calls": 524, "tool_calls_by_class": { "cc_builtin": 506, "subagent": 18 }, "tool_result_tokens": 448333, "tool_time_ms": 12993, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.984293249999999, "cost_usd": 12.984293249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2430885, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 238, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8130509, "tokens_cache_write": 941138, "tokens_input": 361, "tokens_output": 56012, "tool_calls": 543, "tool_calls_by_class": { "cc_builtin": 526, "subagent": 17 }, "tool_result_tokens": 431498, "tool_time_ms": 9051, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.516336249999998, "cost_usd": 13.516336249999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2033541, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 235, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8346995, "tokens_cache_write": 1036431, "tokens_input": 11747, "tokens_output": 50972, "tool_calls": 504, "tool_calls_by_class": { "cc_builtin": 488, "subagent": 16 }, "tool_result_tokens": 493243, "tool_time_ms": 11852, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.609076499999999, "cost_usd": 8.609076499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1475425, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 172, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5030788, "tokens_cache_write": 630542, "tokens_input": 1962, "tokens_output": 35318, "tool_calls": 334, "tool_calls_by_class": { "cc_builtin": 323, "subagent": 11 }, "tool_result_tokens": 271247, "tool_time_ms": 7916, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.28479075, "cost_usd": 9.28479075, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1552370, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 168, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4918819, "tokens_cache_write": 755634, "tokens_input": 1035, "tokens_output": 36974, "tool_calls": 333, "tool_calls_by_class": { "cc_builtin": 319, "subagent": 14 }, "tool_result_tokens": 349540, "tool_time_ms": 8782, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.465661, "cost_usd": 9.465661, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1476364, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 199, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5922307, "tokens_cache_write": 679089, "tokens_input": 2314, "tokens_output": 36280, "tool_calls": 383, "tool_calls_by_class": { "cc_builtin": 367, "subagent": 16 }, "tool_result_tokens": 280679, "tool_time_ms": 9020, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.56229875, "cost_usd": 9.56229875, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1489655, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 171, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5231775, "tokens_cache_write": 768897, "tokens_input": 1318, "tokens_output": 39421, "tool_calls": 326, "tool_calls_by_class": { "cc_builtin": 315, "subagent": 11 }, "tool_result_tokens": 354039, "tool_time_ms": 8414, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.464788749999999, "cost_usd": 8.464788749999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1422786, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 173, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5274890, "tokens_cache_write": 608536, "tokens_input": 253, "tokens_output": 30790, "tool_calls": 313, "tool_calls_by_class": { "cc_builtin": 300, "subagent": 13 }, "tool_result_tokens": 252866, "tool_time_ms": 7472, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.605924250000001, "cost_usd": 8.605924250000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1578496, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 179, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5197011, "tokens_cache_write": 617129, "tokens_input": 925, "tokens_output": 34283, "tool_calls": 359, "tool_calls_by_class": { "cc_builtin": 345, "subagent": 14 }, "tool_result_tokens": 253843, "tool_time_ms": 7439, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.5566425, "cost_usd": 8.5566425, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1467062, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 174, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4869090, "tokens_cache_write": 644006, "tokens_input": 258, "tokens_output": 34606, "tool_calls": 338, "tool_calls_by_class": { "cc_builtin": 324, "subagent": 14 }, "tool_result_tokens": 260117, "tool_time_ms": 7249, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.013798749999998, "cost_usd": 9.013798749999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1581552, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 172, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5019315, "tokens_cache_write": 697116, "tokens_input": 256, "tokens_output": 34621, "tool_calls": 359, "tool_calls_by_class": { "cc_builtin": 345, "subagent": 14 }, "tool_result_tokens": 301947, "tool_time_ms": 8148, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.8936695, "cost_usd": 9.8936695, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1802215, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 192, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5748419, "tokens_cache_write": 754448, "tokens_input": 4239, "tokens_output": 40616, "tool_calls": 382, "tool_calls_by_class": { "cc_builtin": 367, "subagent": 15 }, "tool_result_tokens": 335517, "tool_time_ms": 10729, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.21596625, "cost_usd": 9.21596625, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1665403, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 182, "model": "claude-opus-4-6", "path": "runs/graphify/claude-opus-4-6/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5204305, "tokens_cache_write": 720580, "tokens_input": 1559, "tokens_output": 31138, "tool_calls": 376, "tool_calls_by_class": { "cc_builtin": 362, "subagent": 14 }, "tool_result_tokens": 317179, "tool_time_ms": 8085, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.933553, "cost_usd": 4.933553, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 553133, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5468126, "tokens_cache_write": 130661, "tokens_input": 153, "tokens_output": 36735, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 63, "subagent": 1 }, "tool_result_tokens": 28215, "tool_time_ms": 3141, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.283521499999999, "cost_usd": 4.283521499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 507668, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 78, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5503713, "tokens_cache_write": 81623, "tokens_input": 147, "tokens_output": 28588, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 26163, "tool_time_ms": 2816, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.429111499999999, "cost_usd": 5.429111499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 626209, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7195063, "tokens_cache_write": 94711, "tokens_input": 1814, "tokens_output": 35016, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 71, "skill": 1 }, "tool_result_tokens": 20088, "tool_time_ms": 3613, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.1260485, "cost_usd": 6.1260485, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 698644, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8094357, "tokens_cache_write": 108740, "tokens_input": 174, "tokens_output": 39624, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 71, "skill": 1 }, "tool_result_tokens": 25802, "tool_time_ms": 8782, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.4601915000000005, "cost_usd": 4.4601915000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 553204, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5693013, "tokens_cache_write": 80371, "tokens_input": 155, "tokens_output": 32368, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 23041, "tool_time_ms": 2886, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.5032425, "cost_usd": 6.5032425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 719886, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 98, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8824125, "tokens_cache_write": 107353, "tokens_input": 185, "tokens_output": 40669, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 77, "skill": 1 }, "tool_result_tokens": 24337, "tool_time_ms": 4204, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.061169500000001, "cost_usd": 5.061169500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 644510, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 91, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6475179, "tokens_cache_write": 90086, "tokens_input": 169, "tokens_output": 36875, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 25983, "tool_time_ms": 4202, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.839279999999999, "cost_usd": 4.839279999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 564553, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 76, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6038670, "tokens_cache_write": 98121, "tokens_input": 142, "tokens_output": 33521, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 32880, "tool_time_ms": 3379, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.226393000000001, "cost_usd": 5.226393000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 661186, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 90, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6681696, "tokens_cache_write": 89094, "tokens_input": 171, "tokens_output": 39750, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 23736, "tool_time_ms": 7479, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1547909999999995, "cost_usd": 4.1547909999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 506855, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 86, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5441802, "tokens_cache_write": 70824, "tokens_input": 160, "tokens_output": 28994, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 76 }, "tool_result_tokens": 19201, "tool_time_ms": 2802, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2611495, "cost_usd": 3.2611495, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 394946, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4142919, "tokens_cache_write": 73080, "tokens_input": 168, "tokens_output": 18322, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 11843, "tool_time_ms": 2681, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6510155, "cost_usd": 2.6510155, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 467091, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3584821, "tokens_cache_write": 43074, "tokens_input": 153, "tokens_output": 17084, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 9753, "tool_time_ms": 2354, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7948640000000005, "cost_usd": 3.7948640000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 490713, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 78, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5258918, "tokens_cache_write": 69684, "tokens_input": 148, "tokens_output": 18713, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 47, "skill": 1 }, "tool_result_tokens": 13928, "tool_time_ms": 2797, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0704425000000004, "cost_usd": 3.0704425000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 486801, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4285345, "tokens_cache_write": 49222, "tokens_input": 170, "tokens_output": 17388, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 12995, "tool_time_ms": 2673, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1628765, "cost_usd": 3.1628765, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 412785, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4222203, "tokens_cache_write": 53135, "tokens_input": 160, "tokens_output": 20785, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 13221, "tool_time_ms": 2565, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1345405, "cost_usd": 3.1345405, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 446099, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4311801, "tokens_cache_write": 50022, "tokens_input": 169, "tokens_output": 19103, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 12146, "tool_time_ms": 2839, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.14544, "cost_usd": 3.14544, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 403257, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4388420, "tokens_cache_write": 49743, "tokens_input": 170, "tokens_output": 18118, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 12546, "tool_time_ms": 2630, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4511225000000003, "cost_usd": 3.4511225000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 446030, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4807575, "tokens_cache_write": 52131, "tokens_input": 175, "tokens_output": 21006, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 14598, "tool_time_ms": 2557, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0280889999999996, "cost_usd": 3.0280889999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 356546, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4276548, "tokens_cache_write": 43714, "tokens_input": 170, "tokens_output": 18073, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 12250, "tool_time_ms": 2359, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5300344999999997, "cost_usd": 3.5300344999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 422876, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4195229, "tokens_cache_write": 89632, "tokens_input": 160, "tokens_output": 21412, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 13461, "tool_time_ms": 3105, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.6933279999999999, "cost_usd": 0.6933279999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 69147, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 465486, "tokens_cache_write": 37497, "tokens_input": 23, "tokens_output": 3420, "tool_calls": 12, "tool_calls_by_class": { "cc_builtin": 11, "skill": 1 }, "tool_result_tokens": 6841, "tool_time_ms": 785, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.732944, "cost_usd": 0.732944, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 70114, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 227593, "tokens_cache_write": 55462, "tokens_input": 1676, "tokens_output": 3529, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 12, "subagent": 2 }, "tool_result_tokens": 6895, "tool_time_ms": 375, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.9755165000000001, "cost_usd": 0.9755165000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 69882, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 15, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 651553, "tokens_cache_write": 55692, "tokens_input": 29, "tokens_output": 3707, "tool_calls": 13, "tool_calls_by_class": { "cc_builtin": 11, "skill": 2 }, "tool_result_tokens": 7359, "tool_time_ms": 1354, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.8738217500000001, "cost_usd": 0.8738217500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 42095, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 16, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 455656, "tokens_cache_write": 55260, "tokens_input": 32, "tokens_output": 5096, "tool_calls": 16, "tool_calls_by_class": { "cc_builtin": 14, "skill": 1, "subagent": 1 }, "tool_result_tokens": 9592, "tool_time_ms": 1192, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.6033155, "cost_usd": 0.6033155, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 52961, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 446241, "tokens_cache_write": 30840, "tokens_input": 24, "tokens_output": 2867, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 9, "skill": 1 }, "tool_result_tokens": 4658, "tool_time_ms": 1055, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.5657909999999999, "cost_usd": 0.5657909999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 59187, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 8, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 284752, "tokens_cache_write": 34781, "tokens_input": 16, "tokens_output": 3021, "tool_calls": 8, "tool_calls_by_class": { "cc_builtin": 7, "skill": 1 }, "tool_result_tokens": 5555, "tool_time_ms": 820, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.59465, "cost_usd": 0.59465, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 57967, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 348290, "tokens_cache_write": 34938, "tokens_input": 20, "tokens_output": 2841, "tool_calls": 8, "tool_calls_by_class": { "cc_builtin": 7, "skill": 1 }, "tool_result_tokens": 5319, "tool_time_ms": 769, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.5459732500000001, "cost_usd": 0.5459732500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 72661, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 227934, "tokens_cache_write": 40387, "tokens_input": 1674, "tokens_output": 3744, "tool_calls": 13, "tool_calls_by_class": { "cc_builtin": 11, "subagent": 2 }, "tool_result_tokens": 6877, "tool_time_ms": 561, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.314357, "cost_usd": 0.314357, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 49897, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 10, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 301724, "tokens_cache_write": 9142, "tokens_input": 20, "tokens_output": 2879, "tool_calls": 10, "tool_calls_by_class": { "cc_builtin": 10 }, "tool_result_tokens": 4380, "tool_time_ms": 259, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.297534, "cost_usd": 0.297534, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 46827, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 9, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 271878, "tokens_cache_write": 8182, "tokens_input": 1670, "tokens_output": 2857, "tool_calls": 9, "tool_calls_by_class": { "cc_builtin": 9 }, "tool_result_tokens": 4968, "tool_time_ms": 328, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.094898249999999, "cost_usd": 7.094898249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 976577, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 136, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|11", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4861154, "tokens_cache_write": 324347, "tokens_input": 264, "tokens_output": 55046, "tool_calls": 155, "tool_calls_by_class": { "cc_builtin": 147, "skill": 6, "subagent": 2 }, "tool_result_tokens": 65849, "tool_time_ms": 11087, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.40258725, "cost_usd": 8.40258725, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1011620, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 144, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|12", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5644356, "tokens_cache_write": 413812, "tokens_input": 274, "tokens_output": 58339, "tool_calls": 154, "tool_calls_by_class": { "cc_builtin": 142, "skill": 10, "subagent": 2 }, "tool_result_tokens": 82689, "tool_time_ms": 11911, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.725890000000001, "cost_usd": 7.725890000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 981408, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 138, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|13", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5189600, "tokens_cache_write": 370452, "tokens_input": 266, "tokens_output": 58705, "tool_calls": 145, "tool_calls_by_class": { "cc_builtin": 137, "skill": 6, "subagent": 2 }, "tool_result_tokens": 96235, "tool_time_ms": 9952, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.446830000000002, "cost_usd": 8.446830000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 984408, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 150, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|14", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5802985, "tokens_cache_write": 422354, "tokens_input": 288, "tokens_output": 53870, "tool_calls": 151, "tool_calls_by_class": { "cc_builtin": 138, "skill": 11, "subagent": 2 }, "tool_result_tokens": 76539, "tool_time_ms": 11384, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.975160500000001, "cost_usd": 7.975160500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 947597, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 146, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|15", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5919211, "tokens_cache_write": 365489, "tokens_input": 283, "tokens_output": 54370, "tool_calls": 152, "tool_calls_by_class": { "cc_builtin": 143, "skill": 9 }, "tool_result_tokens": 69488, "tool_time_ms": 9511, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.053721, "cost_usd": 8.053721, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 936986, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 152, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep16", "query_calls": 0, "rep": 16, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|16", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5495202, "tokens_cache_write": 373302, "tokens_input": 1946, "tokens_output": 59138, "tool_calls": 157, "tool_calls_by_class": { "cc_builtin": 148, "skill": 8, "subagent": 1 }, "tool_result_tokens": 71202, "tool_time_ms": 9654, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.382379249999996, "cost_usd": 9.382379249999996, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1058930, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 174, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep17", "query_calls": 0, "rep": 17, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|17", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6576566, "tokens_cache_write": 444241, "tokens_input": 338, "tokens_output": 68980, "tool_calls": 178, "tool_calls_by_class": { "cc_builtin": 167, "skill": 9, "subagent": 2 }, "tool_result_tokens": 93400, "tool_time_ms": 12684, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.4824715, "cost_usd": 8.4824715, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 953077, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 148, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep18", "query_calls": 0, "rep": 18, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|18", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5873372, "tokens_cache_write": 419336, "tokens_input": 285, "tokens_output": 57169, "tool_calls": 158, "tool_calls_by_class": { "cc_builtin": 147, "skill": 9, "subagent": 2 }, "tool_result_tokens": 92764, "tool_time_ms": 10843, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.1538065, "cost_usd": 9.1538065, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 837712, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 168, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep19", "query_calls": 0, "rep": 19, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|19", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5934188, "tokens_cache_write": 471577, "tokens_input": 613, "tokens_output": 63548, "tool_calls": 185, "tool_calls_by_class": { "cc_builtin": 173, "skill": 9, "subagent": 3 }, "tool_result_tokens": 99127, "tool_time_ms": 10132, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.66244575, "cost_usd": 7.66244575, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 895424, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 146, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos-hard/rep20", "query_calls": 0, "rep": 20, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|20", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5247659, "tokens_cache_write": 360835, "tokens_input": 280, "tokens_output": 59659, "tool_calls": 151, "tool_calls_by_class": { "cc_builtin": 142, "skill": 6, "subagent": 3 }, "tool_result_tokens": 82298, "tool_time_ms": 8889, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.205154, "cost_usd": 0.205154, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 24261, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 138988, "tokens_cache_write": 11371, "tokens_input": 10, "tokens_output": 876, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 1790, "tool_time_ms": 1884, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.20516600000000002, "cost_usd": 0.20516600000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 23461, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 139322, "tokens_cache_write": 11148, "tokens_input": 10, "tokens_output": 959, "tool_calls": 3, "tool_calls_by_class": { "cc_builtin": 3 }, "tool_result_tokens": 1700, "tool_time_ms": 1575, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2054075, "cost_usd": 0.2054075, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 28133, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 168925, "tokens_cache_write": 9611, "tokens_input": 12, "tokens_output": 991, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 828, "tool_time_ms": 2047, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2108985, "cost_usd": 0.2108985, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 25189, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 138087, "tokens_cache_write": 11623, "tokens_input": 10, "tokens_output": 1023, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 950, "tool_time_ms": 2461, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.18391600000000002, "cost_usd": 0.18391600000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 29721, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 171842, "tokens_cache_write": 7001, "tokens_input": 12, "tokens_output": 1117, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 835, "tool_time_ms": 2812, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2247105, "cost_usd": 0.2247105, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 26307, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 167451, "tokens_cache_write": 11215, "tokens_input": 12, "tokens_output": 1151, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 632, "tool_time_ms": 1486, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.226896, "cost_usd": 0.226896, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 28721, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 167442, "tokens_cache_write": 10929, "tokens_input": 12, "tokens_output": 1353, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 449, "tool_time_ms": 1505, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2102665, "cost_usd": 0.2102665, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 24949, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 169103, "tokens_cache_write": 9688, "tokens_input": 12, "tokens_output": 1151, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 775, "tool_time_ms": 2167, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.4287055000000001, "cost_usd": 0.4287055000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 44283, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 8, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 287351, "tokens_cache_write": 23600, "tokens_input": 16, "tokens_output": 1958, "tool_calls": 7, "tool_calls_by_class": { "cc_builtin": 6, "skill": 1 }, "tool_result_tokens": 2936, "tool_time_ms": 6466, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.14509899999999998, "cost_usd": 0.14509899999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 22361, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 5, "model": "claude-opus-4-8", "path": "runs/graphify/claude-opus-4-8/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 146428, "tokens_cache_write": 4179, "tokens_input": 494, "tokens_output": 1105, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 1750, "tool_time_ms": 1839, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8671670000000007, "cost_usd": 3.8671670000000007, "denominator": 20, "duration_basis": "run", "duration_ms": 394561, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 46, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3878304, "tokens_cache_write": 127144, "tokens_input": 85, "tokens_output": 26246, "tool_calls": 68, "tool_calls_by_class": { "cc_builtin": 68 }, "tool_result_tokens": 43365, "tool_time_ms": 7396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.693698, "cost_usd": 2.693698, "denominator": 20, "duration_basis": "run", "duration_ms": 341116, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 30, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2377856, "tokens_cache_write": 90719, "tokens_input": 56, "tokens_output": 23892, "tool_calls": 46, "tool_calls_by_class": { "cc_builtin": 46 }, "tool_result_tokens": 37455, "tool_time_ms": 8292, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3476525000000006, "cost_usd": 3.3476525000000006, "denominator": 20, "duration_basis": "run", "duration_ms": 408807, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 39, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3490785, "tokens_cache_write": 97520, "tokens_input": 72, "tokens_output": 25068, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 46260, "tool_time_ms": 24575, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.084358999999999, "cost_usd": 4.084358999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 450794, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 48, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4467058, "tokens_cache_write": 106281, "tokens_input": 89, "tokens_output": 31503, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 45931, "tool_time_ms": 4199, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.059823, "cost_usd": 3.059823, "denominator": 20, "duration_basis": "run", "duration_ms": 387248, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 45, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3282316, "tokens_cache_write": 81590, "tokens_input": 83, "tokens_output": 24094, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 30952, "tool_time_ms": 4536, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5293484999999998, "cost_usd": 3.5293484999999998, "denominator": 20, "duration_basis": "run", "duration_ms": 381449, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 45, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3776197, "tokens_cache_write": 99821, "tokens_input": 83, "tokens_output": 25705, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 41004, "tool_time_ms": 6367, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0840509999999997, "cost_usd": 3.0840509999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 396486, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 36, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2856082, "tokens_cache_write": 94300, "tokens_input": 67, "tokens_output": 28507, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 36363, "tool_time_ms": 4248, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0430960000000002, "cost_usd": 3.0430960000000002, "denominator": 20, "duration_basis": "run", "duration_ms": 428741, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 40, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3154672, "tokens_cache_write": 85054, "tokens_input": 74, "tokens_output": 24594, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 33042, "tool_time_ms": 17205, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4675754999999997, "cost_usd": 3.4675754999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 451140, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 39, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3290441, "tokens_cache_write": 106217, "tokens_input": 72, "tokens_output": 30393, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 42020, "tool_time_ms": 25809, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2449675, "cost_usd": 3.2449675, "denominator": 20, "duration_basis": "run", "duration_ms": 378200, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 44, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3423095, "tokens_cache_write": 90739, "tokens_input": 81, "tokens_output": 25025, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 37114, "tool_time_ms": 8520, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0676930000000002, "cost_usd": 2.0676930000000002, "denominator": 30, "duration_basis": "run", "duration_ms": 251723, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 28, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1781586, "tokens_cache_write": 72892, "tokens_input": 51, "tokens_output": 17909, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 28698, "tool_time_ms": 1769, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6949070000000002, "cost_usd": 1.6949070000000002, "denominator": 30, "duration_basis": "run", "duration_ms": 228155, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1446674, "tokens_cache_write": 56220, "tokens_input": 44, "tokens_output": 16366, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 25442, "tool_time_ms": 1562, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6621739999999998, "cost_usd": 1.6621739999999998, "denominator": 30, "duration_basis": "run", "duration_ms": 213581, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 20, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1203078, "tokens_cache_write": 67008, "tokens_input": 36, "tokens_output": 15615, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 26717, "tool_time_ms": 1474, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7421195, "cost_usd": 2.7421195, "denominator": 30, "duration_basis": "run", "duration_ms": 330852, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 36, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2687919, "tokens_cache_write": 86165, "tokens_input": 67, "tokens_output": 21447, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 33610, "tool_time_ms": 1816, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.241583, "cost_usd": 4.241583, "denominator": 30, "duration_basis": "run", "duration_ms": 409988, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 63, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5582086, "tokens_cache_write": 93294, "tokens_input": 120, "tokens_output": 20680, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 61, "skill": 1 }, "tool_result_tokens": 35878, "tool_time_ms": 2409, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.7877969999999999, "cost_usd": 1.7877969999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 231053, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1501654, "tokens_cache_write": 61920, "tokens_input": 44, "tokens_output": 16702, "tool_calls": 44, "tool_calls_by_class": { "cc_builtin": 44 }, "tool_result_tokens": 27111, "tool_time_ms": 1703, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.7842065, "cost_usd": 1.7842065, "denominator": 30, "duration_basis": "run", "duration_ms": 238338, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 25, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1454323, "tokens_cache_write": 65749, "tokens_input": 46, "tokens_output": 15973, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 48 }, "tool_result_tokens": 25488, "tool_time_ms": 1542, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6096855000000003, "cost_usd": 2.6096855000000003, "denominator": 30, "duration_basis": "run", "duration_ms": 320601, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 35, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2496491, "tokens_cache_write": 84109, "tokens_input": 65, "tokens_output": 20801, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 33346, "tool_time_ms": 1926, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.743546, "cost_usd": 2.743546, "denominator": 30, "duration_basis": "run", "duration_ms": 433849, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 50, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3184802, "tokens_cache_write": 68777, "tokens_input": 95, "tokens_output": 18516, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 26353, "tool_time_ms": 2004, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.850926, "cost_usd": 1.850926, "denominator": 30, "duration_basis": "run", "duration_ms": 275548, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 28, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1636092, "tokens_cache_write": 65082, "tokens_input": 52, "tokens_output": 15272, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 47 }, "tool_result_tokens": 26051, "tool_time_ms": 1533, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.500259999999998, "cost_usd": 7.500259999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 908698, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 110, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9858120, "tokens_cache_write": 148153, "tokens_input": 209, "tokens_output": 43545, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 90 }, "tool_result_tokens": 40645, "tool_time_ms": 7038, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.133869499999999, "cost_usd": 6.133869499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 639347, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8099469, "tokens_cache_write": 118031, "tokens_input": 175, "tokens_output": 36118, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 42674, "tool_time_ms": 8123, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.5722429999999985, "cost_usd": 7.5722429999999985, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 823794, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 112, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10361706, "tokens_cache_write": 124085, "tokens_input": 213, "tokens_output": 45979, "tool_calls": 92, "tool_calls_by_class": { "cc_builtin": 92 }, "tool_result_tokens": 40330, "tool_time_ms": 16454, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.000183500000002, "cost_usd": 7.000183500000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 800633, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 100, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9202367, "tokens_cache_write": 127090, "tokens_input": 190, "tokens_output": 45086, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 88 }, "tool_result_tokens": 42614, "tool_time_ms": 10981, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.883232499999999, "cost_usd": 6.883232499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 702450, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9414075, "tokens_cache_write": 122745, "tokens_input": 194, "tokens_output": 37911, "tool_calls": 82, "tool_calls_by_class": { "cc_builtin": 82 }, "tool_result_tokens": 44203, "tool_time_ms": 5769, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.038971000000001, "cost_usd": 6.038971000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 663692, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7755652, "tokens_cache_write": 121423, "tokens_input": 163, "tokens_output": 37844, "tool_calls": 102, "tool_calls_by_class": { "cc_builtin": 102 }, "tool_result_tokens": 41902, "tool_time_ms": 16712, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.344346499999999, "cost_usd": 6.344346499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 681242, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 109, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8768683, "tokens_cache_write": 104487, "tokens_input": 207, "tokens_output": 36564, "tool_calls": 91, "tool_calls_by_class": { "cc_builtin": 91 }, "tool_result_tokens": 34271, "tool_time_ms": 7553, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.010853499999999, "cost_usd": 6.010853499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 607036, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8100027, "tokens_cache_write": 109552, "tokens_input": 179, "tokens_output": 34577, "tool_calls": 75, "tool_calls_by_class": { "cc_builtin": 75 }, "tool_result_tokens": 38807, "tool_time_ms": 5616, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.147104, "cost_usd": 6.147104, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 720366, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 98, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8105008, "tokens_cache_write": 109284, "tokens_input": 187, "tokens_output": 40033, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 78 }, "tool_result_tokens": 35534, "tool_time_ms": 12524, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.228187000000001, "cost_usd": 7.228187000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 743131, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9816034, "tokens_cache_write": 130399, "tokens_input": 196, "tokens_output": 40608, "tool_calls": 83, "tool_calls_by_class": { "cc_builtin": 83 }, "tool_result_tokens": 47149, "tool_time_ms": 5442, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8481575, "cost_usd": 4.8481575, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 512144, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6409935, "tokens_cache_write": 104510, "tokens_input": 198, "tokens_output": 23884, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 74 }, "tool_result_tokens": 26320, "tool_time_ms": 3212, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.021708000000002, "cost_usd": 5.021708000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 567568, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6781606, "tokens_cache_write": 89895, "tokens_input": 191, "tokens_output": 29240, "tool_calls": 81, "tool_calls_by_class": { "cc_builtin": 81 }, "tool_result_tokens": 29058, "tool_time_ms": 3371, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.6177505, "cost_usd": 4.6177505, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 545795, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6232841, "tokens_cache_write": 81840, "tokens_input": 196, "tokens_output": 27278, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 73 }, "tool_result_tokens": 25919, "tool_time_ms": 2979, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.21719, "cost_usd": 4.21719, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 503721, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5724590, "tokens_cache_write": 73930, "tokens_input": 189, "tokens_output": 24586, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 69 }, "tool_result_tokens": 22660, "tool_time_ms": 2696, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.920723000000001, "cost_usd": 4.920723000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 558548, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6854936, "tokens_cache_write": 86278, "tokens_input": 320, "tokens_output": 25155, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 77 }, "tool_result_tokens": 29198, "tool_time_ms": 3269, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8861205, "cost_usd": 3.8861205, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 494509, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5160471, "tokens_cache_write": 74542, "tokens_input": 283, "tokens_output": 22362, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 24054, "tool_time_ms": 2626, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.789868000000001, "cost_usd": 3.789868000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 450123, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5002886, "tokens_cache_write": 71712, "tokens_input": 166, "tokens_output": 22819, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 22680, "tool_time_ms": 2584, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1471155, "cost_usd": 4.1471155, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 497995, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 96, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5664721, "tokens_cache_write": 66739, "tokens_input": 183, "tokens_output": 25858, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 66 }, "tool_result_tokens": 21363, "tool_time_ms": 2917, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.4698775, "cost_usd": 4.4698775, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 651593, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6125495, "tokens_cache_write": 74345, "tokens_input": 196, "tokens_output": 26508, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 74 }, "tool_result_tokens": 21398, "tool_time_ms": 2831, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.501471999999999, "cost_usd": 4.501471999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 502744, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6224714, "tokens_cache_write": 73203, "tokens_input": 312, "tokens_output": 26221, "tool_calls": 73, "tool_calls_by_class": { "cc_builtin": 73 }, "tool_result_tokens": 24202, "tool_time_ms": 3052, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.092897999999998, "cost_usd": 8.092897999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1051901, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 168, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6267816, "tokens_cache_write": 336047, "tokens_input": 324, "tokens_output": 63876, "tool_calls": 188, "tool_calls_by_class": { "cc_builtin": 185, "skill": 3 }, "tool_result_tokens": 104669, "tool_time_ms": 9297, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.6774365, "cost_usd": 7.6774365, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 991695, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 159, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5679113, "tokens_cache_write": 332981, "tokens_input": 1404, "tokens_output": 60042, "tool_calls": 196, "tool_calls_by_class": { "cc_builtin": 195, "skill": 1 }, "tool_result_tokens": 102657, "tool_time_ms": 7763, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.112419, "cost_usd": 8.112419, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1060427, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 162, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6116388, "tokens_cache_write": 340824, "tokens_input": 312, "tokens_output": 65777, "tool_calls": 201, "tool_calls_by_class": { "cc_builtin": 197, "skill": 4 }, "tool_result_tokens": 95256, "tool_time_ms": 27650, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.408845999999999, "cost_usd": 8.408845999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1032736, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 170, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6155802, "tokens_cache_write": 374348, "tokens_input": 328, "tokens_output": 63433, "tool_calls": 211, "tool_calls_by_class": { "cc_builtin": 210, "skill": 1 }, "tool_result_tokens": 122166, "tool_time_ms": 18586, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.412036500000001, "cost_usd": 7.412036500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1048550, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5652683, "tokens_cache_write": 297223, "tokens_input": 893, "tokens_output": 64360, "tool_calls": 193, "tool_calls_by_class": { "cc_builtin": 192, "skill": 1 }, "tool_result_tokens": 99963, "tool_time_ms": 13941, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.578942499999998, "cost_usd": 8.578942499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1091388, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 163, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6266815, "tokens_cache_write": 372271, "tokens_input": 585, "tokens_output": 68796, "tool_calls": 206, "tool_calls_by_class": { "cc_builtin": 203, "skill": 3 }, "tool_result_tokens": 120771, "tool_time_ms": 10900, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.286758500000001, "cost_usd": 7.286758500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1072210, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 161, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5654397, "tokens_cache_write": 290178, "tokens_input": 311, "tokens_output": 62249, "tool_calls": 182, "tool_calls_by_class": { "cc_builtin": 182 }, "tool_result_tokens": 107743, "tool_time_ms": 15290, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.141333999999999, "cost_usd": 8.141333999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1068468, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 160, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5947828, "tokens_cache_write": 344929, "tokens_input": 306, "tokens_output": 68664, "tool_calls": 205, "tool_calls_by_class": { "cc_builtin": 203, "skill": 2 }, "tool_result_tokens": 115721, "tool_time_ms": 6849, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.269046500000002, "cost_usd": 9.269046500000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1090251, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 163, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5814383, "tokens_cache_write": 467322, "tokens_input": 1617, "tokens_output": 67222, "tool_calls": 196, "tool_calls_by_class": { "cc_builtin": 195, "skill": 1 }, "tool_result_tokens": 110085, "tool_time_ms": 16416, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.059165000000001, "cost_usd": 7.059165000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 977781, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5462920, "tokens_cache_write": 284252, "tokens_input": 302, "tokens_output": 59347, "tool_calls": 180, "tool_calls_by_class": { "cc_builtin": 179, "skill": 1 }, "tool_result_tokens": 94332, "tool_time_ms": 14252, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.3585935000000005, "cost_usd": 6.3585935000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 756842, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 154, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5040057, "tokens_cache_write": 271847, "tokens_input": 1934, "tokens_output": 44417, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 184 }, "tool_result_tokens": 87807, "tool_time_ms": 5835, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.42843, "cost_usd": 6.42843, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 768553, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 161, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5157260, "tokens_cache_write": 267194, "tokens_input": 3092, "tokens_output": 46496, "tool_calls": 197, "tool_calls_by_class": { "cc_builtin": 197 }, "tool_result_tokens": 85253, "tool_time_ms": 6471, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.401044999999998, "cost_usd": 6.401044999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 788690, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5065530, "tokens_cache_write": 269272, "tokens_input": 312, "tokens_output": 46960, "tool_calls": 192, "tool_calls_by_class": { "cc_builtin": 191, "skill": 1 }, "tool_result_tokens": 76598, "tool_time_ms": 5365, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.1507115, "cost_usd": 7.1507115, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 798106, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 169, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5705563, "tokens_cache_write": 314138, "tokens_input": 330, "tokens_output": 46196, "tool_calls": 189, "tool_calls_by_class": { "cc_builtin": 186, "skill": 3 }, "tool_result_tokens": 82114, "tool_time_ms": 6739, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.994601500000001, "cost_usd": 6.994601500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 782238, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 157, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5376683, "tokens_cache_write": 315264, "tokens_input": 309, "tokens_output": 46083, "tool_calls": 177, "tool_calls_by_class": { "cc_builtin": 174, "skill": 3 }, "tool_result_tokens": 83724, "tool_time_ms": 6410, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.263274, "cost_usd": 6.263274, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 757137, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 163, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5226388, "tokens_cache_write": 250188, "tokens_input": 450, "tokens_output": 45838, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 183, "skill": 1 }, "tool_result_tokens": 71192, "tool_time_ms": 5333, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2466395, "cost_usd": 6.2466395, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 749028, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5011469, "tokens_cache_write": 266440, "tokens_input": 341, "tokens_output": 42992, "tool_calls": 176, "tool_calls_by_class": { "cc_builtin": 175, "skill": 1 }, "tool_result_tokens": 76405, "tool_time_ms": 5557, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.849883, "cost_usd": 6.849883, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 772389, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 155, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5149736, "tokens_cache_write": 310814, "tokens_input": 665, "tokens_output": 46542, "tool_calls": 195, "tool_calls_by_class": { "cc_builtin": 194, "skill": 1 }, "tool_result_tokens": 103335, "tool_time_ms": 8772, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.284182500000001, "cost_usd": 7.284182500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 848806, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 171, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5924325, "tokens_cache_write": 311170, "tokens_input": 334, "tokens_output": 48346, "tool_calls": 199, "tool_calls_by_class": { "cc_builtin": 196, "skill": 3 }, "tool_result_tokens": 83661, "tool_time_ms": 7613, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.437879999999999, "cost_usd": 6.437879999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 774195, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 160, "model": "claude-opus-5", "path": "runs/graphify/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5267390, "tokens_cache_write": 264456, "tokens_input": 970, "tokens_output": 46191, "tool_calls": 192, "tool_calls_by_class": { "cc_builtin": 192 }, "tool_result_tokens": 94482, "tool_time_ms": 7186, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.0468632, "cost_usd": 7.0468632, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1133595, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 192, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12540534, "tokens_cache_write": 442680, "tokens_input": 384, "tokens_output": 39136, "tool_calls": 181, "tool_calls_by_class": { "cc_builtin": 155, "skill": 5, "subagent": 21 }, "tool_result_tokens": 101221, "tool_time_ms": 9131, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.590765199999999, "cost_usd": 5.590765199999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 705359, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 137, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13343421, "tokens_cache_write": 161308, "tokens_input": 274, "tokens_output": 47713, "tool_calls": 121, "tool_calls_by_class": { "cc_builtin": 116, "skill": 1, "subagent": 4 }, "tool_result_tokens": 43665, "tool_time_ms": 5838, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.862339749999999, "cost_usd": 5.862339749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 649133, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 214, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10799825, "tokens_cache_write": 324298, "tokens_input": 428, "tokens_output": 46129, "tool_calls": 203, "tool_calls_by_class": { "cc_builtin": 184, "subagent": 19 }, "tool_result_tokens": 104484, "tool_time_ms": 8528, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.30805045, "cost_usd": 5.30805045, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 128564, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 218, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7901314, "tokens_cache_write": 373694, "tokens_input": 436, "tokens_output": 40161, "tool_calls": 211, "tool_calls_by_class": { "cc_builtin": 193, "subagent": 18 }, "tool_result_tokens": 120157, "tool_time_ms": 19753, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.683123749999999, "cost_usd": 4.683123749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 658360, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 132, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10433410, "tokens_cache_write": 179874, "tokens_input": 264, "tokens_output": 33165, "tool_calls": 121, "tool_calls_by_class": { "cc_builtin": 118, "subagent": 3 }, "tool_result_tokens": 51988, "tool_time_ms": 4158, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.392804300000001, "cost_usd": 4.392804300000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 490484, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 115, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10071531, "tokens_cache_write": 158658, "tokens_input": 230, "tokens_output": 33061, "tool_calls": 93, "tool_calls_by_class": { "cc_builtin": 91, "skill": 1, "subagent": 1 }, "tool_result_tokens": 43482, "tool_time_ms": 6110, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5484934, "cost_usd": 5.5484934, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 469085, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 168, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10963488, "tokens_cache_write": 308168, "tokens_input": 336, "tokens_output": 43109, "tool_calls": 145, "tool_calls_by_class": { "cc_builtin": 134, "skill": 3, "subagent": 8 }, "tool_result_tokens": 72944, "tool_time_ms": 7472, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.671868750000002, "cost_usd": 9.671868750000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 515457, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 226, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19837950, "tokens_cache_write": 467530, "tokens_input": 452, "tokens_output": 62312, "tool_calls": 211, "tool_calls_by_class": { "cc_builtin": 203, "subagent": 8 }, "tool_result_tokens": 102129, "tool_time_ms": 6775, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.61961825, "cost_usd": 7.61961825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 431421, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 239, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13674385, "tokens_cache_write": 428504, "tokens_input": 10695, "tokens_output": 64747, "tool_calls": 256, "tool_calls_by_class": { "cc_builtin": 238, "subagent": 18 }, "tool_result_tokens": 147332, "tool_time_ms": 8267, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.35313845, "cost_usd": 4.35313845, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 550642, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 117, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10280579, "tokens_cache_write": 127036, "tokens_input": 234, "tokens_output": 35756, "tool_calls": 100, "tool_calls_by_class": { "cc_builtin": 99, "subagent": 1 }, "tool_result_tokens": 36582, "tool_time_ms": 5751, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8512543000000004, "cost_usd": 2.8512543000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 331637, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6685701, "tokens_cache_write": 98860, "tokens_input": 188, "tokens_output": 16788, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 19386, "tool_time_ms": 2667, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3665905, "cost_usd": 2.3665905, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 345078, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 93, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5975455, "tokens_cache_write": 55886, "tokens_input": 186, "tokens_output": 15872, "tool_calls": 63, "tool_calls_by_class": { "cc_builtin": 63 }, "tool_result_tokens": 15102, "tool_time_ms": 2515, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1991017, "cost_usd": 2.1991017, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 307566, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 90, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5620199, "tokens_cache_write": 52272, "tokens_input": 180, "tokens_output": 13258, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 14853, "tool_time_ms": 2428, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4825987000000005, "cost_usd": 2.4825987000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 313976, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 87, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6227109, "tokens_cache_write": 67494, "tokens_input": 174, "tokens_output": 13932, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 22630, "tool_time_ms": 2325, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.3760983999999996, "cost_usd": 2.3760983999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 320290, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5916138, "tokens_cache_write": 61345, "tokens_input": 174, "tokens_output": 15511, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 18344, "tool_time_ms": 2527, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5440296999999994, "cost_usd": 2.5440296999999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 324341, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6428369, "tokens_cache_write": 64502, "tokens_input": 184, "tokens_output": 15197, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 20447, "tool_time_ms": 2115, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2500171, "cost_usd": 2.2500171, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 329469, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 90, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5651977, "tokens_cache_write": 54724, "tokens_input": 180, "tokens_output": 15036, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 15343, "tool_time_ms": 2138, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1151404, "cost_usd": 2.1151404, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 296051, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 89, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5428198, "tokens_cache_write": 46887, "tokens_input": 178, "tokens_output": 13655, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 16117, "tool_time_ms": 2056, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1732762000000005, "cost_usd": 2.1732762000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 322927, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 91, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5430264, "tokens_cache_write": 51456, "tokens_input": 182, "tokens_output": 15661, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 12610, "tool_time_ms": 2327, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2743015, "cost_usd": 2.2743015, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 335339, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 91, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5704735, "tokens_cache_write": 53690, "tokens_input": 182, "tokens_output": 16013, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 14233, "tool_time_ms": 2457, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.748179749999999, "cost_usd": 6.748179749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 790846, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 206, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9909035, "tokens_cache_write": 515985, "tokens_input": 412, "tokens_output": 55604, "tool_calls": 203, "tool_calls_by_class": { "cc_builtin": 188, "skill": 9, "subagent": 6 }, "tool_result_tokens": 92481, "tool_time_ms": 7512, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.604249549999999, "cost_usd": 7.604249549999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 759290, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 220, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10463381, "tokens_cache_write": 636680, "tokens_input": 440, "tokens_output": 60001, "tool_calls": 223, "tool_calls_by_class": { "cc_builtin": 207, "skill": 9, "subagent": 7 }, "tool_result_tokens": 108026, "tool_time_ms": 8163, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.583288699999998, "cost_usd": 7.583288699999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 852601, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 212, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10994399, "tokens_cache_write": 598512, "tokens_input": 423, "tokens_output": 57157, "tool_calls": 189, "tool_calls_by_class": { "cc_builtin": 173, "skill": 11, "subagent": 5 }, "tool_result_tokens": 98174, "tool_time_ms": 8405, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.5551879500000005, "cost_usd": 6.5551879500000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 716535, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 223, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9651317, "tokens_cache_write": 517107, "tokens_input": 446, "tokens_output": 48040, "tool_calls": 211, "tool_calls_by_class": { "cc_builtin": 196, "skill": 5, "subagent": 10 }, "tool_result_tokens": 111480, "tool_time_ms": 6852, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.814947749999999, "cost_usd": 6.814947749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 805751, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 227, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10752425, "tokens_cache_write": 471554, "tokens_input": 454, "tokens_output": 58498, "tool_calls": 213, "tool_calls_by_class": { "cc_builtin": 202, "skill": 8, "subagent": 3 }, "tool_result_tokens": 98365, "tool_time_ms": 7794, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.0268317499999995, "cost_usd": 7.0268317499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 816695, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 222, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10357490, "tokens_cache_write": 531306, "tokens_input": 444, "tokens_output": 61078, "tool_calls": 211, "tool_calls_by_class": { "cc_builtin": 195, "skill": 8, "subagent": 8 }, "tool_result_tokens": 104412, "tool_time_ms": 8115, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.6273240499999995, "cost_usd": 7.6273240499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 945093, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 231, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11487046, "tokens_cache_write": 575516, "tokens_input": 462, "tokens_output": 58601, "tool_calls": 218, "tool_calls_by_class": { "cc_builtin": 201, "skill": 11, "subagent": 6 }, "tool_result_tokens": 115721, "tool_time_ms": 8322, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.242714899999999, "cost_usd": 7.242714899999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 994124, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 230, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10949598, "tokens_cache_write": 531132, "tokens_input": 460, "tokens_output": 59802, "tool_calls": 236, "tool_calls_by_class": { "cc_builtin": 221, "skill": 8, "subagent": 7 }, "tool_result_tokens": 101290, "tool_time_ms": 7817, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.42305685, "cost_usd": 6.42305685, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 941904, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 213, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10122672, "tokens_cache_write": 456185, "tokens_input": 426, "tokens_output": 54897, "tool_calls": 209, "tool_calls_by_class": { "cc_builtin": 195, "skill": 6, "subagent": 8 }, "tool_result_tokens": 90163, "tool_time_ms": 6438, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.0001573, "cost_usd": 8.0001573, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 855343, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 258, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12131376, "tokens_cache_write": 621283, "tokens_input": 516, "tokens_output": 61916, "tool_calls": 246, "tool_calls_by_class": { "cc_builtin": 227, "skill": 11, "subagent": 8 }, "tool_result_tokens": 124493, "tool_time_ms": 9568, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.051420299999997, "cost_usd": 9.051420299999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 970171, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 261, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12818041, "tokens_cache_write": 743819, "tokens_input": 522, "tokens_output": 54432, "tool_calls": 233, "tool_calls_by_class": { "cc_builtin": 210, "skill": 19, "subagent": 4 }, "tool_result_tokens": 127735, "tool_time_ms": 18065, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.255627800000001, "cost_usd": 7.255627800000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 822543, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 227, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9976136, "tokens_cache_write": 638361, "tokens_input": 454, "tokens_output": 48575, "tool_calls": 220, "tool_calls_by_class": { "cc_builtin": 200, "skill": 11, "subagent": 9 }, "tool_result_tokens": 109259, "tool_time_ms": 12486, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.64941215, "cost_usd": 7.64941215, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 895378, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 227, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11085863, "tokens_cache_write": 621199, "tokens_input": 453, "tokens_output": 50373, "tool_calls": 201, "tool_calls_by_class": { "cc_builtin": 182, "skill": 16, "subagent": 3 }, "tool_result_tokens": 91870, "tool_time_ms": 14706, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.90136675, "cost_usd": 8.90136675, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1143574, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 243, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12760570, "tokens_cache_write": 720332, "tokens_input": 486, "tokens_output": 55705, "tool_calls": 219, "tool_calls_by_class": { "cc_builtin": 197, "skill": 20, "subagent": 2 }, "tool_result_tokens": 119513, "tool_time_ms": 197415, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.969442549999998, "cost_usd": 7.969442549999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 844871, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 244, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11083486, "tokens_cache_write": 679688, "tokens_input": 488, "tokens_output": 49392, "tool_calls": 238, "tool_calls_by_class": { "cc_builtin": 213, "skill": 15, "subagent": 10 }, "tool_result_tokens": 94746, "tool_time_ms": 13634, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.665533600000002, "cost_usd": 8.665533600000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 912977, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 244, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11427607, "tokens_cache_write": 773092, "tokens_input": 488, "tokens_output": 54305, "tool_calls": 237, "tool_calls_by_class": { "cc_builtin": 209, "skill": 20, "subagent": 8 }, "tool_result_tokens": 111256, "tool_time_ms": 16093, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.24064765, "cost_usd": 8.24064765, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 869871, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 236, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11644308, "tokens_cache_write": 690009, "tokens_input": 472, "tokens_output": 52003, "tool_calls": 215, "tool_calls_by_class": { "cc_builtin": 190, "skill": 19, "subagent": 6 }, "tool_result_tokens": 97548, "tool_time_ms": 14770, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.568386200000001, "cost_usd": 7.568386200000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 816619, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 232, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10942504, "tokens_cache_write": 616286, "tokens_input": 464, "tokens_output": 49855, "tool_calls": 204, "tool_calls_by_class": { "cc_builtin": 183, "skill": 14, "subagent": 7 }, "tool_result_tokens": 104330, "tool_time_ms": 12703, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.9599258, "cost_usd": 7.9599258, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 868111, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 244, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11656186, "tokens_cache_write": 645448, "tokens_input": 488, "tokens_output": 50795, "tool_calls": 227, "tool_calls_by_class": { "cc_builtin": 203, "skill": 17, "subagent": 7 }, "tool_result_tokens": 97637, "tool_time_ms": 13380, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "graphify", "arm_label": "CC+Graphify", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 8.419593, "cost_usd": 8.419593, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 890112, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 242, "model": "claude-sonnet-5", "path": "runs/graphify/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|graphify|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12721990, "tokens_cache_write": 663867, "tokens_input": 484, "tokens_output": 50942, "tool_calls": 210, "tool_calls_by_class": { "cc_builtin": 187, "skill": 20, "subagent": 3 }, "tool_result_tokens": 94806, "tool_time_ms": 16125, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.869691, "cost_usd": 4.869691, "denominator": 20, "duration_basis": "run", "duration_ms": 307408, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 42, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2837021, "tokens_cache_write": 54656, "tokens_input": 80, "tokens_output": 18775, "tool_calls": 43, "tool_calls_by_class": { "cc_builtin": 43 }, "tool_result_tokens": 19331, "tool_time_ms": 4555, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.927596749999998, "cost_usd": 6.927596749999998, "denominator": 20, "duration_basis": "run", "duration_ms": 320734, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 95, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3323790, "tokens_cache_write": 339411, "tokens_input": 189, "tokens_output": 19074, "tool_calls": 174, "tool_calls_by_class": { "cc_builtin": 162, "subagent": 12 }, "tool_result_tokens": 126882, "tool_time_ms": 46711, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.912603500000004, "cost_usd": 5.912603500000004, "denominator": 20, "duration_basis": "run", "duration_ms": 317006, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 96, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3903177, "tokens_cache_write": 263770, "tokens_input": 189, "tokens_output": 25643, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 134, "subagent": 6 }, "tool_result_tokens": 75015, "tool_time_ms": 7975, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.847052249999999, "cost_usd": 4.847052249999999, "denominator": 20, "duration_basis": "run", "duration_ms": 309387, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 63, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1912808, "tokens_cache_write": 236429, "tokens_input": 125, "tokens_output": 16141, "tool_calls": 127, "tool_calls_by_class": { "cc_builtin": 117, "subagent": 10 }, "tool_result_tokens": 71911, "tool_time_ms": 49247, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.826309250000001, "cost_usd": 3.826309250000001, "denominator": 20, "duration_basis": "run", "duration_ms": 262257, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 65, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2123135, "tokens_cache_write": 207011, "tokens_input": 9824, "tokens_output": 15431, "tool_calls": 114, "tool_calls_by_class": { "cc_builtin": 110, "subagent": 4 }, "tool_result_tokens": 60761, "tool_time_ms": 108132, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.799376000000002, "cost_usd": 8.799376000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 574965, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 82, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6191556, "tokens_cache_write": 67951, "tokens_input": 1920, "tokens_output": 24592, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 67 }, "tool_result_tokens": 21474, "tool_time_ms": 38080, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.178266499999998, "cost_usd": 9.178266499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 535036, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 86, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6542594, "tokens_cache_write": 77766, "tokens_input": 276, "tokens_output": 25908, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 67 }, "tool_result_tokens": 23198, "tool_time_ms": 7919, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.759970000000003, "cost_usd": 8.759970000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 488185, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 80, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6129770, "tokens_cache_write": 68673, "tokens_input": 264, "tokens_output": 25082, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 23349, "tool_time_ms": 16777, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.42023, "cost_usd": 9.42023, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 522960, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 86, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6553340, "tokens_cache_write": 72502, "tokens_input": 275, "tokens_output": 28282, "tool_calls": 79, "tool_calls_by_class": { "cc_builtin": 79 }, "tool_result_tokens": 21716, "tool_time_ms": 7672, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.323415, "cost_usd": 9.323415, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 503456, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 86, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6654605, "tokens_cache_write": 72004, "tokens_input": 163, "tokens_output": 24542, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 71 }, "tool_result_tokens": 23870, "tool_time_ms": 7672, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.961713, "cost_usd": 6.961713, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 393777, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5092563, "tokens_cache_write": 49081, "tokens_input": 158, "tokens_output": 17719, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 14722, "tool_time_ms": 4167, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.621277, "cost_usd": 6.621277, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 383594, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4810377, "tokens_cache_write": 45986, "tokens_input": 158, "tokens_output": 17792, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 13246, "tool_time_ms": 3376, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.8697870000000005, "cost_usd": 6.8697870000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 437764, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4967847, "tokens_cache_write": 48502, "tokens_input": 160, "tokens_output": 18606, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 12819, "tool_time_ms": 4152, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.388450000000001, "cost_usd": 7.388450000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 402803, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5402640, "tokens_cache_write": 52124, "tokens_input": 168, "tokens_output": 18833, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 15892, "tool_time_ms": 4438, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.308467000000001, "cost_usd": 7.308467000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 454798, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 86, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5288987, "tokens_cache_write": 52042, "tokens_input": 164, "tokens_output": 19540, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 15644, "tool_time_ms": 3740, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.140818000000001, "cost_usd": 11.140818000000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 785256, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 145, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5957442, "tokens_cache_write": 223205, "tokens_input": 535, "tokens_output": 48175, "tool_calls": 138, "tool_calls_by_class": { "cc_builtin": 137, "subagent": 1 }, "tool_result_tokens": 64368, "tool_time_ms": 61665, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.35055425, "cost_usd": 11.35055425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 801179, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 145, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6144067, "tokens_cache_write": 208180, "tokens_input": 398, "tokens_output": 47891, "tool_calls": 137, "tool_calls_by_class": { "cc_builtin": 136, "subagent": 1 }, "tool_result_tokens": 75699, "tool_time_ms": 35396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.2840065, "cost_usd": 10.2840065, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 746552, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 132, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5509703, "tokens_cache_write": 189874, "tokens_input": 372, "tokens_output": 46423, "tool_calls": 123, "tool_calls_by_class": { "cc_builtin": 122, "subagent": 1 }, "tool_result_tokens": 62016, "tool_time_ms": 34399, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.340045750000002, "cost_usd": 10.340045750000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 774770, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 145, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5997974, "tokens_cache_write": 191089, "tokens_input": 506, "tokens_output": 44934, "tool_calls": 139, "tool_calls_by_class": { "cc_builtin": 137, "subagent": 2 }, "tool_result_tokens": 65751, "tool_time_ms": 23701, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.733343, "cost_usd": 8.733343, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 693947, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 119, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5058860, "tokens_cache_write": 132375, "tokens_input": 235, "tokens_output": 36486, "tool_calls": 115, "tool_calls_by_class": { "cc_builtin": 115 }, "tool_result_tokens": 52757, "tool_time_ms": 20299, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.170795, "cost_usd": 2.170795, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 74297, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 21, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 787215, "tokens_cache_write": 61528, "tokens_input": 42, "tokens_output": 3052, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 14 }, "tool_result_tokens": 2293, "tool_time_ms": 885, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6419279999999998, "cost_usd": 1.6419279999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 94602, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 23, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 904108, "tokens_cache_write": 28258, "tokens_input": 46, "tokens_output": 3444, "tool_calls": 16, "tool_calls_by_class": { "cc_builtin": 16 }, "tool_result_tokens": 2712, "tool_time_ms": 1163, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.592935, "cost_usd": 1.592935, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 83389, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 23, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 902975, "tokens_cache_write": 26530, "tokens_input": 46, "tokens_output": 3178, "tool_calls": 16, "tool_calls_by_class": { "cc_builtin": 16 }, "tool_result_tokens": 1868, "tool_time_ms": 1160, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.4598220000000002, "cost_usd": 1.4598220000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 84939, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 23, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 914732, "tokens_cache_write": 18209, "tokens_input": 46, "tokens_output": 3609, "tool_calls": 17, "tool_calls_by_class": { "cc_builtin": 17 }, "tool_result_tokens": 2736, "tool_time_ms": 1138, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.181414, "cost_usd": 1.181414, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 60543, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 20, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 799254, "tokens_cache_write": 11348, "tokens_input": 40, "tokens_output": 3096, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 14 }, "tool_result_tokens": 2281, "tool_time_ms": 877, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.65735, "cost_usd": 9.65735, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 585387, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 125, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5095470, "tokens_cache_write": 152038, "tokens_input": 362, "tokens_output": 30350, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 106 }, "tool_result_tokens": 39139, "tool_time_ms": 6562, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.460626, "cost_usd": 9.460626, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 596431, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 125, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5084156, "tokens_cache_write": 140820, "tokens_input": 362, "tokens_output": 31129, "tool_calls": 101, "tool_calls_by_class": { "cc_builtin": 100, "vendor_mcp": 1 }, "tool_result_tokens": 36831, "tool_time_ms": 6964, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.589604999999999, "cost_usd": 9.589604999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 626184, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 125, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5130845, "tokens_cache_write": 142542, "tokens_input": 362, "tokens_output": 32086, "tool_calls": 104, "tool_calls_by_class": { "cc_builtin": 104 }, "tool_result_tokens": 40879, "tool_time_ms": 6665, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.760453999999998, "cost_usd": 9.760453999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 676090, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 136, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5589154, "tokens_cache_write": 123395, "tokens_input": 380, "tokens_output": 33992, "tool_calls": 112, "tool_calls_by_class": { "cc_builtin": 112 }, "tool_result_tokens": 36498, "tool_time_ms": 7251, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.719100999999998, "cost_usd": 9.719100999999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 654716, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 137, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5677271, "tokens_cache_write": 117180, "tokens_input": 273, "tokens_output": 33910, "tool_calls": 116, "tool_calls_by_class": { "cc_builtin": 116 }, "tool_result_tokens": 41576, "tool_time_ms": 7618, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.977775000000001, "cost_usd": 11.977775000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 558612, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 127, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|11", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5104905, "tokens_cache_write": 263044, "tokens_input": 254, "tokens_output": 32189, "tool_calls": 105, "tool_calls_by_class": { "cc_builtin": 105 }, "tool_result_tokens": 47138, "tool_time_ms": 8286, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.865787000000001, "cost_usd": 10.865787000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 610213, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 136, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|12", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5574027, "tokens_cache_write": 182612, "tokens_input": 272, "tokens_output": 32736, "tool_calls": 111, "tool_calls_by_class": { "cc_builtin": 111 }, "tool_result_tokens": 44295, "tool_time_ms": 8036, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.467870999999999, "cost_usd": 10.467870999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 599403, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 129, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|13", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5262691, "tokens_cache_write": 178114, "tokens_input": 370, "tokens_output": 32784, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 110 }, "tool_result_tokens": 42014, "tool_time_ms": 7908, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 11.411692749999998, "cost_usd": 11.411692749999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 681396, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 141, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|14", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5686107, "tokens_cache_write": 218916, "tokens_input": 280, "tokens_output": 37362, "tool_calls": 127, "tool_calls_by_class": { "cc_builtin": 126, "subagent": 1 }, "tool_result_tokens": 47867, "tool_time_ms": 9553, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.548859000000002, "cost_usd": 10.548859000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 589233, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 132, "model": "claude-fable-5", "path": "runs/serena/claude-fable-5/ZeroShotExam/memos/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-fable-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|15", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5374239, "tokens_cache_write": 177942, "tokens_input": 383, "tokens_output": 32239, "tool_calls": 110, "tool_calls_by_class": { "cc_builtin": 110 }, "tool_result_tokens": 43138, "tool_time_ms": 10551, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.510773749999999, "cost_usd": 5.510773749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 915775, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 292, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 26619190, "tokens_cache_write": 1344498, "tokens_input": 1788, "tokens_output": 35694, "tool_calls": 291, "tool_calls_by_class": { "cc_builtin": 288, "harness_builtin": 1, "subagent": 2 }, "tool_result_tokens": 84296, "tool_time_ms": 49396, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.980802, "cost_usd": 2.980802, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1681376, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 422, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16539655, "tokens_cache_write": 623723, "tokens_input": 3447, "tokens_output": 31068, "tool_calls": 522, "tool_calls_by_class": { "cc_builtin": 472, "subagent": 20, "vendor_mcp": 30 }, "tool_result_tokens": 298825, "tool_time_ms": 30548, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2982829, "cost_usd": 5.2982829, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 964515, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 240, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 25820389, "tokens_cache_write": 1268534, "tokens_input": 1436, "tokens_output": 35548, "tool_calls": 220, "tool_calls_by_class": { "cc_builtin": 219, "vendor_mcp": 1 }, "tool_result_tokens": 68444, "tool_time_ms": 13307, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8255571, "cost_usd": 5.8255571, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 822790, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 340, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 28692091, "tokens_cache_write": 1377008, "tokens_input": 2006, "tokens_output": 38273, "tool_calls": 318, "tool_calls_by_class": { "cc_builtin": 316, "subagent": 2 }, "tool_result_tokens": 90977, "tool_time_ms": 30786, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.571077850000001, "cost_usd": 5.571077850000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 898499, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 239, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 26260951, "tokens_cache_write": 1397677, "tokens_input": 1424, "tokens_output": 34983, "tool_calls": 217, "tool_calls_by_class": { "cc_builtin": 215, "subagent": 2 }, "tool_result_tokens": 95476, "tool_time_ms": 9868, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.55498225, "cost_usd": 6.55498225, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 940030, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 282, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 30973685, "tokens_cache_write": 1650292, "tokens_input": 1694, "tokens_output": 35815, "tool_calls": 260, "tool_calls_by_class": { "cc_builtin": 258, "subagent": 2 }, "tool_result_tokens": 107769, "tool_time_ms": 15337, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.828526800000001, "cost_usd": 5.828526800000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1152970, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 288, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 30413348, "tokens_cache_write": 1300865, "tokens_input": 1707, "tokens_output": 36751, "tool_calls": 268, "tool_calls_by_class": { "cc_builtin": 268 }, "tool_result_tokens": 61311, "tool_time_ms": 28755, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7064090000000003, "cost_usd": 2.7064090000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1554815, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 371, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13849005, "tokens_cache_write": 659894, "tokens_input": 2479, "tokens_output": 29754, "tool_calls": 551, "tool_calls_by_class": { "cc_builtin": 518, "subagent": 20, "vendor_mcp": 13 }, "tool_result_tokens": 303781, "tool_time_ms": 29451, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2447435, "cost_usd": 6.2447435, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 967133, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 291, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 29238595, "tokens_cache_write": 1581170, "tokens_input": 1773, "tokens_output": 38190, "tool_calls": 286, "tool_calls_by_class": { "cc_builtin": 284, "subagent": 2 }, "tool_result_tokens": 126260, "tool_time_ms": 29415, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4400369000000004, "cost_usd": 2.4400369000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 513286, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 155, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13316394, "tokens_cache_write": 514202, "tokens_input": 1025, "tokens_output": 19957, "tool_calls": 125, "tool_calls_by_class": { "cc_builtin": 123, "subagent": 2 }, "tool_result_tokens": 73741, "tool_time_ms": 4736, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3462712999999997, "cost_usd": 3.3462712999999997, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 652307, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 179, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16687263, "tokens_cache_write": 778629, "tokens_input": 1157, "tokens_output": 23826, "tool_calls": 149, "tool_calls_by_class": { "cc_builtin": 149 }, "tool_result_tokens": 65104, "tool_time_ms": 4644, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.7285928, "cost_usd": 2.7285928, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 613141, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 173, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15780848, "tokens_cache_write": 514510, "tokens_input": 1123, "tokens_output": 24073, "tool_calls": 143, "tool_calls_by_class": { "cc_builtin": 143 }, "tool_result_tokens": 57678, "tool_time_ms": 5180, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2071222, "cost_usd": 3.2071222, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 692573, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 186, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16965232, "tokens_cache_write": 687275, "tokens_input": 1194, "tokens_output": 26971, "tool_calls": 156, "tool_calls_by_class": { "cc_builtin": 156 }, "tool_result_tokens": 56881, "tool_time_ms": 4591, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.5147109000000007, "cost_usd": 3.5147109000000007, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 675532, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 192, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16181829, "tokens_cache_write": 882681, "tokens_input": 1216, "tokens_output": 25990, "tool_calls": 162, "tool_calls_by_class": { "cc_builtin": 161, "vendor_mcp": 1 }, "tool_result_tokens": 48970, "tool_time_ms": 5387, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0447908000000004, "cost_usd": 3.0447908000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 583838, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 174, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15911208, "tokens_cache_write": 674024, "tokens_input": 1117, "tokens_output": 20901, "tool_calls": 144, "tool_calls_by_class": { "cc_builtin": 144 }, "tool_result_tokens": 59770, "tool_time_ms": 5080, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9858091000000004, "cost_usd": 2.9858091000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 622977, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 184, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 23, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15733666, "tokens_cache_write": 662210, "tokens_input": 1195, "tokens_output": 27157, "tool_calls": 178, "tool_calls_by_class": { "cc_builtin": 171, "subagent": 7 }, "tool_result_tokens": 115433, "tool_time_ms": 5376, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8934263000000002, "cost_usd": 2.8934263000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1418756, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 331, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13311858, "tokens_cache_write": 839138, "tokens_input": 10142, "tokens_output": 40777, "tool_calls": 471, "tool_calls_by_class": { "cc_builtin": 443, "subagent": 28 }, "tool_result_tokens": 445423, "tool_time_ms": 15448, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.963273999999999, "cost_usd": 2.963273999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 695254, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 188, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16712760, "tokens_cache_write": 585456, "tokens_input": 1196, "tokens_output": 23978, "tool_calls": 158, "tool_calls_by_class": { "cc_builtin": 158 }, "tool_result_tokens": 54440, "tool_time_ms": 4749, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4434594999999995, "cost_usd": 3.4434594999999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1185147, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 424, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 18902205, "tokens_cache_write": 708466, "tokens_input": 2582, "tokens_output": 50679, "tool_calls": 483, "tool_calls_by_class": { "cc_builtin": 455, "subagent": 9, "vendor_mcp": 19 }, "tool_result_tokens": 298854, "tool_time_ms": 29347, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.40483435, "cost_usd": 4.40483435, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1141857, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 547, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 22665736, "tokens_cache_write": 1007692, "tokens_input": 3348, "tokens_output": 61216, "tool_calls": 591, "tool_calls_by_class": { "cc_builtin": 561, "harness_builtin": 1, "subagent": 14, "vendor_mcp": 15 }, "tool_result_tokens": 373689, "tool_time_ms": 152759, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9849813000000003, "cost_usd": 3.9849813000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 905091, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 10, "family": "cc_competitor", "llm_calls": 494, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 20798293, "tokens_cache_write": 896994, "tokens_input": 3021, "tokens_output": 55138, "tool_calls": 558, "tool_calls_by_class": { "cc_builtin": 530, "subagent": 9, "vendor_mcp": 19 }, "tool_result_tokens": 338690, "tool_time_ms": 36668, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.2371684, "cost_usd": 4.2371684, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 837246, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 507, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 20534074, "tokens_cache_write": 1023953, "tokens_input": 3126, "tokens_output": 59437, "tool_calls": 562, "tool_calls_by_class": { "cc_builtin": 531, "harness_builtin": 1, "subagent": 11, "vendor_mcp": 19 }, "tool_result_tokens": 384992, "tool_time_ms": 148592, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.307951999999999, "cost_usd": 4.307951999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 657728, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "cc_competitor", "llm_calls": 533, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 20618930, "tokens_cache_write": 1075351, "tokens_input": 3333, "tokens_output": 63051, "tool_calls": 646, "tool_calls_by_class": { "cc_builtin": 619, "subagent": 13, "vendor_mcp": 14 }, "tool_result_tokens": 373140, "tool_time_ms": 28490, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8065632500000004, "cost_usd": 3.8065632500000004, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 984236, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 461, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 19714690, "tokens_cache_write": 875699, "tokens_input": 2879, "tokens_output": 54616, "tool_calls": 562, "tool_calls_by_class": { "cc_builtin": 532, "harness_builtin": 1, "subagent": 11, "vendor_mcp": 18 }, "tool_result_tokens": 349877, "tool_time_ms": 48557, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8493209999999998, "cost_usd": 3.8493209999999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 765396, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 448, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 18851385, "tokens_cache_write": 899223, "tokens_input": 2780, "tokens_output": 59334, "tool_calls": 591, "tool_calls_by_class": { "cc_builtin": 552, "harness_builtin": 1, "subagent": 12, "vendor_mcp": 26 }, "tool_result_tokens": 336211, "tool_time_ms": 72466, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8805844000000005, "cost_usd": 3.8805844000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1130436, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 10, "family": "cc_competitor", "llm_calls": 507, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 10, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 20933299, "tokens_cache_write": 851357, "tokens_input": 3053, "tokens_output": 57580, "tool_calls": 540, "tool_calls_by_class": { "cc_builtin": 513, "subagent": 12, "vendor_mcp": 15 }, "tool_result_tokens": 336151, "tool_time_ms": 92690, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.2586764000000015, "cost_usd": 4.2586764000000015, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 763233, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 502, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 21439654, "tokens_cache_write": 1004287, "tokens_input": 3079, "tokens_output": 62441, "tool_calls": 570, "tool_calls_by_class": { "cc_builtin": 546, "subagent": 11, "vendor_mcp": 13 }, "tool_result_tokens": 390519, "tool_time_ms": 32810, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6324647999999997, "cost_usd": 3.6324647999999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 784124, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 9, "family": "cc_competitor", "llm_calls": 444, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 11, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 17826818, "tokens_cache_write": 880316, "tokens_input": 2737, "tokens_output": 53854, "tool_calls": 518, "tool_calls_by_class": { "cc_builtin": 495, "subagent": 15, "vendor_mcp": 8 }, "tool_result_tokens": 318363, "tool_time_ms": 21347, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.08013145, "cost_usd": 3.08013145, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 742488, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 348, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13865782, "tokens_cache_write": 824938, "tokens_input": 2225, "tokens_output": 44621, "tool_calls": 371, "tool_calls_by_class": { "cc_builtin": 331, "subagent": 13, "vendor_mcp": 27 }, "tool_result_tokens": 307806, "tool_time_ms": 12167, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.8782002, "cost_usd": 3.8782002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 712974, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 447, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16517587, "tokens_cache_write": 1097810, "tokens_input": 2867, "tokens_output": 57689, "tool_calls": 508, "tool_calls_by_class": { "cc_builtin": 472, "subagent": 19, "vendor_mcp": 17 }, "tool_result_tokens": 411351, "tool_time_ms": 15629, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4658858499999994, "cost_usd": 3.4658858499999994, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 561245, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 364, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14147461, "tokens_cache_write": 999459, "tokens_input": 2366, "tokens_output": 51831, "tool_calls": 426, "tool_calls_by_class": { "cc_builtin": 394, "subagent": 13, "vendor_mcp": 19 }, "tool_result_tokens": 351363, "tool_time_ms": 12875, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.360429, "cost_usd": 3.360429, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 608442, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 366, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13463610, "tokens_cache_write": 1054358, "tokens_input": 2368, "tokens_output": 47254, "tool_calls": 397, "tool_calls_by_class": { "cc_builtin": 358, "subagent": 18, "vendor_mcp": 21 }, "tool_result_tokens": 360883, "tool_time_ms": 11995, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.2896727500000003, "cost_usd": 3.2896727500000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 688940, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 336, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13238865, "tokens_cache_write": 985159, "tokens_input": 2202, "tokens_output": 46690, "tool_calls": 401, "tool_calls_by_class": { "cc_builtin": 367, "subagent": 18, "vendor_mcp": 16 }, "tool_result_tokens": 363032, "tool_time_ms": 12495, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.470364500000001, "cost_usd": 3.470364500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 565816, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 415, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15750955, "tokens_cache_write": 927681, "tokens_input": 2620, "tokens_output": 45279, "tool_calls": 432, "tool_calls_by_class": { "cc_builtin": 402, "subagent": 16, "vendor_mcp": 14 }, "tool_result_tokens": 390997, "tool_time_ms": 13082, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.275681450000001, "cost_usd": 3.275681450000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 629601, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 378, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 23, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 15028297, "tokens_cache_write": 838482, "tokens_input": 2414, "tokens_output": 50801, "tool_calls": 391, "tool_calls_by_class": { "cc_builtin": 361, "subagent": 14, "vendor_mcp": 16 }, "tool_result_tokens": 312574, "tool_time_ms": 12598, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.18874835, "cost_usd": 3.18874835, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 797952, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 396, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16017821, "tokens_cache_write": 756475, "tokens_input": 2539, "tokens_output": 49020, "tool_calls": 451, "tool_calls_by_class": { "cc_builtin": 425, "subagent": 11, "vendor_mcp": 15 }, "tool_result_tokens": 344783, "tool_time_ms": 13710, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.87118765, "cost_usd": 2.87118765, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 675588, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 355, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13210244, "tokens_cache_write": 718107, "tokens_input": 2264, "tokens_output": 45496, "tool_calls": 367, "tool_calls_by_class": { "cc_builtin": 334, "subagent": 18, "vendor_mcp": 15 }, "tool_result_tokens": 315463, "tool_time_ms": 11972, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0561883000000005, "cost_usd": 3.0561883000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 652621, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 370, "model": "claude-haiku-4-5", "path": "runs/serena/claude-haiku-4-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-haiku-4-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14336993, "tokens_cache_write": 793741, "tokens_input": 2414, "tokens_output": 44924, "tool_calls": 430, "tool_calls_by_class": { "cc_builtin": 398, "subagent": 15, "vendor_mcp": 17 }, "tool_result_tokens": 310370, "tool_time_ms": 14027, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.322088249999999, "cost_usd": 6.322088249999999, "denominator": 20, "duration_basis": "run", "duration_ms": 465258, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 142, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5957879, "tokens_cache_write": 276940, "tokens_input": 170, "tokens_output": 25973, "tool_calls": 264, "tool_calls_by_class": { "cc_builtin": 250, "subagent": 11, "vendor_mcp": 3 }, "tool_result_tokens": 116399, "tool_time_ms": 162987, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.440920249999999, "cost_usd": 6.440920249999999, "denominator": 20, "duration_basis": "run", "duration_ms": 399898, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 127, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5886778, "tokens_cache_write": 303659, "tokens_input": 1541, "tokens_output": 29340, "tool_calls": 235, "tool_calls_by_class": { "cc_builtin": 222, "subagent": 12, "vendor_mcp": 1 }, "tool_result_tokens": 114030, "tool_time_ms": 22086, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8279535000000005, "cost_usd": 4.8279535000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 304558, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4254072, "tokens_cache_write": 256601, "tokens_input": 611, "tokens_output": 19951, "tool_calls": 209, "tool_calls_by_class": { "cc_builtin": 197, "subagent": 9, "vendor_mcp": 3 }, "tool_result_tokens": 120392, "tool_time_ms": 12376, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.384097, "cost_usd": 2.384097, "denominator": 20, "duration_basis": "run", "duration_ms": 347470, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 20, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1933144, "tokens_cache_write": 94669, "tokens_input": 22, "tokens_output": 18829, "tool_calls": 95, "tool_calls_by_class": { "cc_builtin": 94, "vendor_mcp": 1 }, "tool_result_tokens": 61566, "tool_time_ms": 11741, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6018395, "cost_usd": 3.6018395, "denominator": 20, "duration_basis": "run", "duration_ms": 563527, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 40, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3987539, "tokens_cache_write": 99786, "tokens_input": 42, "tokens_output": 24400, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 102, "vendor_mcp": 1 }, "tool_result_tokens": 52494, "tool_time_ms": 45492, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1832659999999997, "cost_usd": 2.1832659999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 374778, "exam": "HumanExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 23, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1934432, "tokens_cache_write": 75870, "tokens_input": 25, "tokens_output": 18289, "tool_calls": 94, "tool_calls_by_class": { "cc_builtin": 93, "vendor_mcp": 1 }, "tool_result_tokens": 43901, "tool_time_ms": 13121, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4519819999999997, "cost_usd": 2.4519819999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 401893, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 19, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1879854, "tokens_cache_write": 100860, "tokens_input": 21, "tokens_output": 20134, "tool_calls": 85, "tool_calls_by_class": { "cc_builtin": 84, "vendor_mcp": 1 }, "tool_result_tokens": 57474, "tool_time_ms": 24792, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.0768725, "cost_usd": 2.0768725, "denominator": 20, "duration_basis": "run", "duration_ms": 309636, "exam": "HumanExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 19, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1604315, "tokens_cache_write": 86406, "tokens_input": 21, "tokens_output": 16422, "tool_calls": 105, "tool_calls_by_class": { "cc_builtin": 104, "vendor_mcp": 1 }, "tool_result_tokens": 47842, "tool_time_ms": 15569, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4666815, "cost_usd": 2.4666815, "denominator": 20, "duration_basis": "run", "duration_ms": 325148, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 22, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2132143, "tokens_cache_write": 98209, "tokens_input": 24, "tokens_output": 16736, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 105, "vendor_mcp": 1 }, "tool_result_tokens": 55275, "tool_time_ms": 8689, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.733415999999999, "cost_usd": 4.733415999999999, "denominator": 20, "duration_basis": "run", "duration_ms": 427882, "exam": "HumanExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4291576, "tokens_cache_write": 229782, "tokens_input": 4567, "tokens_output": 19931, "tool_calls": 208, "tool_calls_by_class": { "cc_builtin": 199, "subagent": 8, "vendor_mcp": 1 }, "tool_result_tokens": 126576, "tool_time_ms": 16154, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1917714999999998, "cost_usd": 2.1917714999999998, "denominator": 30, "duration_basis": "run", "duration_ms": 306183, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 21, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1480063, "tokens_cache_write": 102910, "tokens_input": 23, "tokens_output": 16901, "tool_calls": 145, "tool_calls_by_class": { "cc_builtin": 144, "vendor_mcp": 1 }, "tool_result_tokens": 32803, "tool_time_ms": 4262, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.703528, "cost_usd": 5.703528, "denominator": 30, "duration_basis": "run", "duration_ms": 414501, "exam": "HumanExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 135, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4725081, "tokens_cache_write": 301930, "tokens_input": 171, "tokens_output": 26894, "tool_calls": 307, "tool_calls_by_class": { "cc_builtin": 294, "subagent": 11, "vendor_mcp": 2 }, "tool_result_tokens": 146075, "tool_time_ms": 7880, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.558824, "cost_usd": 4.558824, "denominator": 30, "duration_basis": "run", "duration_ms": 365989, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 67, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4060933, "tokens_cache_write": 217754, "tokens_input": 75, "tokens_output": 25144, "tool_calls": 239, "tool_calls_by_class": { "cc_builtin": 233, "subagent": 3, "vendor_mcp": 3 }, "tool_result_tokens": 95563, "tool_time_ms": 6198, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2281950000000004, "cost_usd": 2.2281950000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 281347, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 23, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1868350, "tokens_cache_write": 91886, "tokens_input": 112, "tokens_output": 14984, "tool_calls": 133, "tool_calls_by_class": { "cc_builtin": 132, "vendor_mcp": 1 }, "tool_result_tokens": 46426, "tool_time_ms": 3209, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.689709, "cost_usd": 1.689709, "denominator": 30, "duration_basis": "run", "duration_ms": 277305, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 22, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1475938, "tokens_cache_write": 60507, "tokens_input": 24, "tokens_output": 13862, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 108, "vendor_mcp": 1 }, "tool_result_tokens": 28135, "tool_time_ms": 3312, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.606616, "cost_usd": 4.606616, "denominator": 30, "duration_basis": "run", "duration_ms": 418719, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 120, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4635472, "tokens_cache_write": 202022, "tokens_input": 144, "tokens_output": 22574, "tool_calls": 201, "tool_calls_by_class": { "cc_builtin": 191, "subagent": 6, "vendor_mcp": 4 }, "tool_result_tokens": 92389, "tool_time_ms": 5734, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.8599759999999999, "cost_usd": 1.8599759999999999, "denominator": 30, "duration_basis": "run", "duration_ms": 248396, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 22, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1620662, "tokens_cache_write": 73184, "tokens_input": 111, "tokens_output": 12690, "tool_calls": 113, "tool_calls_by_class": { "cc_builtin": 112, "vendor_mcp": 1 }, "tool_result_tokens": 39042, "tool_time_ms": 3116, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.144388000000004, "cost_usd": 7.144388000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 586687, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 150, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8916406, "tokens_cache_write": 220285, "tokens_input": 160, "tokens_output": 29782, "tool_calls": 295, "tool_calls_by_class": { "cc_builtin": 289, "subagent": 3, "vendor_mcp": 3 }, "tool_result_tokens": 99289, "tool_time_ms": 8638, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9959135000000001, "cost_usd": 1.9959135000000001, "denominator": 30, "duration_basis": "run", "duration_ms": 265272, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 17, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1211987, "tokens_cache_write": 100800, "tokens_input": 19, "tokens_output": 15273, "tool_calls": 129, "tool_calls_by_class": { "cc_builtin": 128, "vendor_mcp": 1 }, "tool_result_tokens": 33221, "tool_time_ms": 4075, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3622619999999994, "cost_usd": 3.3622619999999994, "denominator": 30, "duration_basis": "run", "duration_ms": 309204, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 60, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2208889, "tokens_cache_write": 204441, "tokens_input": 86, "tokens_output": 19078, "tool_calls": 196, "tool_calls_by_class": { "cc_builtin": 186, "subagent": 6, "vendor_mcp": 4 }, "tool_result_tokens": 97624, "tool_time_ms": 5706, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.447146250000001, "cost_usd": 8.447146250000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1122806, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 149, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11625665, "tokens_cache_write": 213694, "tokens_input": 282, "tokens_output": 26671, "tool_calls": 174, "tool_calls_by_class": { "cc_builtin": 171, "subagent": 3 }, "tool_result_tokens": 86908, "tool_time_ms": 30733, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.1369750000000005, "cost_usd": 7.1369750000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 961760, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 130, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10242105, "tokens_cache_write": 159541, "tokens_input": 176, "tokens_output": 23613, "tool_calls": 140, "tool_calls_by_class": { "cc_builtin": 137, "subagent": 3 }, "tool_result_tokens": 76167, "tool_time_ms": 26545, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.4858730000000016, "cost_usd": 6.4858730000000016, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1146189, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 138, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9179611, "tokens_cache_write": 145810, "tokens_input": 1394, "tokens_output": 24254, "tool_calls": 160, "tool_calls_by_class": { "cc_builtin": 157, "subagent": 3 }, "tool_result_tokens": 63478, "tool_time_ms": 24501, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.923341749999999, "cost_usd": 8.923341749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1158071, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 151, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12794911, "tokens_cache_write": 203949, "tokens_input": 199, "tokens_output": 29392, "tool_calls": 179, "tool_calls_by_class": { "cc_builtin": 176, "subagent": 3 }, "tool_result_tokens": 104910, "tool_time_ms": 18470, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.60464975, "cost_usd": 8.60464975, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1638228, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10702077, "tokens_cache_write": 264042, "tokens_input": 10484, "tokens_output": 35959, "tool_calls": 240, "tool_calls_by_class": { "cc_builtin": 235, "subagent": 5 }, "tool_result_tokens": 123812, "tool_time_ms": 26929, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.003214749999998, "cost_usd": 9.003214749999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2052043, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 163, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9068962, "tokens_cache_write": 386074, "tokens_input": 1910, "tokens_output": 25318, "tool_calls": 399, "tool_calls_by_class": { "cc_builtin": 392, "subagent": 7 }, "tool_result_tokens": 140342, "tool_time_ms": 28847, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.823505500000001, "cost_usd": 8.823505500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2083506, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 170, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9686411, "tokens_cache_write": 352786, "tokens_input": 10073, "tokens_output": 37071, "tool_calls": 311, "tool_calls_by_class": { "cc_builtin": 303, "subagent": 8 }, "tool_result_tokens": 185640, "tool_time_ms": 33687, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.7516215000000015, "cost_usd": 7.7516215000000015, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1506625, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 171, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11051588, "tokens_cache_write": 164049, "tokens_input": 221, "tokens_output": 31440, "tool_calls": 195, "tool_calls_by_class": { "cc_builtin": 190, "subagent": 5 }, "tool_result_tokens": 68373, "tool_time_ms": 30549, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.811766499999999, "cost_usd": 6.811766499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1378994, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 142, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8963923, "tokens_cache_write": 188782, "tokens_input": 1470, "tokens_output": 22974, "tool_calls": 183, "tool_calls_by_class": { "cc_builtin": 179, "subagent": 4 }, "tool_result_tokens": 93939, "tool_time_ms": 17297, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.763731750000001, "cost_usd": 7.763731750000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1529453, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 148, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8494626, "tokens_cache_write": 288088, "tokens_input": 533, "tokens_output": 24454, "tool_calls": 317, "tool_calls_by_class": { "cc_builtin": 310, "subagent": 7 }, "tool_result_tokens": 136541, "tool_time_ms": 18377, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.1027325, "cost_usd": 4.1027325, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 729424, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 108, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5987255, "tokens_cache_write": 77569, "tokens_input": 168, "tokens_output": 13303, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 89 }, "tool_result_tokens": 15719, "tool_time_ms": 3304, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.406793, "cost_usd": 3.406793, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 532873, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 98, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5374836, "tokens_cache_write": 43046, "tokens_input": 158, "tokens_output": 11525, "tool_calls": 93, "tool_calls_by_class": { "cc_builtin": 93 }, "tool_result_tokens": 14876, "tool_time_ms": 3742, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.959921499999999, "cost_usd": 3.959921499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 710737, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 115, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6372533, "tokens_cache_write": 45053, "tokens_input": 175, "tokens_output": 12890, "tool_calls": 87, "tool_calls_by_class": { "cc_builtin": 87 }, "tool_result_tokens": 16171, "tool_time_ms": 3113, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9077149999999996, "cost_usd": 3.9077149999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 527389, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 110, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6249350, "tokens_cache_write": 48029, "tokens_input": 170, "tokens_output": 12076, "tool_calls": 80, "tool_calls_by_class": { "cc_builtin": 80 }, "tool_result_tokens": 19551, "tool_time_ms": 2844, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7181865000000003, "cost_usd": 3.7181865000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 539427, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 112, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6046143, "tokens_cache_write": 42553, "tokens_input": 172, "tokens_output": 10749, "tool_calls": 87, "tool_calls_by_class": { "cc_builtin": 87 }, "tool_result_tokens": 15775, "tool_time_ms": 3176, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6191050000000002, "cost_usd": 3.6191050000000002, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 528220, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5778220, "tokens_cache_write": 44040, "tokens_input": 164, "tokens_output": 11551, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 86 }, "tool_result_tokens": 16001, "tool_time_ms": 3245, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6024425000000004, "cost_usd": 3.6024425000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 553644, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 103, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5731275, "tokens_cache_write": 45589, "tokens_input": 163, "tokens_output": 11204, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 88 }, "tool_result_tokens": 16956, "tool_time_ms": 3692, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.564380000000001, "cost_usd": 4.564380000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 669498, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 128, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7440860, "tokens_cache_write": 48731, "tokens_input": 188, "tokens_output": 14228, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 98 }, "tool_result_tokens": 17522, "tool_time_ms": 3521, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.9535015000000007, "cost_usd": 3.9535015000000007, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 625252, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 115, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6478633, "tokens_cache_write": 39021, "tokens_input": 175, "tokens_output": 12924, "tool_calls": 87, "tool_calls_by_class": { "cc_builtin": 87 }, "tool_result_tokens": 16938, "tool_time_ms": 3412, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.7387059999999996, "cost_usd": 3.7387059999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 588956, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 109, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6050672, "tokens_cache_write": 39907, "tokens_input": 625, "tokens_output": 12447, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 88 }, "tool_result_tokens": 17157, "tool_time_ms": 3643, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.9171615, "cost_usd": 13.9171615, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2008405, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 248, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10344058, "tokens_cache_write": 902082, "tokens_input": 1747, "tokens_output": 52724, "tool_calls": 500, "tool_calls_by_class": { "cc_builtin": 484, "subagent": 12, "vendor_mcp": 4 }, "tool_result_tokens": 358078, "tool_time_ms": 41404, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.446410750000002, "cost_usd": 11.446410750000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1865801, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 194, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7423944, "tokens_cache_write": 804622, "tokens_input": 1937, "tokens_output": 44619, "tool_calls": 445, "tool_calls_by_class": { "cc_builtin": 427, "subagent": 12, "vendor_mcp": 6 }, "tool_result_tokens": 306011, "tool_time_ms": 18151, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.13288275, "cost_usd": 12.13288275, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1977424, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 216, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8429618, "tokens_cache_write": 814779, "tokens_input": 371, "tokens_output": 46820, "tool_calls": 478, "tool_calls_by_class": { "cc_builtin": 460, "subagent": 13, "vendor_mcp": 5 }, "tool_result_tokens": 317642, "tool_time_ms": 20019, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.000827499999998, "cost_usd": 12.000827499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2231273, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 246, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9810070, "tokens_cache_write": 700697, "tokens_input": 4079, "tokens_output": 52924, "tool_calls": 474, "tool_calls_by_class": { "cc_builtin": 458, "subagent": 11, "vendor_mcp": 5 }, "tool_result_tokens": 262609, "tool_time_ms": 55934, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.814115500000003, "cost_usd": 12.814115500000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2241179, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 249, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9433796, "tokens_cache_write": 809637, "tokens_input": 321, "tokens_output": 59145, "tool_calls": 514, "tool_calls_by_class": { "cc_builtin": 496, "subagent": 14, "vendor_mcp": 4 }, "tool_result_tokens": 295056, "tool_time_ms": 39765, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.997338749999999, "cost_usd": 10.997338749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1918685, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 210, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8404550, "tokens_cache_write": 700341, "tokens_input": 7129, "tokens_output": 36865, "tool_calls": 420, "tool_calls_by_class": { "cc_builtin": 402, "subagent": 11, "vendor_mcp": 7 }, "tool_result_tokens": 277206, "tool_time_ms": 49738, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 13.03717475, "cost_usd": 13.03717475, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2250304, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 247, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10065882, "tokens_cache_write": 798625, "tokens_input": 3095, "tokens_output": 59412, "tool_calls": 536, "tool_calls_by_class": { "cc_builtin": 522, "subagent": 12, "vendor_mcp": 2 }, "tool_result_tokens": 308203, "tool_time_ms": 71581, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.7412025, "cost_usd": 10.7412025, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1901432, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 210, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8138935, "tokens_cache_write": 660770, "tokens_input": 997, "tokens_output": 40100, "tool_calls": 426, "tool_calls_by_class": { "cc_builtin": 408, "subagent": 11, "vendor_mcp": 7 }, "tool_result_tokens": 246969, "tool_time_ms": 25009, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.888297250000003, "cost_usd": 11.888297250000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 2144005, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 227, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8803177, "tokens_cache_write": 759033, "tokens_input": 2617, "tokens_output": 49941, "tool_calls": 481, "tool_calls_by_class": { "cc_builtin": 465, "subagent": 13, "vendor_mcp": 3 }, "tool_result_tokens": 262089, "tool_time_ms": 33812, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.467519000000003, "cost_usd": 12.467519000000003, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1952378, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 238, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9407568, "tokens_cache_write": 778199, "tokens_input": 393, "tokens_output": 47832, "tool_calls": 484, "tool_calls_by_class": { "cc_builtin": 467, "subagent": 13, "vendor_mcp": 4 }, "tool_result_tokens": 284497, "tool_time_ms": 39035, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.667937250000001, "cost_usd": 10.667937250000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1376326, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 188, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6638127, "tokens_cache_write": 801446, "tokens_input": 7589, "tokens_output": 31003, "tool_calls": 365, "tool_calls_by_class": { "cc_builtin": 347, "subagent": 14, "vendor_mcp": 4 }, "tool_result_tokens": 273021, "tool_time_ms": 15885, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.601068750000001, "cost_usd": 9.601068750000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1561972, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 187, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6570885, "tokens_cache_write": 650023, "tokens_input": 269, "tokens_output": 31654, "tool_calls": 386, "tool_calls_by_class": { "cc_builtin": 371, "subagent": 11, "vendor_mcp": 4 }, "tool_result_tokens": 206772, "tool_time_ms": 13813, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.423435249999999, "cost_usd": 10.423435249999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1615726, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 189, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6737218, "tokens_cache_write": 782359, "tokens_input": 3988, "tokens_output": 37492, "tool_calls": 373, "tool_calls_by_class": { "cc_builtin": 355, "subagent": 13, "vendor_mcp": 5 }, "tool_result_tokens": 285570, "tool_time_ms": 15464, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.5302445, "cost_usd": 9.5302445, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1509785, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 190, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6709909, "tokens_cache_write": 666243, "tokens_input": 2205, "tokens_output": 39400, "tool_calls": 342, "tool_calls_by_class": { "cc_builtin": 325, "subagent": 11, "vendor_mcp": 6 }, "tool_result_tokens": 241422, "tool_time_ms": 13963, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.3208835, "cost_usd": 9.3208835, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1563293, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 189, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6590972, "tokens_cache_write": 642357, "tokens_input": 273, "tokens_output": 38535, "tool_calls": 355, "tool_calls_by_class": { "cc_builtin": 338, "subagent": 12, "vendor_mcp": 5 }, "tool_result_tokens": 201739, "tool_time_ms": 14722, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 9.561066, "cost_usd": 9.561066, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1332245, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 182, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6395307, "tokens_cache_write": 667682, "tokens_input": 307, "tokens_output": 38910, "tool_calls": 339, "tool_calls_by_class": { "cc_builtin": 322, "subagent": 11, "vendor_mcp": 6 }, "tool_result_tokens": 251921, "tool_time_ms": 13719, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.385425750000001, "cost_usd": 10.385425750000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1815524, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 208, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7365769, "tokens_cache_write": 730668, "tokens_input": 3979, "tokens_output": 35303, "tool_calls": 400, "tool_calls_by_class": { "cc_builtin": 377, "subagent": 14, "vendor_mcp": 9 }, "tool_result_tokens": 227631, "tool_time_ms": 15416, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.00391725, "cost_usd": 10.00391725, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1564712, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 194, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6897087, "tokens_cache_write": 717781, "tokens_input": 1063, "tokens_output": 33611, "tool_calls": 379, "tool_calls_by_class": { "cc_builtin": 363, "subagent": 12, "vendor_mcp": 4 }, "tool_result_tokens": 261526, "tool_time_ms": 14118, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.21728875, "cost_usd": 10.21728875, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1626974, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 199, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7027900, "tokens_cache_write": 716441, "tokens_input": 2386, "tokens_output": 38578, "tool_calls": 378, "tool_calls_by_class": { "cc_builtin": 360, "subagent": 14, "vendor_mcp": 4 }, "tool_result_tokens": 245333, "tool_time_ms": 15706, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 10.160860749999998, "cost_usd": 10.160860749999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 1466881, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 203, "model": "claude-opus-4-6", "path": "runs/serena/claude-opus-4-6/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-6|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7253529, "tokens_cache_write": 716536, "tokens_input": 295, "tokens_output": 36284, "tool_calls": 374, "tool_calls_by_class": { "cc_builtin": 353, "subagent": 15, "vendor_mcp": 6 }, "tool_result_tokens": 252362, "tool_time_ms": 15090, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.954167, "cost_usd": 5.954167, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 590373, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7383069, "tokens_cache_write": 147946, "tokens_input": 177, "tokens_output": 33974, "tool_calls": 74, "tool_calls_by_class": { "cc_builtin": 72, "subagent": 2 }, "tool_result_tokens": 30070, "tool_time_ms": 21792, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.817213249999999, "cost_usd": 5.817213249999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 637740, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7869469, "tokens_cache_write": 108410, "tokens_input": 204, "tokens_output": 33783, "tool_calls": 92, "tool_calls_by_class": { "cc_builtin": 90, "subagent": 2 }, "tool_result_tokens": 25568, "tool_time_ms": 7870, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2366972500000015, "cost_usd": 5.2366972500000015, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 619509, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6562982, "tokens_cache_write": 111467, "tokens_input": 165, "tokens_output": 36323, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 69, "subagent": 1 }, "tool_result_tokens": 25613, "tool_time_ms": 8402, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.6721825, "cost_usd": 5.6721825, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 728860, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7231555, "tokens_cache_write": 121123, "tokens_input": 174, "tokens_output": 36069, "tool_calls": 88, "tool_calls_by_class": { "cc_builtin": 87, "subagent": 1 }, "tool_result_tokens": 30947, "tool_time_ms": 8940, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.07157425, "cost_usd": 6.07157425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 663441, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8304546, "tokens_cache_write": 110380, "tokens_input": 203, "tokens_output": 34990, "tool_calls": 89, "tool_calls_by_class": { "cc_builtin": 88, "subagent": 1 }, "tool_result_tokens": 27097, "tool_time_ms": 9524, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.788169, "cost_usd": 5.788169, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 733389, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7706118, "tokens_cache_write": 108160, "tokens_input": 203, "tokens_output": 36117, "tool_calls": 90, "tool_calls_by_class": { "cc_builtin": 88, "subagent": 2 }, "tool_result_tokens": 25408, "tool_time_ms": 22374, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.918376, "cost_usd": 4.918376, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 551266, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 80, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6636282, "tokens_cache_write": 90105, "tokens_input": 152, "tokens_output": 27937, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 32149, "tool_time_ms": 7655, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.162902999999999, "cost_usd": 5.162902999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 613925, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 89, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6728415, "tokens_cache_write": 110132, "tokens_input": 169, "tokens_output": 31199, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 70, "subagent": 2 }, "tool_result_tokens": 27557, "tool_time_ms": 38763, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.098306749999999, "cost_usd": 8.098306749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1145812, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 136, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8681151, "tokens_cache_write": 265448, "tokens_input": 2226, "tokens_output": 42730, "tool_calls": 157, "tool_calls_by_class": { "cc_builtin": 138, "subagent": 19 }, "tool_result_tokens": 75477, "tool_time_ms": 32517, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.52421425, "cost_usd": 5.52421425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 636344, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 95, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7185816, "tokens_cache_write": 107392, "tokens_input": 181, "tokens_output": 36711, "tool_calls": 76, "tool_calls_by_class": { "cc_builtin": 75, "subagent": 1 }, "tool_result_tokens": 24470, "tool_time_ms": 18293, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.6273000000000013, "cost_usd": 3.6273000000000013, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 392867, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 85, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4720980, "tokens_cache_write": 81220, "tokens_input": 162, "tokens_output": 18152, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 11103, "tool_time_ms": 3276, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4425805000000005, "cost_usd": 3.4425805000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 432329, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 84, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4986641, "tokens_cache_write": 51151, "tokens_input": 160, "tokens_output": 17478, "tool_calls": 54, "tool_calls_by_class": { "cc_builtin": 54 }, "tool_result_tokens": 13570, "tool_time_ms": 3226, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.3344015, "cost_usd": 3.3344015, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 422993, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4685063, "tokens_cache_write": 48318, "tokens_input": 158, "tokens_output": 20316, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 10859, "tool_time_ms": 3706, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.008766, "cost_usd": 3.008766, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 377034, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 78, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4280742, "tokens_cache_write": 44690, "tokens_input": 149, "tokens_output": 16830, "tool_calls": 48, "tool_calls_by_class": { "cc_builtin": 48 }, "tool_result_tokens": 10994, "tool_time_ms": 3326, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1387094999999996, "cost_usd": 3.1387094999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 413444, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 82, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4547129, "tokens_cache_write": 45314, "tokens_input": 156, "tokens_output": 16449, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 11237, "tool_time_ms": 3658, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4011795, "cost_usd": 3.4011795, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 445336, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 83, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4798309, "tokens_cache_write": 52576, "tokens_input": 158, "tokens_output": 19019, "tool_calls": 53, "tool_calls_by_class": { "cc_builtin": 53 }, "tool_result_tokens": 13560, "tool_time_ms": 5744, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.9789025, "cost_usd": 2.9789025, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 375342, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4282785, "tokens_cache_write": 43028, "tokens_input": 151, "tokens_output": 16259, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 10211, "tool_time_ms": 5594, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.340848, "cost_usd": 3.340848, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 409545, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4909586, "tokens_cache_write": 42490, "tokens_input": 166, "tokens_output": 18413, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 11376, "tool_time_ms": 4588, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8559715000000003, "cost_usd": 2.8559715000000003, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 351204, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 76, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4133113, "tokens_cache_write": 40239, "tokens_input": 145, "tokens_output": 15452, "tool_calls": 46, "tool_calls_by_class": { "cc_builtin": 46 }, "tool_result_tokens": 11428, "tool_time_ms": 4693, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1565325, "cost_usd": 3.1565325, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 365225, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 79, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4583985, "tokens_cache_write": 47404, "tokens_input": 150, "tokens_output": 15590, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 12401, "tool_time_ms": 5464, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.89952875, "cost_usd": 0.89952875, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 90135, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 18, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 545015, "tokens_cache_write": 68052, "tokens_input": 35, "tokens_output": 4127, "tool_calls": 19, "tool_calls_by_class": { "cc_builtin": 15, "subagent": 4 }, "tool_result_tokens": 11051, "tool_time_ms": 656, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.95200975, "cost_usd": 0.95200975, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 47528, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 19, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 585577, "tokens_cache_write": 72196, "tokens_input": 37, "tokens_output": 4665, "tool_calls": 20, "tool_calls_by_class": { "cc_builtin": 18, "subagent": 2 }, "tool_result_tokens": 9103, "tool_time_ms": 783, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.9505645, "cost_usd": 0.9505645, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 25514, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 21, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 668564, "tokens_cache_write": 61488, "tokens_input": 41, "tokens_output": 4649, "tool_calls": 23, "tool_calls_by_class": { "cc_builtin": 20, "subagent": 3 }, "tool_result_tokens": 8769, "tool_time_ms": 1123, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.8706502500000001, "cost_usd": 0.8706502500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 39983, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 19, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 582273, "tokens_cache_write": 61268, "tokens_input": 38, "tokens_output": 3623, "tool_calls": 20, "tool_calls_by_class": { "cc_builtin": 16, "subagent": 4 }, "tool_result_tokens": 8463, "tool_time_ms": 792, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 1.2726245, "cost_usd": 1.2726245, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 20740, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 28, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 829329, "tokens_cache_write": 89638, "tokens_input": 55, "tokens_output": 5442, "tool_calls": 32, "tool_calls_by_class": { "cc_builtin": 28, "subagent": 4 }, "tool_result_tokens": 13543, "tool_time_ms": 1401, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.6145974999999999, "cost_usd": 0.6145974999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 29851, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 12, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 350650, "tokens_cache_write": 41943, "tokens_input": 24, "tokens_output": 3475, "tool_calls": 14, "tool_calls_by_class": { "cc_builtin": 13, "subagent": 1 }, "tool_result_tokens": 6489, "tool_time_ms": 829, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.99244925, "cost_usd": 0.99244925, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 79035, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 23, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 742106, "tokens_cache_write": 65422, "tokens_input": 45, "tokens_output": 3760, "tool_calls": 28, "tool_calls_by_class": { "cc_builtin": 24, "subagent": 4 }, "tool_result_tokens": 11250, "tool_time_ms": 1066, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.82753875, "cost_usd": 0.82753875, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 27758, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 22, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 725090, "tokens_cache_write": 42432, "tokens_input": 43, "tokens_output": 4027, "tool_calls": 23, "tool_calls_by_class": { "cc_builtin": 20, "subagent": 3 }, "tool_result_tokens": 10851, "tool_time_ms": 1124, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.8800267500000001, "cost_usd": 0.8800267500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 69614, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 18, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 554766, "tokens_cache_write": 65474, "tokens_input": 36, "tokens_output": 3437, "tool_calls": 21, "tool_calls_by_class": { "cc_builtin": 17, "subagent": 4 }, "tool_result_tokens": 10979, "tool_time_ms": 822, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 0.5766210000000002, "cost_usd": 0.5766210000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 46337, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 16, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 521082, "tokens_cache_write": 31359, "tokens_input": 31, "tokens_output": 2319, "tool_calls": 18, "tool_calls_by_class": { "cc_builtin": 16, "subagent": 2 }, "tool_result_tokens": 4122, "tool_time_ms": 1136, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.303342749999999, "cost_usd": 10.303342749999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1127865, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 214, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep11", "query_calls": 0, "rep": 11, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|11", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7397168, "tokens_cache_write": 591425, "tokens_input": 2381, "tokens_output": 60399, "tool_calls": 238, "tool_calls_by_class": { "cc_builtin": 218, "subagent": 20 }, "tool_result_tokens": 120153, "tool_time_ms": 29852, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.460608500000001, "cost_usd": 11.460608500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1124304, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 237, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep12", "query_calls": 0, "rep": 12, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|12", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8028022, "tokens_cache_write": 646714, "tokens_input": 2113, "tokens_output": 66999, "tool_calls": 277, "tool_calls_by_class": { "cc_builtin": 256, "subagent": 21 }, "tool_result_tokens": 125873, "tool_time_ms": 80848, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 12.082794499999999, "cost_usd": 12.082794499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 647276, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 247, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep13", "query_calls": 0, "rep": 13, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|13", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8391829, "tokens_cache_write": 701988, "tokens_input": 7894, "tokens_output": 63620, "tool_calls": 288, "tool_calls_by_class": { "cc_builtin": 270, "subagent": 18 }, "tool_result_tokens": 144061, "tool_time_ms": 50529, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.7579505, "cost_usd": 10.7579505, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 867130, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 223, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep14", "query_calls": 0, "rep": 14, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|14", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7736841, "tokens_cache_write": 609301, "tokens_input": 4991, "tokens_output": 59973, "tool_calls": 275, "tool_calls_by_class": { "cc_builtin": 259, "subagent": 16 }, "tool_result_tokens": 125599, "tool_time_ms": 24370, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.670795749999998, "cost_usd": 11.670795749999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1225173, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 7, "family": "cc_competitor", "llm_calls": 236, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep15", "query_calls": 0, "rep": 15, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|15", "score": 13, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8158638, "tokens_cache_write": 676753, "tokens_input": 639, "tokens_output": 50315, "tool_calls": 282, "tool_calls_by_class": { "cc_builtin": 263, "subagent": 19 }, "tool_result_tokens": 135514, "tool_time_ms": 103498, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.20512075, "cost_usd": 10.20512075, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 823293, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 203, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep16", "query_calls": 0, "rep": 16, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|16", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7117395, "tokens_cache_write": 571941, "tokens_input": 396, "tokens_output": 54824, "tool_calls": 250, "tool_calls_by_class": { "cc_builtin": 236, "subagent": 14 }, "tool_result_tokens": 131153, "tool_time_ms": 30514, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 9.926456499999999, "cost_usd": 9.926456499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 688803, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 213, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep17", "query_calls": 0, "rep": 17, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|17", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7394448, "tokens_cache_write": 525111, "tokens_input": 1369, "tokens_output": 60035, "tool_calls": 244, "tool_calls_by_class": { "cc_builtin": 229, "subagent": 15 }, "tool_result_tokens": 103306, "tool_time_ms": 37240, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.096497499999998, "cost_usd": 10.096497499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 717704, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 218, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep18", "query_calls": 0, "rep": 18, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|18", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7573913, "tokens_cache_write": 557856, "tokens_input": 426, "tokens_output": 57298, "tool_calls": 248, "tool_calls_by_class": { "cc_builtin": 228, "subagent": 20 }, "tool_result_tokens": 107038, "tool_time_ms": 25419, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 11.56871425, "cost_usd": 11.56871425, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1607066, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 229, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep19", "query_calls": 0, "rep": 19, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|19", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8104383, "tokens_cache_write": 674915, "tokens_input": 442, "tokens_output": 66477, "tool_calls": 277, "tool_calls_by_class": { "cc_builtin": 257, "subagent": 18, "vendor_mcp": 2 }, "tool_result_tokens": 129632, "tool_time_ms": 408503, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 10.715879, "cost_usd": 10.715879, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 842377, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 215, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos-hard/rep20", "query_calls": 0, "rep": 20, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|ffc82068051276c8ecade67885ff45af|20", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7742504, "tokens_cache_write": 623903, "tokens_input": 729, "tokens_output": 58099, "tool_calls": 239, "tool_calls_by_class": { "cc_builtin": 224, "subagent": 15 }, "tool_result_tokens": 141432, "tool_time_ms": 96731, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2009855, "cost_usd": 0.2009855, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 20524, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 197861, "tokens_cache_write": 8237, "tokens_input": 12, "tokens_output": 785, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 1110, "tool_time_ms": 2034, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.2292925, "cost_usd": 0.2292925, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 26393, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 8, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 276675, "tokens_cache_write": 6760, "tokens_input": 16, "tokens_output": 931, "tool_calls": 6, "tool_calls_by_class": { "cc_builtin": 6 }, "tool_result_tokens": 574, "tool_time_ms": 3631, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.1690125, "cost_usd": 0.1690125, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 25435, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 200995, "tokens_cache_write": 4753, "tokens_input": 12, "tokens_output": 837, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 887, "tool_time_ms": 1015, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.1886345, "cost_usd": 0.1886345, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 25950, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 198909, "tokens_cache_write": 6632, "tokens_input": 487, "tokens_output": 817, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 906, "tool_time_ms": 10504, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.164374, "cost_usd": 0.164374, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 24958, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 200878, "tokens_cache_write": 4580, "tokens_input": 12, "tokens_output": 723, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 820, "tool_time_ms": 1670, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.141625, "cost_usd": 0.141625, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 17691, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 203630, "tokens_cache_write": 1950, "tokens_input": 12, "tokens_output": 810, "tool_calls": 5, "tool_calls_by_class": { "cc_builtin": 5 }, "tool_result_tokens": 711, "tool_time_ms": 477, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.1926465, "cost_usd": 0.1926465, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 24042, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 228503, "tokens_cache_write": 5991, "tokens_input": 12, "tokens_output": 737, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 582, "tool_time_ms": 245, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.548303, "cost_usd": 0.548303, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 22129, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 7, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 233636, "tokens_cache_write": 40549, "tokens_input": 14, "tokens_output": 1037, "tool_calls": 6, "tool_calls_by_class": { "cc_builtin": 6 }, "tool_result_tokens": 546, "tool_time_ms": 384, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.4714542500000001, "cost_usd": 0.4714542500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 11848, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 11, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 346246, "tokens_cache_write": 32500, "tokens_input": 22, "tokens_output": 1715, "tool_calls": 8, "tool_calls_by_class": { "cc_builtin": 7, "subagent": 1 }, "tool_result_tokens": 2530, "tool_time_ms": 419, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 0.221173, "cost_usd": 0.221173, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 17445, "exam": "ZeroShotExam", "exam_epoch": null, "fails": null, "family": "cc_competitor", "llm_calls": 6, "model": "claude-opus-4-8", "path": "runs/serena/claude-opus-4-8/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-4-8|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": null, "scorer_era": null, "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 225956, "tokens_cache_write": 9021, "tokens_input": 12, "tokens_output": 717, "tool_calls": 4, "tool_calls_by_class": { "cc_builtin": 4 }, "tool_result_tokens": 873, "tool_time_ms": 260, "validity_ok": false, "validity_reason": "partial_exam" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.4291365000000007, "cost_usd": 3.4291365000000007, "denominator": 20, "duration_basis": "run", "duration_ms": 373931, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 43, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3857353, "tokens_cache_write": 91061, "tokens_input": 80, "tokens_output": 23578, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 66, "vendor_mcp": 1 }, "tool_result_tokens": 36960, "tool_time_ms": 38213, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.1661090000000005, "cost_usd": 3.1661090000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 433677, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 30, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|2", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2525088, "tokens_cache_write": 122219, "tokens_input": 55, "tokens_output": 27244, "tool_calls": 51, "tool_calls_by_class": { "cc_builtin": 51 }, "tool_result_tokens": 34148, "tool_time_ms": 76494, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6375115000000005, "cost_usd": 2.6375115000000005, "denominator": 20, "duration_basis": "run", "duration_ms": 397970, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 33, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2715093, "tokens_cache_write": 73686, "tokens_input": 61, "tokens_output": 21712, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 31720, "tool_time_ms": 178549, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0473820000000003, "cost_usd": 3.0473820000000003, "denominator": 20, "duration_basis": "run", "duration_ms": 361593, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 34, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3028524, "tokens_cache_write": 92613, "tokens_input": 63, "tokens_output": 24267, "tool_calls": 52, "tool_calls_by_class": { "cc_builtin": 52 }, "tool_result_tokens": 39037, "tool_time_ms": 24236, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.0520549999999997, "cost_usd": 3.0520549999999997, "denominator": 20, "duration_basis": "run", "duration_ms": 414150, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 39, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|5", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3340040, "tokens_cache_write": 77252, "tokens_input": 73, "tokens_output": 24366, "tool_calls": 59, "tool_calls_by_class": { "cc_builtin": 59 }, "tool_result_tokens": 33170, "tool_time_ms": 79009, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.605967499999999, "cost_usd": 3.605967499999999, "denominator": 20, "duration_basis": "run", "duration_ms": 439517, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 46, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 4059335, "tokens_cache_write": 95297, "tokens_input": 4431, "tokens_output": 24047, "tool_calls": 78, "tool_calls_by_class": { "cc_builtin": 78 }, "tool_result_tokens": 39652, "tool_time_ms": 68332, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.420106, "cost_usd": 2.420106, "denominator": 20, "duration_basis": "run", "duration_ms": 329537, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2118332, "tokens_cache_write": 87486, "tokens_input": 2011, "tokens_output": 19041, "tool_calls": 41, "tool_calls_by_class": { "cc_builtin": 41 }, "tool_result_tokens": 38347, "tool_time_ms": 122427, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.722453, "cost_usd": 2.722453, "denominator": 20, "duration_basis": "run", "duration_ms": 417891, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 33, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2728256, "tokens_cache_write": 80292, "tokens_input": 3261, "tokens_output": 21564, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 37415, "tool_time_ms": 98657, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8922475, "cost_usd": 2.8922475, "denominator": 20, "duration_basis": "run", "duration_ms": 418954, "exam": "HumanExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 36, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3068845, "tokens_cache_write": 81094, "tokens_input": 67, "tokens_output": 21862, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 37801, "tool_time_ms": 99530, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 3.480105000000001, "cost_usd": 3.480105000000001, "denominator": 20, "duration_basis": "run", "duration_ms": 448071, "exam": "HumanExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 41, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|ffc82068051276c8ecade67885ff45af|10", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 3782170, "tokens_cache_write": 99109, "tokens_input": 796, "tokens_output": 23758, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 65 }, "tool_result_tokens": 42089, "tool_time_ms": 98678, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8280809999999996, "cost_usd": 2.8280809999999996, "denominator": 30, "duration_basis": "run", "duration_ms": 272490, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 33, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2505962, "tokens_cache_write": 111595, "tokens_input": 60, "tokens_output": 18354, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 30428, "tool_time_ms": 3014, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.6358145000000002, "cost_usd": 1.6358145000000002, "denominator": 30, "duration_basis": "run", "duration_ms": 210535, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 19, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1293599, "tokens_cache_write": 59990, "tokens_input": 33, "tokens_output": 15558, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 22358, "tool_time_ms": 2337, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9307755, "cost_usd": 1.9307755, "denominator": 30, "duration_basis": "run", "duration_ms": 216765, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 23, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1652731, "tokens_cache_write": 72300, "tokens_input": 42, "tokens_output": 15248, "tool_calls": 44, "tool_calls_by_class": { "cc_builtin": 44 }, "tool_result_tokens": 29631, "tool_time_ms": 2225, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2544805, "cost_usd": 2.2544805, "denominator": 30, "duration_basis": "run", "duration_ms": 260404, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 28, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2089401, "tokens_cache_write": 75147, "tokens_input": 52, "tokens_output": 18322, "tool_calls": 50, "tool_calls_by_class": { "cc_builtin": 50 }, "tool_result_tokens": 29667, "tool_time_ms": 3322, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4091264999999997, "cost_usd": 2.4091264999999997, "denominator": 30, "duration_basis": "run", "duration_ms": 279154, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 32, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2401553, "tokens_cache_write": 73618, "tokens_input": 59, "tokens_output": 18875, "tool_calls": 56, "tool_calls_by_class": { "cc_builtin": 56 }, "tool_result_tokens": 28411, "tool_time_ms": 2454, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9089449999999997, "cost_usd": 1.9089449999999997, "denominator": 30, "duration_basis": "run", "duration_ms": 213001, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 25, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1781550, "tokens_cache_write": 65464, "tokens_input": 46, "tokens_output": 14532, "tool_calls": 46, "tool_calls_by_class": { "cc_builtin": 46 }, "tool_result_tokens": 25635, "tool_time_ms": 1961, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.83908, "cost_usd": 1.83908, "denominator": 30, "duration_basis": "run", "duration_ms": 214552, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 22, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1542910, "tokens_cache_write": 66845, "tokens_input": 40, "tokens_output": 15959, "tool_calls": 41, "tool_calls_by_class": { "cc_builtin": 41 }, "tool_result_tokens": 27130, "tool_time_ms": 2238, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.031392, "cost_usd": 2.031392, "denominator": 30, "duration_basis": "run", "duration_ms": 234054, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 27, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1882304, "tokens_cache_write": 67929, "tokens_input": 50, "tokens_output": 16428, "tool_calls": 49, "tool_calls_by_class": { "cc_builtin": 49 }, "tool_result_tokens": 26380, "tool_time_ms": 2309, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 1.9339445, "cost_usd": 1.9339445, "denominator": 30, "duration_basis": "run", "duration_ms": 217700, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 24, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 1706009, "tokens_cache_write": 70102, "tokens_input": 44, "tokens_output": 15188, "tool_calls": 45, "tool_calls_by_class": { "cc_builtin": 45 }, "tool_result_tokens": 29056, "tool_time_ms": 2185, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1366085000000004, "cost_usd": 2.1366085000000004, "denominator": 30, "duration_basis": "run", "duration_ms": 216465, "exam": "HumanExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 26, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/HumanExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|HumanExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 2008957, "tokens_cache_write": 73999, "tokens_input": 48, "tokens_output": 15676, "tool_calls": 47, "tool_calls_by_class": { "cc_builtin": 47 }, "tool_result_tokens": 30014, "tool_time_ms": 1992, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.760269500000001, "cost_usd": 7.760269500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 901490, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 102, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10198019, "tokens_cache_write": 159560, "tokens_input": 192, "tokens_output": 42588, "tool_calls": 108, "tool_calls_by_class": { "cc_builtin": 105, "vendor_mcp": 3 }, "tool_result_tokens": 41591, "tool_time_ms": 63134, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.557179500000001, "cost_usd": 7.557179500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 769759, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 94, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10286369, "tokens_cache_write": 140338, "tokens_input": 1238, "tokens_output": 40177, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 106 }, "tool_result_tokens": 51879, "tool_time_ms": 25245, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.872197999999999, "cost_usd": 6.872197999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 848717, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9372206, "tokens_cache_write": 117533, "tokens_input": 183, "tokens_output": 40394, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 98 }, "tool_result_tokens": 38693, "tool_time_ms": 47538, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.829804500000001, "cost_usd": 6.829804500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 884266, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9216479, "tokens_cache_write": 122191, "tokens_input": 1811, "tokens_output": 39624, "tool_calls": 91, "tool_calls_by_class": { "cc_builtin": 91 }, "tool_result_tokens": 42394, "tool_time_ms": 126643, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.1292485, "cost_usd": 7.1292485, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 849623, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 96, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9645427, "tokens_cache_write": 125706, "tokens_input": 180, "tokens_output": 41943, "tool_calls": 109, "tool_calls_by_class": { "cc_builtin": 109 }, "tool_result_tokens": 42497, "tool_time_ms": 52550, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.9468195, "cost_usd": 7.9468195, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 974901, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 105, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10993019, "tokens_cache_write": 129507, "tokens_input": 198, "tokens_output": 46170, "tool_calls": 104, "tool_calls_by_class": { "cc_builtin": 104 }, "tool_result_tokens": 42430, "tool_time_ms": 87036, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.456089499999997, "cost_usd": 8.456089499999997, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1010549, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11868199, "tokens_cache_write": 138673, "tokens_input": 202, "tokens_output": 45370, "tool_calls": 104, "tool_calls_by_class": { "cc_builtin": 104 }, "tool_result_tokens": 48917, "tool_time_ms": 149120, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.382405, "cost_usd": 7.382405, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 811221, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 106, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10549090, "tokens_cache_write": 119213, "tokens_input": 5306, "tokens_output": 35568, "tool_calls": 115, "tool_calls_by_class": { "cc_builtin": 115 }, "tool_result_tokens": 41741, "tool_time_ms": 63006, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.693307999999999, "cost_usd": 6.693307999999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 752318, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 91, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9155396, "tokens_cache_write": 122669, "tokens_input": 284, "tokens_output": 35500, "tool_calls": 84, "tool_calls_by_class": { "cc_builtin": 84 }, "tool_result_tokens": 45775, "tool_time_ms": 34955, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.0457645, "cost_usd": 7.0457645, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 808042, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 104, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9884569, "tokens_cache_write": 110414, "tokens_input": 198, "tokens_output": 39934, "tool_calls": 86, "tool_calls_by_class": { "cc_builtin": 86 }, "tool_result_tokens": 35649, "tool_time_ms": 32976, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.0088995, "cost_usd": 5.0088995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 545657, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 97, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6599179, "tokens_cache_write": 109026, "tokens_input": 185, "tokens_output": 24725, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 67 }, "tool_result_tokens": 23194, "tool_time_ms": 3142, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.857658499999999, "cost_usd": 4.857658499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 515134, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6959037, "tokens_cache_write": 78130, "tokens_input": 188, "tokens_output": 23836, "tool_calls": 70, "tool_calls_by_class": { "cc_builtin": 70 }, "tool_result_tokens": 25580, "tool_time_ms": 3319, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.258203999999999, "cost_usd": 5.258203999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 525992, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 107, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7627518, "tokens_cache_write": 79170, "tokens_input": 204, "tokens_output": 26069, "tool_calls": 77, "tool_calls_by_class": { "cc_builtin": 77 }, "tool_result_tokens": 24769, "tool_time_ms": 3669, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.612025999999999, "cost_usd": 4.612025999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 536058, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 98, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6615122, "tokens_cache_write": 73471, "tokens_input": 186, "tokens_output": 22753, "tool_calls": 72, "tool_calls_by_class": { "cc_builtin": 72 }, "tool_result_tokens": 23272, "tool_time_ms": 3653, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.2004215, "cost_usd": 5.2004215, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 524997, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 99, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6836603, "tokens_cache_write": 114240, "tokens_input": 189, "tokens_output": 25551, "tool_calls": 69, "tool_calls_by_class": { "cc_builtin": 69 }, "tool_result_tokens": 25309, "tool_time_ms": 3859, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.483899500000001, "cost_usd": 4.483899500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 503450, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 96, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6441609, "tokens_cache_write": 70288, "tokens_input": 183, "tokens_output": 22372, "tool_calls": 66, "tool_calls_by_class": { "cc_builtin": 66 }, "tool_result_tokens": 22025, "tool_time_ms": 3161, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.3775699999999995, "cost_usd": 4.3775699999999995, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 445010, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6220570, "tokens_cache_write": 66841, "tokens_input": 175, "tokens_output": 23920, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 21647, "tool_time_ms": 3600, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.229973000000001, "cost_usd": 5.229973000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 513070, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 101, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7518486, "tokens_cache_write": 78667, "tokens_input": 192, "tokens_output": 27324, "tool_calls": 71, "tool_calls_by_class": { "cc_builtin": 71 }, "tool_result_tokens": 26644, "tool_time_ms": 3677, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.6975565, "cost_usd": 4.6975565, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 642495, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 95, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6639453, "tokens_cache_write": 77720, "tokens_input": 181, "tokens_output": 23989, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 65 }, "tool_result_tokens": 25378, "tool_time_ms": 3625, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 4.3911765, "cost_usd": 4.3911765, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 475887, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 90, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6090003, "tokens_cache_write": 76887, "tokens_input": 171, "tokens_output": 23058, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 24917, "tool_time_ms": 3915, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.130121500000001, "cost_usd": 8.130121500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 971584, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 140, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6143983, "tokens_cache_write": 361319, "tokens_input": 563, "tokens_output": 57685, "tool_calls": 202, "tool_calls_by_class": { "cc_builtin": 202 }, "tool_result_tokens": 111241, "tool_time_ms": 57544, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.534974, "cost_usd": 7.534974, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1029795, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 144, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6238718, "tokens_cache_write": 299691, "tokens_input": 276, "tokens_output": 56693, "tool_calls": 201, "tool_calls_by_class": { "cc_builtin": 201 }, "tool_result_tokens": 95599, "tool_time_ms": 93439, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.64597, "cost_usd": 7.64597, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 951403, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 142, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6369940, "tokens_cache_write": 303530, "tokens_input": 570, "tokens_output": 56914, "tool_calls": 199, "tool_calls_by_class": { "cc_builtin": 199 }, "tool_result_tokens": 116397, "tool_time_ms": 66318, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.832736500000001, "cost_usd": 7.832736500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 998908, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 141, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6350143, "tokens_cache_write": 308050, "tokens_input": 268, "tokens_output": 63033, "tool_calls": 204, "tool_calls_by_class": { "cc_builtin": 204 }, "tool_result_tokens": 114792, "tool_time_ms": 44196, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.264650499999998, "cost_usd": 8.264650499999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1187985, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 151, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6951151, "tokens_cache_write": 314984, "tokens_input": 287, "tokens_output": 65512, "tool_calls": 215, "tool_calls_by_class": { "cc_builtin": 215 }, "tool_result_tokens": 117468, "tool_time_ms": 149159, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.5279620000000005, "cost_usd": 7.5279620000000005, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 970342, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 145, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6382084, "tokens_cache_write": 283924, "tokens_input": 276, "tokens_output": 59852, "tool_calls": 205, "tool_calls_by_class": { "cc_builtin": 205 }, "tool_result_tokens": 101867, "tool_time_ms": 54506, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.915423, "cost_usd": 7.915423, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1076310, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 147, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6657216, "tokens_cache_write": 308328, "tokens_input": 282, "tokens_output": 60085, "tool_calls": 206, "tool_calls_by_class": { "cc_builtin": 206 }, "tool_result_tokens": 116837, "tool_time_ms": 129952, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.925015000000002, "cost_usd": 7.925015000000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1030623, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 153, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 20, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6762530, "tokens_cache_write": 295424, "tokens_input": 587, "tokens_output": 63463, "tool_calls": 224, "tool_calls_by_class": { "cc_builtin": 224 }, "tool_result_tokens": 107181, "tool_time_ms": 21590, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.612069500000001, "cost_usd": 7.612069500000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1009524, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 146, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6476379, "tokens_cache_write": 290246, "tokens_input": 279, "tokens_output": 58801, "tool_calls": 208, "tool_calls_by_class": { "cc_builtin": 208 }, "tool_result_tokens": 106535, "tool_time_ms": 77292, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.854230499999999, "cost_usd": 7.854230499999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1061256, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 148, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6688101, "tokens_cache_write": 298312, "tokens_input": 282, "tokens_output": 61026, "tool_calls": 209, "tool_calls_by_class": { "cc_builtin": 209 }, "tool_result_tokens": 109878, "tool_time_ms": 96434, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.441677, "cost_usd": 7.441677, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 728965, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 154, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6476434, "tokens_cache_write": 310980, "tokens_input": 657, "tokens_output": 43615, "tool_calls": 181, "tool_calls_by_class": { "cc_builtin": 181 }, "tool_result_tokens": 94709, "tool_time_ms": 8863, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.832409500000001, "cost_usd": 6.832409500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 714763, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 142, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5835209, "tokens_cache_write": 285414, "tokens_input": 278, "tokens_output": 42371, "tool_calls": 159, "tool_calls_by_class": { "cc_builtin": 159 }, "tool_result_tokens": 81867, "tool_time_ms": 8817, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.755997999999999, "cost_usd": 6.755997999999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 676775, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 149, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6177346, "tokens_cache_write": 263883, "tokens_input": 294, "tokens_output": 41081, "tool_calls": 170, "tool_calls_by_class": { "cc_builtin": 170 }, "tool_result_tokens": 88837, "tool_time_ms": 8466, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.758977000000001, "cost_usd": 6.758977000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 712101, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 146, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6116674, "tokens_cache_write": 262290, "tokens_input": 2393, "tokens_output": 42631, "tool_calls": 174, "tool_calls_by_class": { "cc_builtin": 174 }, "tool_result_tokens": 86452, "tool_time_ms": 9006, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.876274500000001, "cost_usd": 6.876274500000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 752502, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 149, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6254099, "tokens_cache_write": 264578, "tokens_input": 404, "tokens_output": 44057, "tool_calls": 175, "tool_calls_by_class": { "cc_builtin": 175 }, "tool_result_tokens": 87580, "tool_time_ms": 8564, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.566043, "cost_usd": 6.566043, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 713881, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 147, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6121196, "tokens_cache_write": 248812, "tokens_input": 480, "tokens_output": 40597, "tool_calls": 164, "tool_calls_by_class": { "cc_builtin": 164 }, "tool_result_tokens": 81861, "tool_time_ms": 8832, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.970592000000001, "cost_usd": 6.970592000000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 786713, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 158, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6626294, "tokens_cache_write": 256977, "tokens_input": 535, "tokens_output": 43400, "tool_calls": 177, "tool_calls_by_class": { "cc_builtin": 177 }, "tool_result_tokens": 85339, "tool_time_ms": 8684, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.8636965000000005, "cost_usd": 6.8636965000000005, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 770401, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 156, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6537003, "tokens_cache_write": 245963, "tokens_input": 2048, "tokens_output": 45013, "tool_calls": 189, "tool_calls_by_class": { "cc_builtin": 189 }, "tool_result_tokens": 80217, "tool_time_ms": 8671, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.601424499999999, "cost_usd": 6.601424499999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 740531, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 151, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6321129, "tokens_cache_write": 239820, "tokens_input": 3167, "tokens_output": 41073, "tool_calls": 174, "tool_calls_by_class": { "cc_builtin": 174 }, "tool_result_tokens": 88810, "tool_time_ms": 9580, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.8186695, "cost_usd": 6.8186695, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 743117, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 155, "model": "claude-opus-5", "path": "runs/serena/claude-opus-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-opus-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6531789, "tokens_cache_write": 245558, "tokens_input": 714, "tokens_output": 43745, "tool_calls": 184, "tool_calls_by_class": { "cc_builtin": 184 }, "tool_result_tokens": 88020, "tool_time_ms": 9021, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.742533150000002, "cost_usd": 6.742533150000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 782658, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 150, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|1", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16469928, "tokens_cache_write": 199946, "tokens_input": 300, "tokens_output": 44142, "tool_calls": 132, "tool_calls_by_class": { "cc_builtin": 130, "subagent": 2 }, "tool_result_tokens": 40276, "tool_time_ms": 28605, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.176762650000001, "cost_usd": 5.176762650000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 591863, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 110, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|2", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12186398, "tokens_cache_write": 165940, "tokens_input": 8703, "tokens_output": 36300, "tool_calls": 107, "tool_calls_by_class": { "cc_builtin": 105, "subagent": 2 }, "tool_result_tokens": 47638, "tool_time_ms": 11507, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.373134149999999, "cost_usd": 6.373134149999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 716782, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 136, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|3", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 16281703, "tokens_cache_write": 161390, "tokens_input": 272, "tokens_output": 37822, "tool_calls": 118, "tool_calls_by_class": { "cc_builtin": 116, "subagent": 2 }, "tool_result_tokens": 51631, "tool_time_ms": 20916, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.526963750000001, "cost_usd": 5.526963750000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 429725, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 157, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|4", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11771475, "tokens_cache_write": 262993, "tokens_input": 2091, "tokens_output": 36869, "tool_calls": 162, "tool_calls_by_class": { "cc_builtin": 154, "subagent": 8 }, "tool_result_tokens": 83914, "tool_time_ms": 10444, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 4.8465018, "cost_usd": 4.8465018, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 676146, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 125, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|5", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12213681, "tokens_cache_write": 130885, "tokens_input": 250, "tokens_output": 28803, "tool_calls": 106, "tool_calls_by_class": { "cc_builtin": 105, "subagent": 1 }, "tool_result_tokens": 37159, "tool_time_ms": 8520, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.1690585, "cost_usd": 5.1690585, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 611528, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 122, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|6", "score": 17, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12968640, "tokens_cache_write": 140138, "tokens_input": 244, "tokens_output": 33032, "tool_calls": 103, "tool_calls_by_class": { "cc_builtin": 101, "subagent": 2 }, "tool_result_tokens": 40002, "tool_time_ms": 10929, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.1936538, "cost_usd": 7.1936538, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 1413210, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 239, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|7", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12485876, "tokens_cache_write": 444714, "tokens_input": 478, "tokens_output": 44380, "tool_calls": 281, "tool_calls_by_class": { "cc_builtin": 243, "subagent": 38 }, "tool_result_tokens": 151257, "tool_time_ms": 34146, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.8990236, "cost_usd": 5.8990236, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 612632, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 137, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 14956057, "tokens_cache_write": 157643, "tokens_input": 399, "tokens_output": 34729, "tool_calls": 119, "tool_calls_by_class": { "cc_builtin": 117, "subagent": 1, "vendor_mcp": 1 }, "tool_result_tokens": 47573, "tool_time_ms": 14419, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.0069124, "cost_usd": 5.0069124, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 663459, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 118, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|9", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12356946, "tokens_cache_write": 140680, "tokens_input": 236, "tokens_output": 34977, "tool_calls": 99, "tool_calls_by_class": { "cc_builtin": 97, "subagent": 2 }, "tool_result_tokens": 36613, "tool_time_ms": 9313, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 5.06123805, "cost_usd": 5.06123805, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 596030, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 111, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|ffc82068051276c8ecade67885ff45af|10", "score": 19, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12182251, "tokens_cache_write": 160565, "tokens_input": 222, "tokens_output": 32539, "tool_calls": 98, "tool_calls_by_class": { "cc_builtin": 96, "subagent": 2 }, "tool_result_tokens": 47250, "tool_time_ms": 21890, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.8661564999999993, "cost_usd": 2.8661564999999993, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 335085, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 95, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6743335, "tokens_cache_write": 100721, "tokens_input": 190, "tokens_output": 15884, "tool_calls": 65, "tool_calls_by_class": { "cc_builtin": 65 }, "tool_result_tokens": 15562, "tool_time_ms": 2748, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.6361219, "cost_usd": 2.6361219, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 428266, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 91, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6837053, "tokens_cache_write": 60250, "tokens_input": 182, "tokens_output": 14864, "tool_calls": 61, "tool_calls_by_class": { "cc_builtin": 61 }, "tool_result_tokens": 17795, "tool_time_ms": 2859, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4507009, "cost_usd": 2.4507009, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 330767, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 88, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6322293, "tokens_cache_write": 54970, "tokens_input": 176, "tokens_output": 14911, "tool_calls": 58, "tool_calls_by_class": { "cc_builtin": 58 }, "tool_result_tokens": 15249, "tool_time_ms": 2897, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.814560399999999, "cost_usd": 2.814560399999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 351140, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 97, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 7300238, "tokens_cache_write": 62312, "tokens_input": 194, "tokens_output": 16669, "tool_calls": 67, "tool_calls_by_class": { "cc_builtin": 67 }, "tool_result_tokens": 18113, "tool_time_ms": 3238, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4834324, "cost_usd": 2.4834324, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 351598, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 92, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6578808, "tokens_cache_write": 50098, "tokens_input": 184, "tokens_output": 13910, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 13236, "tool_time_ms": 2744, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.5937172, "cost_usd": 2.5937172, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 330991, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 94, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6804764, "tokens_cache_write": 56229, "tokens_input": 188, "tokens_output": 14290, "tool_calls": 64, "tool_calls_by_class": { "cc_builtin": 64 }, "tool_result_tokens": 15880, "tool_time_ms": 2702, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.2370484000000004, "cost_usd": 2.2370484000000004, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 345399, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 85, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5825318, "tokens_cache_write": 48268, "tokens_input": 170, "tokens_output": 13289, "tool_calls": 55, "tool_calls_by_class": { "cc_builtin": 55 }, "tool_result_tokens": 12203, "tool_time_ms": 2648, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.1183477, "cost_usd": 2.1183477, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 314593, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 87, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 5658939, "tokens_cache_write": 40509, "tokens_input": 174, "tokens_output": 11806, "tool_calls": 57, "tool_calls_by_class": { "cc_builtin": 57 }, "tool_result_tokens": 8929, "tool_time_ms": 2505, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.4672281999999996, "cost_usd": 2.4672281999999996, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 345242, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 92, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6483044, "tokens_cache_write": 50083, "tokens_input": 184, "tokens_output": 14751, "tool_calls": 62, "tool_calls_by_class": { "cc_builtin": 62 }, "tool_result_tokens": 12604, "tool_time_ms": 2957, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 2.413536299999999, "cost_usd": 2.413536299999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 411415, "exam": "MultiTurnExam", "exam_epoch": null, "fails": 0, "family": "cc_competitor", "llm_calls": 90, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/MultiTurnExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|MultiTurnExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 30, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 6351701, "tokens_cache_write": 52536, "tokens_input": 180, "tokens_output": 12818, "tool_calls": 60, "tool_calls_by_class": { "cc_builtin": 60 }, "tool_result_tokens": 15486, "tool_time_ms": 2641, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.883118800000002, "cost_usd": 8.883118800000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 875644, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 293, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|1", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12583066, "tokens_cache_write": 829482, "tokens_input": 586, "tokens_output": 41370, "tool_calls": 326, "tool_calls_by_class": { "cc_builtin": 303, "subagent": 22, "vendor_mcp": 1 }, "tool_result_tokens": 157024, "tool_time_ms": 72347, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.684678649999999, "cost_usd": 7.684678649999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 596455, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 256, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|2", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 11176563, "tokens_cache_write": 678053, "tokens_input": 512, "tokens_output": 40442, "tool_calls": 262, "tool_calls_by_class": { "cc_builtin": 246, "subagent": 16 }, "tool_result_tokens": 130192, "tool_time_ms": 110026, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.038255350000002, "cost_usd": 8.038255350000002, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 620014, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 8, "family": "cc_competitor", "llm_calls": 273, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|3", "score": 12, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 13097062, "tokens_cache_write": 620166, "tokens_input": 820, "tokens_output": 49944, "tool_calls": 274, "tool_calls_by_class": { "cc_builtin": 260, "subagent": 14 }, "tool_result_tokens": 149347, "tool_time_ms": 18239, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.666507049999999, "cost_usd": 7.666507049999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 682887, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 255, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|4", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12926661, "tokens_cache_write": 549073, "tokens_input": 510, "tokens_output": 61275, "tool_calls": 268, "tool_calls_by_class": { "cc_builtin": 258, "subagent": 10 }, "tool_result_tokens": 140522, "tool_time_ms": 15895, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.545976200000001, "cost_usd": 7.545976200000001, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 887694, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 5, "family": "cc_competitor", "llm_calls": 258, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|5", "score": 15, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12003679, "tokens_cache_write": 617784, "tokens_input": 516, "tokens_output": 48352, "tool_calls": 266, "tool_calls_by_class": { "cc_builtin": 248, "subagent": 15, "vendor_mcp": 3 }, "tool_result_tokens": 138844, "tool_time_ms": 41516, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.19783385, "cost_usd": 7.19783385, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 862829, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 245, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|6", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10673227, "tokens_cache_write": 656869, "tokens_input": 490, "tokens_output": 48036, "tool_calls": 257, "tool_calls_by_class": { "cc_builtin": 238, "subagent": 19 }, "tool_result_tokens": 128650, "tool_time_ms": 17861, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 6.0668982, "cost_usd": 6.0668982, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 788162, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 207, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|7", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9187744, "tokens_cache_write": 522586, "tokens_input": 2066, "tokens_output": 43894, "tool_calls": 226, "tool_calls_by_class": { "cc_builtin": 212, "subagent": 14 }, "tool_result_tokens": 110934, "tool_time_ms": 32679, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 8.059860299999999, "cost_usd": 8.059860299999999, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 690938, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 271, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|8", "score": 16, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12504336, "tokens_cache_write": 663704, "tokens_input": 542, "tokens_output": 59746, "tool_calls": 285, "tool_calls_by_class": { "cc_builtin": 270, "subagent": 15 }, "tool_result_tokens": 159119, "tool_time_ms": 78153, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.774291199999998, "cost_usd": 7.774291199999998, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 734372, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 269, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|9", "score": 18, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12454519, "tokens_cache_write": 597219, "tokens_input": 538, "tokens_output": 53930, "tool_calls": 279, "tool_calls_by_class": { "cc_builtin": 268, "subagent": 9, "vendor_mcp": 2 }, "tool_result_tokens": 127771, "tool_time_ms": 126738, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos-hard", "cost_basis": "vendor_reported", "cost_reported_usd": 7.8041737499999995, "cost_usd": 7.8041737499999995, "denominator": 20, "duration_basis": "sum_of_questions", "duration_ms": 798029, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 6, "family": "cc_competitor", "llm_calls": 262, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos-hard/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|ffc82068051276c8ecade67885ff45af|10", "score": 14, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 12249080, "tokens_cache_write": 627124, "tokens_input": 524, "tokens_output": 50452, "tool_calls": 266, "tool_calls_by_class": { "cc_builtin": 251, "subagent": 15 }, "tool_result_tokens": 138671, "tool_time_ms": 99617, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.602724900000001, "cost_usd": 6.602724900000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 575203, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 3, "family": "cc_competitor", "llm_calls": 217, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep01", "query_calls": 0, "rep": 1, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|1", "score": 27, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9880093, "tokens_cache_write": 586064, "tokens_input": 434, "tokens_output": 40985, "tool_calls": 201, "tool_calls_by_class": { "cc_builtin": 189, "subagent": 12 }, "tool_result_tokens": 102233, "tool_time_ms": 10604, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.2728029, "cost_usd": 6.2728029, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 558687, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 203, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep02", "query_calls": 0, "rep": 2, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|2", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9199023, "tokens_cache_write": 576841, "tokens_input": 406, "tokens_output": 37957, "tool_calls": 197, "tool_calls_by_class": { "cc_builtin": 186, "subagent": 11 }, "tool_result_tokens": 101805, "tool_time_ms": 9583, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.219705299999998, "cost_usd": 6.219705299999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 592191, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 213, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep03", "query_calls": 0, "rep": 3, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|3", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9457346, "tokens_cache_write": 550505, "tokens_input": 426, "tokens_output": 38005, "tool_calls": 209, "tool_calls_by_class": { "cc_builtin": 195, "subagent": 14 }, "tool_result_tokens": 111570, "tool_time_ms": 100854, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.3760998, "cost_usd": 6.3760998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 491884, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 214, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep04", "query_calls": 0, "rep": 4, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|4", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9739636, "tokens_cache_write": 550420, "tokens_input": 3611, "tokens_output": 41811, "tool_calls": 234, "tool_calls_by_class": { "cc_builtin": 221, "subagent": 13 }, "tool_result_tokens": 116842, "tool_time_ms": 10627, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.39815895, "cost_usd": 6.39815895, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 565559, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 217, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep05", "query_calls": 0, "rep": 5, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|5", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9791255, "tokens_cache_write": 556278, "tokens_input": 3499, "tokens_output": 37748, "tool_calls": 226, "tool_calls_by_class": { "cc_builtin": 212, "subagent": 14 }, "tool_result_tokens": 108257, "tool_time_ms": 11588, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.273275849999999, "cost_usd": 6.273275849999999, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 616454, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 210, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep06", "query_calls": 0, "rep": 6, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|6", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 9358669, "tokens_cache_write": 565397, "tokens_input": 2052, "tokens_output": 40825, "tool_calls": 199, "tool_calls_by_class": { "cc_builtin": 186, "subagent": 13 }, "tool_result_tokens": 106909, "tool_time_ms": 10243, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 5.5770129, "cost_usd": 5.5770129, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 508487, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 192, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep07", "query_calls": 0, "rep": 7, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|7", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 8567678, "tokens_cache_write": 493397, "tokens_input": 384, "tokens_output": 33401, "tool_calls": 183, "tool_calls_by_class": { "cc_builtin": 174, "subagent": 9 }, "tool_result_tokens": 97137, "tool_time_ms": 9315, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.704368200000001, "cost_usd": 6.704368200000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 692915, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 1, "family": "cc_competitor", "llm_calls": 227, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep08", "query_calls": 0, "rep": 8, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|8", "score": 29, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10298634, "tokens_cache_write": 565687, "tokens_input": 454, "tokens_output": 40854, "tool_calls": 244, "tool_calls_by_class": { "cc_builtin": 232, "subagent": 12 }, "tool_result_tokens": 109670, "tool_time_ms": 100359, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 7.083340199999998, "cost_usd": 7.083340199999998, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 533005, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 4, "family": "cc_competitor", "llm_calls": 236, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep09", "query_calls": 0, "rep": 9, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|9", "score": 26, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10407649, "tokens_cache_write": 636847, "tokens_input": 472, "tokens_output": 50924, "tool_calls": 236, "tool_calls_by_class": { "cc_builtin": 224, "subagent": 12 }, "tool_result_tokens": 126701, "tool_time_ms": 12984, "validity_ok": true, "validity_reason": "run_ok" }, { "adjudicated": 0, "arm": "serena", "arm_label": "CC+Serena", "battery": "memos", "cost_basis": "vendor_reported", "cost_reported_usd": 6.493612800000001, "cost_usd": 6.493612800000001, "denominator": 30, "duration_basis": "sum_of_questions", "duration_ms": 533887, "exam": "ZeroShotExam", "exam_epoch": null, "fails": 2, "family": "cc_competitor", "llm_calls": 227, "model": "claude-sonnet-5", "path": "runs/serena/claude-sonnet-5/ZeroShotExam/memos/rep10", "query_calls": 0, "rep": 10, "run_key": "ef773dfc921e2e957e945faf5ffc8679|serena|claude-sonnet-5|ZeroShotExam|33ee333a178d3d40bd6a3f99f45d4162|10", "score": 28, "scorer_era": "tabench-1.1", "side": "comparison", "token_source": "transcript_all_threads", "tokens_cache_read": 10095589, "tokens_cache_write": 557456, "tokens_input": 454, "tokens_output": 42956, "tool_calls": 216, "tool_calls_by_class": { "cc_builtin": 203, "subagent": 13 }, "tool_result_tokens": 97303, "tool_time_ms": 10337, "validity_ok": true, "validity_reason": "run_ok" } ]