{"schemaVersion":1,"canonical":"https://codegraff.com/evaluations","title":"Graff evaluations","note":"Twelve published Kimi K3 configurations plus two recorded Graff runs. Graff used extra terminal instructions and a local Docker runtime; its full-suite totals combine the two evaluated slices. Published costs use cost_first_cold_usd; Graff costs use historical list-price summaries. All recorded failures count toward spend. Partial published costs are lower bounds. Frontier badges consider only fully costed published configurations with at least one pass. Median attempt time includes successes and failures where task timings are available; Graff summary maxima are not medians.","entries":[{"id":"codex","name":"Codex","slices":{"all":{"tasks":30,"passes":20,"totalCost":69.3659265,"costedTasks":30,"costPerPass":3.468296325,"medianSeconds":561.86131},"terminal":{"tasks":21,"passes":15,"totalCost":4.8583788,"costedTasks":21,"costPerPass":0.32389192,"medianSeconds":410.738308},"deepswe":{"tasks":9,"passes":5,"totalCost":64.5075477,"costedTasks":9,"costPerPass":12.90150954,"medianSeconds":2587.068742}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"dsh-creator","name":"DSH Creator","slices":{"all":{"tasks":30,"passes":19,"totalCost":62.4131502,"costedTasks":27,"costPerPass":3.284902642105263,"medianSeconds":748.6974585},"terminal":{"tasks":21,"passes":15,"totalCost":3.3748569,"costedTasks":18,"costPerPass":0.22499046,"medianSeconds":452.85957},"deepswe":{"tasks":9,"passes":4,"totalCost":59.0382933,"costedTasks":9,"costPerPass":14.759573325,"medianSeconds":2232.195525}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"claude-code","name":"Claude Code","slices":{"all":{"tasks":30,"passes":19,"totalCost":348.3998442,"costedTasks":29,"costPerPass":18.33683390526316,"medianSeconds":1108.524014},"terminal":{"tasks":21,"passes":16,"totalCost":15.5788197,"costedTasks":21,"costPerPass":0.97367623125,"medianSeconds":577.971591},"deepswe":{"tasks":9,"passes":3,"totalCost":332.8210245,"costedTasks":8,"costPerPass":110.9403415,"medianSeconds":3595.917111}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"pi-responses","name":"Pi","slices":{"all":{"tasks":30,"passes":18,"totalCost":43.793379,"costedTasks":30,"costPerPass":2.4329655,"medianSeconds":638.7449819999999},"terminal":{"tasks":21,"passes":16,"totalCost":5.4599451,"costedTasks":21,"costPerPass":0.34124656875,"medianSeconds":485.996955},"deepswe":{"tasks":9,"passes":2,"totalCost":38.3334339,"costedTasks":9,"costPerPass":19.16671695,"medianSeconds":2350.514284}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"dsh-ptc","name":"DSH PTC","slices":{"all":{"tasks":30,"passes":18,"totalCost":82.39405289999999,"costedTasks":27,"costPerPass":4.5774473833333325,"medianSeconds":534.2175105},"terminal":{"tasks":21,"passes":16,"totalCost":3.0197976,"costedTasks":18,"costPerPass":0.18873735,"medianSeconds":474.941809},"deepswe":{"tasks":9,"passes":2,"totalCost":79.37425529999999,"costedTasks":9,"costPerPass":39.687127649999994,"medianSeconds":3020.700291}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"dsh-standard","name":"DSH Standard","slices":{"all":{"tasks":30,"passes":18,"totalCost":62.259954,"costedTasks":27,"costPerPass":3.4588863333333335,"medianSeconds":592.9845035000001},"terminal":{"tasks":21,"passes":16,"totalCost":8.1456882,"costedTasks":19,"costPerPass":0.5091055125,"medianSeconds":424.040142},"deepswe":{"tasks":9,"passes":2,"totalCost":54.1142658,"costedTasks":8,"costPerPass":27.0571329,"medianSeconds":2835.545963}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"oh-my-pi","name":"Oh My Pi","slices":{"all":{"tasks":30,"passes":17,"totalCost":80.674218,"costedTasks":30,"costPerPass":4.745542235294117,"medianSeconds":595.9183135000001},"terminal":{"tasks":21,"passes":16,"totalCost":5.4427851,"costedTasks":21,"costPerPass":0.34017406875,"medianSeconds":444.418426},"deepswe":{"tasks":9,"passes":1,"totalCost":75.2314329,"costedTasks":9,"costPerPass":75.2314329,"medianSeconds":2997.462833}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"kimi-code","name":"Kimi Code","slices":{"all":{"tasks":30,"passes":17,"totalCost":62.0004801,"costedTasks":30,"costPerPass":3.6470870647058824,"medianSeconds":560.202486},"terminal":{"tasks":21,"passes":14,"totalCost":9.3917685,"costedTasks":21,"costPerPass":0.6708406071428571,"medianSeconds":475.880631},"deepswe":{"tasks":9,"passes":3,"totalCost":52.6087116,"costedTasks":9,"costPerPass":17.5362372,"medianSeconds":2668.571971}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"dsh-minimal","name":"DSH Minimal","slices":{"all":{"tasks":30,"passes":17,"totalCost":80.2052463,"costedTasks":26,"costPerPass":4.717955664705882,"medianSeconds":893.5280545},"terminal":{"tasks":21,"passes":15,"totalCost":6.2994639,"costedTasks":17,"costPerPass":0.41996426,"medianSeconds":414.517787},"deepswe":{"tasks":9,"passes":2,"totalCost":73.90578239999999,"costedTasks":9,"costPerPass":36.9528912,"medianSeconds":2485.273986}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"exo","name":"Exo Harness","slices":{"all":{"tasks":30,"passes":16,"totalCost":16.7227377,"costedTasks":30,"costPerPass":1.04517110625,"medianSeconds":603.5377495},"terminal":{"tasks":21,"passes":16,"totalCost":6.4250046,"costedTasks":21,"costPerPass":0.4015627875,"medianSeconds":411.090475},"deepswe":{"tasks":9,"passes":0,"totalCost":10.2977331,"costedTasks":9,"costPerPass":null,"medianSeconds":954.5815}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"opencode","name":"OpenCode","slices":{"all":{"tasks":30,"passes":15,"totalCost":48.6651828,"costedTasks":29,"costPerPass":3.24434552,"medianSeconds":701.358377},"terminal":{"tasks":21,"passes":15,"totalCost":3.4770654,"costedTasks":21,"costPerPass":0.23180436,"medianSeconds":463.111223},"deepswe":{"tasks":9,"passes":0,"totalCost":45.1881174,"costedTasks":8,"costPerPass":null,"medianSeconds":2944.994884}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"hermes","name":"Hermes","slices":{"all":{"tasks":30,"passes":15,"totalCost":43.5645522,"costedTasks":28,"costPerPass":2.9043034800000003,"medianSeconds":785.6614105000001},"terminal":{"tasks":21,"passes":13,"totalCost":11.3271525,"costedTasks":21,"costPerPass":0.871319423076923,"medianSeconds":547.60882},"deepswe":{"tasks":9,"passes":2,"totalCost":32.2373997,"costedTasks":7,"costPerPass":16.11869985,"medianSeconds":1728.459216}},"cohort":"published","condition":"Published Kimi K3 board; shared runtime and instructions."},{"id":"graff-grok","name":"Graff / Grok 4.6","cohort":"graff","condition":"Recorded Graff evaluation; extra terminal instructions and local Docker runtime. Full-suite totals combine the terminal and DeepSWE slices. Historical evaluator list prices. Excluded from the published-board frontier. Median attempt time is not reported in the stored summaries.","slices":{"all":{"tasks":30,"passes":21,"totalCost":31.7951,"costedTasks":30,"costPerPass":1.514052380952381,"medianSeconds":null},"terminal":{"tasks":21,"passes":20,"totalCost":6.2335,"costedTasks":21,"costPerPass":0.31167500000000004,"medianSeconds":null},"deepswe":{"tasks":9,"passes":1,"totalCost":25.5616,"costedTasks":9,"costPerPass":25.5616,"medianSeconds":null}}},{"id":"graff-kimi","name":"Graff / Kimi K3","cohort":"graff","condition":"Recorded Graff evaluation; extra terminal instructions and local Docker runtime. Full-suite totals combine the terminal and DeepSWE slices. Historical evaluator list prices. Excluded from the published-board frontier. Median attempt time is not reported in the stored summaries.","slices":{"all":{"tasks":30,"passes":18,"totalCost":25.561148,"costedTasks":30,"costPerPass":1.4200637777777778,"medianSeconds":null},"terminal":{"tasks":21,"passes":17,"totalCost":3.6463,"costedTasks":21,"costPerPass":0.21448823529411765,"medianSeconds":null},"deepswe":{"tasks":9,"passes":1,"totalCost":21.914848,"costedTasks":9,"costPerPass":21.914848,"medianSeconds":null}}}],"description":"Every Graff coding-agent evaluation: same-model runs against Devin, Codex, Claude Code, OpenCode and Pi, release and model comparisons, FrontierHarness terminal tasks, DeepSWE patch verification, and a tuner for your own team.","url":"https://codegraff.com/evaluations","publishedAt":"2026-09-12","articlePublishedAt":"2026-09-06","updatedAt":"2026-10-04","articleUrl":"https://codegraff.com/blog/graff-frontier-harness-evals","sourceNote":"Results are repository-reported evaluations checked against the tracked summaries and protocol at revision b4bd80c, not a fresh rerun or an independent audit. The Graff terminal runs shown here used additional evaluation instructions. Models and execution environments differ across some rows. Published Exo costs are taken from FrontierHarness revision 8f11b13; the terminal subtotal is calculated from its public task-level records.","sources":[{"id":"results","title":"FrontierHarness results and suite definitions","url":"https://github.com/justrach/codegraff/blob/b4bd80c0dbf4332f249b4a1a36fbeb136ed65611/graff-evals/frontier-harness/README.md"},{"id":"protocol","title":"Evaluation protocol and comparison limitations","url":"https://github.com/justrach/codegraff/blob/b4bd80c0dbf4332f249b4a1a36fbeb136ed65611/graff-evals/frontier-harness/PROTOCOL.md"},{"id":"grok-summary","title":"Graff / Grok terminal result summary","url":"https://github.com/justrach/codegraff/blob/b4bd80c0dbf4332f249b4a1a36fbeb136ed65611/graff-evals/frontier-harness/results.maxima.json"},{"id":"kimi-summary","title":"Graff / Kimi K3 terminal result summary","url":"https://github.com/justrach/codegraff/blob/b4bd80c0dbf4332f249b4a1a36fbeb136ed65611/graff-evals/frontier-harness/kimi-results.maxima.json"},{"id":"published-board","title":"Published FrontierHarness task costs and Exo results","url":"https://github.com/frontier-harness-eval/eval/blob/8f11b130c30bbf76ca1f3edeea70abc773bd8d2c/results/eval-data.json"},{"id":"board-cost-method","title":"Published cost accounting and comparison method","url":"https://github.com/frontier-harness-eval/eval/blob/8f11b130c30bbf76ca1f3edeea70abc773bd8d2c/skills/frontierharness-eval/reference.md#cost-comparability"},{"id":"grok-swe-summary","title":"Graff / Grok DeepSWE cost summary","url":"https://github.com/justrach/codegraff/blob/b4bd80c0dbf4332f249b4a1a36fbeb136ed65611/graff-evals/frontier-harness/swe-results.maxima.json"},{"id":"kimi-swe-summary","title":"Graff / Kimi K3 DeepSWE cost summary","url":"https://github.com/justrach/codegraff/blob/b4bd80c0dbf4332f249b4a1a36fbeb136ed65611/graff-evals/frontier-harness/swe-kimi-results.maxima.json"}],"publishedEvals":{"headToHead":[{"slug":"graff-vs-devin","date":"Oct 4, 2026","title":"graff and Devin CLI on the same model","model":"gpt-6.1-sol, medium effort","setup":"7 coding tasks × 3 runs, interleaved","bars":[{"label":"graff v0.0.302.21","value":46,"ours":true},{"label":"Devin CLI","value":128.2}],"unit":"s","takeaway":"2.8× faster and 5.7× cheaper per task ($0.0249 against $0.1416 list). Devin passed one more run: 19/21 against 18/21."},{"slug":"graff-shorter-turns","date":"Oct 3, 2026","title":"Shorter turns against Pi's codemode","model":"gpt-6.1-sol","setup":"11 tasks × 3 runs, every run passed","bars":[{"label":"graff v0.0.302.18","value":33,"ours":true},{"label":"Pi 1.0 + codemode","value":34.2},{"label":"graff v0.0.302.17","value":40.8}],"unit":"s","takeaway":"32% lower list cost per task than Pi with codemode. On gpt-6-astra Pi stays 0.7 s faster on the mean; graff is 21% cheaper."},{"slug":"graff-rlm-vs-pi-codemode","date":"Oct 2, 2026","title":"Pi 1.0's codemode against graff's rlm","model":"gpt-6-astra","setup":"11 tasks × 3 runs per setup","bars":[{"label":"Pi + codemode","value":41.7},{"label":"Pi","value":48.1},{"label":"graff v0.0.302.17","value":48.3,"ours":true},{"label":"graff v0.0.302.16","value":54.1}],"unit":"s","takeaway":"Pi with codemode was the fastest of five setups here. The rlm change in v0.0.302.17 cut MCP join tasks by 27%, and v0.0.302.18 then overtook Pi."},{"slug":"graff-vs-codex-and-claude-code","date":"Oct 1, 2026","title":"graff against the Codex app server","model":"gpt-6.1-sol, same ChatGPT account","setup":"21 tasks × 3 runs, every run passed","bars":[{"label":"graff","value":22.2,"ours":true},{"label":"Codex app server","value":28.6}],"unit":"s","takeaway":"22% less time and 69% lower list cost per task ($0.0131 against $0.0426)."},{"slug":"graff-vs-codex-and-claude-code","date":"Oct 1, 2026","title":"graff against Claude Code and OpenCode 2","model":"one gateway model","setup":"21 tasks × 3 runs, every run passed","bars":[{"label":"graff","value":8.7,"ours":true},{"label":"Claude Code","value":13.2},{"label":"OpenCode 2","value":14.1}],"unit":"s","takeaway":"Half the time on MCP work and 70% fewer prompt tokens than Claude Code; a little slower on the small coding tasks (7.0 s against 6.0 s)."}],"releases":[{"slug":"graff-0-0-302-16","date":"Oct 2, 2026","title":"DeepSeek gets to work sooner","model":"DeepSeek V4 Pro","setup":"21 coding and MCP tasks × 2 runs","bars":[{"label":"v0.0.302.15","value":15.9,"ours":true},{"label":"v0.0.302.14","value":21.6}],"unit":"s","takeaway":"26% less time per task, 39% less on sub-agent tasks, 36% fewer output tokens."},{"slug":"grok-4-6-vs-4-7","date":"Sep 22, 2026","title":"Grok 4.6 against Grok 4.7 in graff","model":"Grok 4.6 / 4.7","setup":"3 real pull requests, 24-point checklist","bars":[{"label":"Grok 4.6","value":507},{"label":"Grok 4.7","value":1351}],"unit":"s","measure":"wall clock for all three PRs","takeaway":"4.7 scored one checklist point higher (21/24) for 2.3× the cost and 3.4× the reasoning tokens."},{"slug":"graff-rlm-token-efficiency","date":"Aug 28, 2026","title":"The rlm runtime, before and after","model":"one task, reused binds","setup":"same task, rlm off then on","bars":[{"label":"With rlm","value":39.2,"ours":true},{"label":"Before","value":99.6}],"unit":"s","measure":"seconds for the task","takeaway":"Input tokens 135k → 25.3k and model calls 11 → 5 on the same task."}]}}