fix: alfworld benchmark graph (#554)

Co-authored-by: dalongbao <v-tinyantsui@microsoft.com>
Co-authored-by: Yuge Zhang <scottyugochang@gmail.com>
This commit is contained in:
dalongbao
2026-08-24 05:39:59 +02:00
committed by GitHub
parent 2bd0875fd3
commit 6f44dcbf68
2 changed files with 310 additions and 245 deletions
+1 -1
View File
@@ -52,7 +52,7 @@ Claude Code uses Claude Opus 4.8; Codex and GitHub Copilot use GPT 5.6 Sol as th
#### Performance versus finale cost
The selected-budget views use the groups with the strongest aggregate skill-over-control lift: \$5 for SpreadsheetBench and \$10 for OfficeQA and ALFWorld. Every harness/treatment point is one of three runs; the x-axis is that run's finale cost, and the y-axis is held-out SpreadsheetBench accuracy, OfficeQA correctness, or ALFWorld success. Finale cost measures LLM gateway spend, so an ALFWorld deterministic controller can have exactly \$0 finale cost while still executing and scoring real environment steps; coincident zero-cost ALFWorld results are offset slightly along the x-axis so each replicate remains visible. SpreadsheetBench and OfficeQA show their aggregate pristine-baseline results as single reference points. The dotted ALFWorld baseline is a score-only reference: the corrected records do not include baseline deployment cost, so assigning it an x-coordinate would invent data.
The selected-budget views use the groups with the strongest aggregate skill-over-control lift: \$5 for SpreadsheetBench and \$10 for OfficeQA and ALFWorld. Every harness/treatment point is one of three runs; the x-axis is that run's finale cost, and the y-axis is held-out SpreadsheetBench accuracy, OfficeQA correctness, or ALFWorld success. Finale cost measures LLM gateway spend, so an ALFWorld deterministic controller can have exactly \$0 finale cost while still executing and scoring real environment steps; coincident zero-cost ALFWorld results are offset slightly along the x-axis so each replicate remains visible. Each chart shows its aggregate pristine-baseline result as a single reference point.
![SpreadsheetBench accuracy versus finale cost](assets/agent-lightning-spreadsheetbench-accuracy-finale-cost.svg)
@@ -6,7 +6,7 @@
<rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<cc:Work>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
<dc:date>2026-08-11T20:51:34.507463</dc:date>
<dc:date>2026-08-21T18:53:08.194566</dc:date>
<dc:format>image/svg+xml</dc:format>
<dc:creator>
<cc:Agent>
@@ -40,23 +40,23 @@ z
<g id="matplotlib.axis_1">
<g id="xtick_1">
<g id="line2d_1">
<path d="M 94.994577 371.448
L 94.994577 48.24
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
<path d="M 88.974307 371.448
L 88.974307 48.24
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_2">
<defs>
<path id="ma6e64deb88" d="M 0 0
<path id="mbe2e6ba5df" d="M 0 0
L 0 3.5
" style="stroke: #000000; stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#ma6e64deb88" x="94.994577" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#mbe2e6ba5df" x="88.974307" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_1">
<!-- 0 -->
<g transform="translate(91.495202 386.805422) scale(0.11 -0.11)">
<g transform="translate(85.474932 386.805422) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-13" d="M 2034 4250
Q 1547 4250 1301 3770
@@ -86,18 +86,18 @@ z
</g>
<g id="xtick_2">
<g id="line2d_3">
<path d="M 210.98629 371.448
L 210.98629 48.24
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
<path d="M 177.601156 371.448
L 177.601156 48.24
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_4">
<g>
<use xlink:href="#ma6e64deb88" x="210.98629" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#mbe2e6ba5df" x="177.601156" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_2">
<!-- 1 -->
<g transform="translate(207.486915 386.805422) scale(0.11 -0.11)">
<g transform="translate(174.101781 386.805422) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-14" d="M 794 531
L 1825 531
@@ -120,18 +120,18 @@ z
</g>
<g id="xtick_3">
<g id="line2d_5">
<path d="M 326.978003 371.448
L 326.978003 48.24
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
<path d="M 266.228005 371.448
L 266.228005 48.24
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_6">
<g>
<use xlink:href="#ma6e64deb88" x="326.978003" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#mbe2e6ba5df" x="266.228005" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_3">
<!-- 2 -->
<g transform="translate(323.478628 386.805422) scale(0.11 -0.11)">
<g transform="translate(262.72863 386.805422) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-15" d="M 1228 531
L 3431 531
@@ -164,18 +164,18 @@ z
</g>
<g id="xtick_4">
<g id="line2d_7">
<path d="M 442.969716 371.448
L 442.969716 48.24
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
<path d="M 354.854855 371.448
L 354.854855 48.24
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_8">
<g>
<use xlink:href="#ma6e64deb88" x="442.969716" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#mbe2e6ba5df" x="354.854855" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_4">
<!-- 3 -->
<g transform="translate(439.470341 386.805422) scale(0.11 -0.11)">
<g transform="translate(351.35548 386.805422) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-16" d="M 2597 2516
Q 3050 2419 3304 2112
@@ -216,18 +216,18 @@ z
</g>
<g id="xtick_5">
<g id="line2d_9">
<path d="M 558.961429 371.448
L 558.961429 48.24
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
<path d="M 443.481704 371.448
L 443.481704 48.24
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_10">
<g>
<use xlink:href="#ma6e64deb88" x="558.961429" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#mbe2e6ba5df" x="443.481704" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_5">
<!-- 4 -->
<g transform="translate(555.462054 386.805422) scale(0.11 -0.11)">
<g transform="translate(439.982329 386.805422) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-17" d="M 2419 4116
L 825 1625
@@ -253,9 +253,104 @@ z
</g>
</g>
</g>
<g id="text_6">
<g id="xtick_6">
<g id="line2d_11">
<path d="M 532.108553 371.448
L 532.108553 48.24
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_12">
<g>
<use xlink:href="#mbe2e6ba5df" x="532.108553" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_6">
<!-- 5 -->
<g transform="translate(528.609178 386.805422) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-18" d="M 691 4666
L 3169 4666
L 3169 4134
L 1269 4134
L 1269 2991
Q 1406 3038 1543 3061
Q 1681 3084 1819 3084
Q 2600 3084 3056 2656
Q 3513 2228 3513 1497
Q 3513 744 3044 326
Q 2575 -91 1722 -91
Q 1428 -91 1123 -41
Q 819 9 494 109
L 494 744
Q 775 591 1075 516
Q 1375 441 1709 441
Q 2250 441 2565 725
Q 2881 1009 2881 1497
Q 2881 1984 2565 2268
Q 2250 2553 1709 2553
Q 1456 2553 1204 2497
Q 953 2441 691 2322
L 691 4666
z
" transform="scale(0.015625)"/>
</defs>
<use xlink:href="#DejaVuSans-18"/>
</g>
</g>
</g>
<g id="xtick_7">
<g id="line2d_13">
<path d="M 620.735403 371.448
L 620.735403 48.24
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_14">
<g>
<use xlink:href="#mbe2e6ba5df" x="620.735403" y="371.448" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_7">
<!-- 6 -->
<g transform="translate(617.236028 386.805422) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-19" d="M 2113 2584
Q 1688 2584 1439 2293
Q 1191 2003 1191 1497
Q 1191 994 1439 701
Q 1688 409 2113 409
Q 2538 409 2786 701
Q 3034 994 3034 1497
Q 3034 2003 2786 2293
Q 2538 2584 2113 2584
z
M 3366 4563
L 3366 3988
Q 3128 4100 2886 4159
Q 2644 4219 2406 4219
Q 1781 4219 1451 3797
Q 1122 3375 1075 2522
Q 1259 2794 1537 2939
Q 1816 3084 2150 3084
Q 2853 3084 3261 2657
Q 3669 2231 3669 1497
Q 3669 778 3244 343
Q 2819 -91 2113 -91
Q 1303 -91 875 529
Q 447 1150 447 2328
Q 447 3434 972 4092
Q 1497 4750 2381 4750
Q 2619 4750 2861 4703
Q 3103 4656 3366 4563
z
" transform="scale(0.015625)"/>
</defs>
<use xlink:href="#DejaVuSans-19"/>
</g>
</g>
</g>
<g id="text_8">
<!-- Finale cost (USD) -->
<g transform="translate(305.058881 403.325969) scale(0.13 -0.13)">
<g transform="translate(305.058881 409.325969) scale(0.13 -0.13)">
<defs>
<path id="DejaVuSans-29" d="M 628 4666
L 3309 4666
@@ -578,22 +673,22 @@ z
</g>
<g id="matplotlib.axis_2">
<g id="ytick_1">
<g id="line2d_11">
<g id="line2d_15">
<path d="M 69.4764 359.477333
L 651.7548 359.477333
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_12">
<g id="line2d_16">
<defs>
<path id="m17999605ca" d="M 0 0
<path id="m593a0d5656" d="M 0 0
L -3.5 0
" style="stroke: #000000; stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#m17999605ca" x="69.4764" y="359.477333" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#m593a0d5656" x="69.4764" y="359.477333" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_7">
<g id="text_9">
<!-- 0 -->
<g transform="translate(55.47765 363.656044) scale(0.11 -0.11)">
<use xlink:href="#DejaVuSans-13"/>
@@ -601,17 +696,17 @@ L -3.5 0
</g>
</g>
<g id="ytick_2">
<g id="line2d_13">
<g id="line2d_17">
<path d="M 69.4764 299.624
L 651.7548 299.624
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_14">
<g id="line2d_18">
<g>
<use xlink:href="#m17999605ca" x="69.4764" y="299.624" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#m593a0d5656" x="69.4764" y="299.624" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_8">
<g id="text_10">
<!-- 20 -->
<g transform="translate(48.4789 303.802711) scale(0.11 -0.11)">
<use xlink:href="#DejaVuSans-15"/>
@@ -620,17 +715,17 @@ L 651.7548 299.624
</g>
</g>
<g id="ytick_3">
<g id="line2d_15">
<g id="line2d_19">
<path d="M 69.4764 239.770667
L 651.7548 239.770667
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_16">
<g id="line2d_20">
<g>
<use xlink:href="#m17999605ca" x="69.4764" y="239.770667" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#m593a0d5656" x="69.4764" y="239.770667" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_9">
<g id="text_11">
<!-- 40 -->
<g transform="translate(48.4789 243.949378) scale(0.11 -0.11)">
<use xlink:href="#DejaVuSans-17"/>
@@ -639,68 +734,36 @@ L 651.7548 239.770667
</g>
</g>
<g id="ytick_4">
<g id="line2d_17">
<g id="line2d_21">
<path d="M 69.4764 179.917333
L 651.7548 179.917333
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_18">
<g id="line2d_22">
<g>
<use xlink:href="#m17999605ca" x="69.4764" y="179.917333" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#m593a0d5656" x="69.4764" y="179.917333" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_10">
<g id="text_12">
<!-- 60 -->
<g transform="translate(48.4789 184.096044) scale(0.11 -0.11)">
<defs>
<path id="DejaVuSans-19" d="M 2113 2584
Q 1688 2584 1439 2293
Q 1191 2003 1191 1497
Q 1191 994 1439 701
Q 1688 409 2113 409
Q 2538 409 2786 701
Q 3034 994 3034 1497
Q 3034 2003 2786 2293
Q 2538 2584 2113 2584
z
M 3366 4563
L 3366 3988
Q 3128 4100 2886 4159
Q 2644 4219 2406 4219
Q 1781 4219 1451 3797
Q 1122 3375 1075 2522
Q 1259 2794 1537 2939
Q 1816 3084 2150 3084
Q 2853 3084 3261 2657
Q 3669 2231 3669 1497
Q 3669 778 3244 343
Q 2819 -91 2113 -91
Q 1303 -91 875 529
Q 447 1150 447 2328
Q 447 3434 972 4092
Q 1497 4750 2381 4750
Q 2619 4750 2861 4703
Q 3103 4656 3366 4563
z
" transform="scale(0.015625)"/>
</defs>
<use xlink:href="#DejaVuSans-19"/>
<use xlink:href="#DejaVuSans-13" transform="translate(63.625 0)"/>
</g>
</g>
</g>
<g id="ytick_5">
<g id="line2d_19">
<g id="line2d_23">
<path d="M 69.4764 120.064
L 651.7548 120.064
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_20">
<g id="line2d_24">
<g>
<use xlink:href="#m17999605ca" x="69.4764" y="120.064" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#m593a0d5656" x="69.4764" y="120.064" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_11">
<g id="text_13">
<!-- 80 -->
<g transform="translate(48.4789 124.242711) scale(0.11 -0.11)">
<defs>
@@ -750,17 +813,17 @@ z
</g>
</g>
<g id="ytick_6">
<g id="line2d_21">
<g id="line2d_25">
<path d="M 69.4764 60.210667
L 651.7548 60.210667
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
" clip-path="url(#p8b1873d79b)" style="fill: none; stroke: #cbd5e1; stroke-opacity: 0.65; stroke-width: 0.7; stroke-linecap: square"/>
</g>
<g id="line2d_22">
<g id="line2d_26">
<g>
<use xlink:href="#m17999605ca" x="69.4764" y="60.210667" style="stroke: #000000; stroke-width: 0.8"/>
<use xlink:href="#m593a0d5656" x="69.4764" y="60.210667" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_12">
<g id="text_14">
<!-- 100 -->
<g transform="translate(41.48015 64.389378) scale(0.11 -0.11)">
<use xlink:href="#DejaVuSans-14"/>
@@ -769,9 +832,9 @@ L 651.7548 60.210667
</g>
</g>
</g>
<g id="text_13">
<g id="text_15">
<!-- Success (%) -->
<g transform="translate(34.357103 249.323375) rotate(-90) scale(0.13 -0.13)">
<g transform="translate(35.357103 249.323375) rotate(-90) scale(0.13 -0.13)">
<defs>
<path id="DejaVuSans-58" d="M 544 1381
L 544 3500
@@ -857,60 +920,6 @@ z
</g>
</g>
</g>
<g id="line2d_23">
<path d="M 69.4764 188.999556
L 651.7548 188.999556
" clip-path="url(#p5f2d4442e1)" style="fill: none; stroke-dasharray: 6,4.5; stroke-dashoffset: 0; stroke: #64748b; stroke-width: 1.5"/>
</g>
<g id="PathCollection_1">
<defs>
<path id="mca781f3804" d="M 0 4.527693
C 1.200758 4.527693 2.352498 4.050626 3.201562 3.201562
C 4.050626 2.352498 4.527693 1.200758 4.527693 0
C 4.527693 -1.200758 4.050626 -2.352498 3.201562 -3.201562
C 2.352498 -4.050626 1.200758 -4.527693 0 -4.527693
C -1.200758 -4.527693 -2.352498 -4.050626 -3.201562 -3.201562
C -4.050626 -2.352498 -4.527693 -1.200758 -4.527693 0
C -4.527693 1.200758 -4.050626 2.352498 -3.201562 3.201562
C -2.352498 4.050626 -1.200758 4.527693 0 4.527693
z
" style="stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p5f2d4442e1)">
<use xlink:href="#mca781f3804" x="568.995438" y="80.310667" style="fill: #e76f00; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#mca781f3804" x="524.343111" y="78.077333" style="fill: #e76f00; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#mca781f3804" x="450.97993" y="78.077333" style="fill: #e76f00; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="PathCollection_2">
<defs>
<path id="m4f4a5673ce" d="M -4.527693 4.527693
L 4.527693 4.527693
L 4.527693 -4.527693
L -4.527693 -4.527693
z
" style="stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p5f2d4442e1)">
<use xlink:href="#m4f4a5673ce" x="89.029289" y="60.210667" style="fill: #059669; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m4f4a5673ce" x="353.762288" y="64.677333" style="fill: #059669; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m4f4a5673ce" x="98.001952" y="64.677333" style="fill: #059669; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="PathCollection_3">
<defs>
<path id="m610e4f79c8" d="M 0 -4.527693
L -4.527693 4.527693
L 4.527693 4.527693
z
" style="stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p5f2d4442e1)">
<use xlink:href="#m610e4f79c8" x="108.913582" y="60.210667" style="fill: #2563eb; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m610e4f79c8" x="104.936724" y="60.210667" style="fill: #2563eb; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m610e4f79c8" x="100.959865" y="60.210667" style="fill: #2563eb; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="patch_3">
<path d="M 69.4764 371.448
L 69.4764 48.24
@@ -921,9 +930,9 @@ L 69.4764 48.24
L 651.7548 371.448
" style="fill: none; stroke: #64748b; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="PathCollection_4">
<g id="PathCollection_1">
<defs>
<path id="mba462c6a78" d="M 0 4.527693
<path id="m0f0e21a0da" d="M 0 4.527693
C 1.200758 4.527693 2.352498 4.050626 3.201562 3.201562
C 4.050626 2.352498 4.527693 1.200758 4.527693 0
C 4.527693 -1.200758 4.050626 -2.352498 3.201562 -3.201562
@@ -935,89 +944,91 @@ C -2.352498 4.050626 -1.200758 4.527693 0 4.527693
z
" style="stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p5f2d4442e1)">
<use xlink:href="#mba462c6a78" x="535.923706" y="75.844" style="fill: #ffffff; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#mba462c6a78" x="616.230627" y="87.010667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#mba462c6a78" x="408.368199" y="69.144" style="fill: #ffffff; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<g clip-path="url(#p8b1873d79b)">
<use xlink:href="#m0f0e21a0da" x="451.148481" y="80.310667" style="fill: #e76f00; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m0f0e21a0da" x="417.030569" y="78.077333" style="fill: #e76f00; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m0f0e21a0da" x="360.975292" y="78.077333" style="fill: #e76f00; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="PathCollection_5">
<g id="PathCollection_2">
<defs>
<path id="me684d9ac2f" d="M -4.527693 4.527693
<path id="m2ba6372aba" d="M -4.527693 4.527693
L 4.527693 4.527693
L 4.527693 -4.527693
L -4.527693 -4.527693
z
" style="stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p5f2d4442e1)">
<use xlink:href="#me684d9ac2f" x="85.05243" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#me684d9ac2f" x="94.994577" y="154.010667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#me684d9ac2f" x="81.075571" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<g clip-path="url(#p8b1873d79b)">
<use xlink:href="#m2ba6372aba" x="84.416228" y="60.210667" style="fill: #059669; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m2ba6372aba" x="286.693319" y="64.677333" style="fill: #059669; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m2ba6372aba" x="91.272401" y="64.677333" style="fill: #059669; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="PathCollection_6">
<g id="PathCollection_3">
<defs>
<path id="m465a713e5e" d="M 0 -4.527693
<path id="m07a2c818b9" d="M 0 -4.527693
L -4.527693 4.527693
L 4.527693 4.527693
z
" style="stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p5f2d4442e1)">
<use xlink:href="#m465a713e5e" x="96.983006" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m465a713e5e" x="94.994577" y="357.244" style="fill: #ffffff; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m465a713e5e" x="93.006148" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<g clip-path="url(#p8b1873d79b)">
<use xlink:href="#m07a2c818b9" x="99.609529" y="60.210667" style="fill: #2563eb; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m07a2c818b9" x="96.569628" y="60.210667" style="fill: #2563eb; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m07a2c818b9" x="93.532386" y="60.210667" style="fill: #2563eb; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="text_14">
<!-- Baseline -->
<g style="fill: #475569" transform="translate(588.477854 183.006012) scale(0.1 -0.1)">
<defs>
<path id="DejaVuSans-25" d="M 1259 2228
L 1259 519
L 2272 519
Q 2781 519 3026 730
Q 3272 941 3272 1375
Q 3272 1813 3026 2020
Q 2781 2228 2272 2228
L 1259 2228
<g id="PathCollection_4">
<defs>
<path id="m952893739d" d="M 0 4.527693
C 1.200758 4.527693 2.352498 4.050626 3.201562 3.201562
C 4.050626 2.352498 4.527693 1.200758 4.527693 0
C 4.527693 -1.200758 4.050626 -2.352498 3.201562 -3.201562
C 2.352498 -4.050626 1.200758 -4.527693 0 -4.527693
C -1.200758 -4.527693 -2.352498 -4.050626 -3.201562 -3.201562
C -4.050626 -2.352498 -4.527693 -1.200758 -4.527693 0
C -4.527693 1.200758 -4.050626 2.352498 -3.201562 3.201562
C -2.352498 4.050626 -1.200758 4.527693 0 4.527693
z
M 1259 4147
L 1259 2741
L 2194 2741
Q 2656 2741 2882 2914
Q 3109 3088 3109 3444
Q 3109 3797 2882 3972
Q 2656 4147 2194 4147
L 1259 4147
z
M 628 4666
L 2241 4666
Q 2963 4666 3353 4366
Q 3744 4066 3744 3513
Q 3744 3084 3544 2831
Q 3344 2578 2956 2516
Q 3422 2416 3680 2098
Q 3938 1781 3938 1306
Q 3938 681 3513 340
Q 3088 0 2303 0
L 628 0
L 628 4666
z
" transform="scale(0.015625)"/>
</defs>
<use xlink:href="#DejaVuSans-25"/>
<use xlink:href="#DejaVuSans-44" transform="translate(68.609375 0)"/>
<use xlink:href="#DejaVuSans-56" transform="translate(129.890625 0)"/>
<use xlink:href="#DejaVuSans-48" transform="translate(181.984375 0)"/>
<use xlink:href="#DejaVuSans-4f" transform="translate(243.515625 0)"/>
<use xlink:href="#DejaVuSans-4c" transform="translate(271.296875 0)"/>
<use xlink:href="#DejaVuSans-51" transform="translate(299.078125 0)"/>
<use xlink:href="#DejaVuSans-48" transform="translate(362.453125 0)"/>
" style="stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p8b1873d79b)">
<use xlink:href="#m952893739d" x="425.87906" y="75.844" style="fill: #ffffff; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m952893739d" x="487.239904" y="87.010667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m952893739d" x="328.416557" y="69.144" style="fill: #ffffff; fill-opacity: 0.95; stroke: #e76f00; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="text_15">
<g id="PathCollection_5">
<defs>
<path id="mb439ac2c4e" d="M -4.527693 4.527693
L 4.527693 4.527693
L 4.527693 -4.527693
L -4.527693 -4.527693
z
" style="stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p8b1873d79b)">
<use xlink:href="#mb439ac2c4e" x="81.378986" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#mb439ac2c4e" x="88.974307" y="154.010667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#mb439ac2c4e" x="78.339085" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #059669; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="PathCollection_6">
<defs>
<path id="m4a3ab4d468" d="M 0 -4.527693
L -4.527693 4.527693
L 4.527693 4.527693
z
" style="stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
</defs>
<g clip-path="url(#p8b1873d79b)">
<use xlink:href="#m4a3ab4d468" x="90.494257" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m4a3ab4d468" x="88.974307" y="357.244" style="fill: #ffffff; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
<use xlink:href="#m4a3ab4d468" x="87.454356" y="60.210667" style="fill: #ffffff; fill-opacity: 0.95; stroke: #2563eb; stroke-opacity: 0.95; stroke-width: 1.7"/>
</g>
</g>
<g id="text_16">
<!-- ALFWorld: success versus finale cost ($10 budget) -->
<g transform="translate(133.722787 36.24) scale(0.18 -0.18)">
<defs>
@@ -1311,10 +1322,23 @@ z
<use xlink:href="#DejaVuSans-c" transform="translate(2482.015625 0)"/>
</g>
</g>
<g id="PathCollection_7">
<defs>
<path id="mc062e6b8b6" d="M -0 6.403124
L 6.403124 0
L 0 -6.403124
L -6.403124 -0
z
" style="stroke: #64748b; stroke-width: 1.4"/>
</defs>
<g clip-path="url(#p8b1873d79b)">
<use xlink:href="#mc062e6b8b6" x="607.296902" y="189.744" style="fill: #64748b; stroke: #64748b; stroke-width: 1.4"/>
</g>
</g>
<g id="legend_1">
<g id="line2d_24">
<g id="line2d_27">
<defs>
<path id="me164640a29" d="M 0 4
<path id="m857380d3c7" d="M 0 4
C 1.060812 4 2.078319 3.578535 2.828427 2.828427
C 3.578535 2.078319 4 1.060812 4 0
C 4 -1.060812 3.578535 -2.078319 2.828427 -2.828427
@@ -1327,12 +1351,12 @@ z
" style="stroke: #e76f00"/>
</defs>
<g>
<use xlink:href="#me164640a29" x="212.654662" y="442.955958" style="fill: #e76f00; stroke: #e76f00"/>
<use xlink:href="#m857380d3c7" x="200.154662" y="445.955958" style="fill: #e76f00; stroke: #e76f00"/>
</g>
</g>
<g id="text_16">
<g id="text_17">
<!-- Claude Code -->
<g transform="translate(226.654662 446.455958) scale(0.1 -0.1)">
<g transform="translate(217.154662 449.455958) scale(0.1 -0.1)">
<defs>
<path id="DejaVuSans-26" d="M 4122 4306
L 4122 3641
@@ -1369,9 +1393,9 @@ z
<use xlink:href="#DejaVuSans-48" transform="translate(573.5625 0)"/>
</g>
</g>
<g id="line2d_25">
<g id="line2d_28">
<defs>
<path id="mbd47da48d8" d="M -4 4
<path id="m7703c09d68" d="M -4 4
L 4 4
L 4 -4
L -4 -4
@@ -1379,12 +1403,12 @@ z
" style="stroke: #059669; stroke-linejoin: miter"/>
</defs>
<g>
<use xlink:href="#mbd47da48d8" x="212.654662" y="459.456739" style="fill: #059669; stroke: #059669; stroke-linejoin: miter"/>
<use xlink:href="#m7703c09d68" x="200.154662" y="460.956739" style="fill: #059669; stroke: #059669; stroke-linejoin: miter"/>
</g>
</g>
<g id="text_17">
<g id="text_18">
<!-- Codex -->
<g transform="translate(226.654662 462.956739) scale(0.1 -0.1)">
<g transform="translate(217.154662 464.456739) scale(0.1 -0.1)">
<defs>
<path id="DejaVuSans-5b" d="M 3513 3500
L 2247 1797
@@ -1409,21 +1433,21 @@ z
<use xlink:href="#DejaVuSans-5b" transform="translate(254.28125 0)"/>
</g>
</g>
<g id="line2d_26">
<g id="line2d_29">
<defs>
<path id="m44164aeb4a" d="M 0 -4
<path id="m1bf1a62a27" d="M 0 -4
L -4 4
L 4 4
z
" style="stroke: #2563eb; stroke-linejoin: miter"/>
</defs>
<g>
<use xlink:href="#m44164aeb4a" x="314.164037" y="442.955958" style="fill: #2563eb; stroke: #2563eb; stroke-linejoin: miter"/>
<use xlink:href="#m1bf1a62a27" x="312.664037" y="445.955958" style="fill: #2563eb; stroke: #2563eb; stroke-linejoin: miter"/>
</g>
</g>
<g id="text_18">
<g id="text_19">
<!-- GitHub Copilot -->
<g transform="translate(328.164037 446.455958) scale(0.1 -0.1)">
<g transform="translate(329.664037 449.455958) scale(0.1 -0.1)">
<defs>
<path id="DejaVuSans-2a" d="M 3809 666
L 3809 1919
@@ -1508,15 +1532,56 @@ z
<use xlink:href="#DejaVuSans-57" transform="translate(689.5625 0)"/>
</g>
</g>
<g id="line2d_27">
<path d="M 304.164037 459.456739
L 314.164037 459.456739
L 324.164037 459.456739
" style="fill: none; stroke-dasharray: 6,4.5; stroke-dashoffset: 0; stroke: #64748b; stroke-width: 1.5"/>
<g id="line2d_30">
<defs>
<path id="m37e0adf3f3" d="M -0 5.656854
L 5.656854 0
L 0 -5.656854
L -5.656854 -0
z
" style="stroke: #64748b; stroke-linejoin: miter"/>
</defs>
<g>
<use xlink:href="#m37e0adf3f3" x="312.664037" y="460.956739" style="fill: #64748b; stroke: #64748b; stroke-linejoin: miter"/>
</g>
</g>
<g id="text_19">
<g id="text_20">
<!-- Baseline -->
<g transform="translate(328.164037 462.956739) scale(0.1 -0.1)">
<g transform="translate(329.664037 464.456739) scale(0.1 -0.1)">
<defs>
<path id="DejaVuSans-25" d="M 1259 2228
L 1259 519
L 2272 519
Q 2781 519 3026 730
Q 3272 941 3272 1375
Q 3272 1813 3026 2020
Q 2781 2228 2272 2228
L 1259 2228
z
M 1259 4147
L 1259 2741
L 2194 2741
Q 2656 2741 2882 2914
Q 3109 3088 3109 3444
Q 3109 3797 2882 3972
Q 2656 4147 2194 4147
L 1259 4147
z
M 628 4666
L 2241 4666
Q 2963 4666 3353 4366
Q 3744 4066 3744 3513
Q 3744 3084 3544 2831
Q 3344 2578 2956 2516
Q 3422 2416 3680 2098
Q 3938 1781 3938 1306
Q 3938 681 3513 340
Q 3088 0 2303 0
L 628 0
L 628 4666
z
" transform="scale(0.015625)"/>
</defs>
<use xlink:href="#DejaVuSans-25"/>
<use xlink:href="#DejaVuSans-44" transform="translate(68.609375 0)"/>
<use xlink:href="#DejaVuSans-56" transform="translate(129.890625 0)"/>
@@ -1527,9 +1592,9 @@ L 324.164037 459.456739
<use xlink:href="#DejaVuSans-48" transform="translate(362.453125 0)"/>
</g>
</g>
<g id="line2d_28">
<g id="line2d_31">
<defs>
<path id="mb21a4bd4d5" d="M 0 4
<path id="m3efa0fd325" d="M 0 4
C 1.060812 4 2.078319 3.578535 2.828427 2.828427
C 3.578535 2.078319 4 1.060812 4 0
C 4 -1.060812 3.578535 -2.078319 2.828427 -2.828427
@@ -1542,12 +1607,12 @@ z
" style="stroke: #334155"/>
</defs>
<g>
<use xlink:href="#mb21a4bd4d5" x="425.0406" y="442.955958" style="fill: #334155; stroke: #334155"/>
<use xlink:href="#m3efa0fd325" x="434.5406" y="445.955958" style="fill: #334155; stroke: #334155"/>
</g>
</g>
<g id="text_20">
<g id="text_21">
<!-- Agent Lightning -->
<g transform="translate(439.0406 446.455958) scale(0.1 -0.1)">
<g transform="translate(451.5406 449.455958) scale(0.1 -0.1)">
<defs>
<path id="DejaVuSans-4b" d="M 3513 2113
L 3513 0
@@ -1586,9 +1651,9 @@ z
<use xlink:href="#DejaVuSans-4a" transform="translate(731.875 0)"/>
</g>
</g>
<g id="line2d_29">
<g id="line2d_32">
<defs>
<path id="m862ab3808e" d="M 0 4
<path id="m98a1092128" d="M 0 4
C 1.060812 4 2.078319 3.578535 2.828427 2.828427
C 3.578535 2.078319 4 1.060812 4 0
C 4 -1.060812 3.578535 -2.078319 2.828427 -2.828427
@@ -1601,12 +1666,12 @@ z
" style="stroke: #334155; stroke-width: 1.6"/>
</defs>
<g>
<use xlink:href="#m862ab3808e" x="425.0406" y="459.456739" style="fill: #ffffff; stroke: #334155; stroke-width: 1.6"/>
<use xlink:href="#m98a1092128" x="434.5406" y="460.956739" style="fill: #ffffff; stroke: #334155; stroke-width: 1.6"/>
</g>
</g>
<g id="text_21">
<g id="text_22">
<!-- No skill -->
<g transform="translate(439.0406 462.956739) scale(0.1 -0.1)">
<g transform="translate(451.5406 464.456739) scale(0.1 -0.1)">
<defs>
<path id="DejaVuSans-31" d="M 628 4666
L 1478 4666
@@ -1650,7 +1715,7 @@ z
</g>
</g>
<defs>
<clipPath id="p5f2d4442e1">
<clipPath id="p8b1873d79b">
<rect x="69.4764" y="48.24" width="582.2784" height="323.208"/>
</clipPath>
</defs>

Before

Width:  |  Height:  |  Size: 46 KiB

After

Width:  |  Height:  |  Size: 47 KiB