|
1
|
+ |
#!/usr/bin/env node
|
|
2
|
+ |
/**
|
|
3
|
+ |
* Regenerate the suite manifests under `bench/suites/`.
|
|
4
|
+ |
*
|
|
5
|
+ |
* A manifest pins each task by its content rather than its name — for a public
|
|
6
|
+ |
* task, the git url, commit, and path Harbor's registry resolves it to; for an
|
|
7
|
+ |
* owned task, the tracker issue and the commit the forge recorded as closing
|
|
8
|
+ |
* it. This script is where those pins are read from their sources once, so that
|
|
9
|
+ |
* afterwards the manifests are self-contained and every consumer (the digest,
|
|
10
|
+ |
* the gate, the store) works offline from the checked-in file alone. Same split
|
|
11
|
+ |
* as `src/pricing.ts`: resolve from the source of truth, then pin the snapshot,
|
|
12
|
+ |
* and keep the path back to the source written down.
|
|
13
|
+ |
*
|
|
14
|
+ |
* Two sources, neither of which this repo owns:
|
|
15
|
+ |
*
|
|
16
|
+ |
* --registry <path> Harbor's `registry.json`, an 80-dataset index that
|
|
17
|
+ |
* already carries `terminal-bench@2.0` and
|
|
18
|
+ |
* `swebench-verified@1.0` behind one contract. Defaults
|
|
19
|
+ |
* to the clone at ../projects/repos/harbor.
|
|
20
|
+ |
* --issues <path> A `openagents issue list --state closed --json` body
|
|
21
|
+ |
* for OpenAgentsInc/openagents. Closed issues whose
|
|
22
|
+ |
* evidence carries a `closing_reference` commit are the
|
|
23
|
+ |
* owned tasks; the rest are skipped by name.
|
|
24
|
+ |
*
|
|
25
|
+ |
* Usage:
|
|
26
|
+ |
* node bench/build-suites.mjs --registry <registry.json> --issues <closed.json>
|
|
27
|
+ |
* node bench/build-suites.mjs ... --check # rebuild and diff, write nothing
|
|
28
|
+ |
*
|
|
29
|
+ |
* `--check` is what CI runs: it rebuilds every manifest and fails if the result
|
|
30
|
+ |
* differs from what is committed, so a manifest cannot drift from the registry
|
|
31
|
+ |
* it claims to pin without somebody noticing.
|
|
32
|
+ |
*/
|
|
33
|
+ |
|
|
34
|
+ |
import { readFileSync, writeFileSync } from "node:fs";
|
|
35
|
+ |
import { dirname, join, resolve } from "node:path";
|
|
36
|
+ |
import { fileURLToPath } from "node:url";
|
|
37
|
+ |
|
|
38
|
+ |
const BENCH_DIR = dirname(fileURLToPath(import.meta.url));
|
|
39
|
+ |
const REPO_ROOT = dirname(BENCH_DIR);
|
|
40
|
+ |
const SUITES_DIR = join(BENCH_DIR, "suites");
|
|
41
|
+ |
|
|
42
|
+ |
const SCHEMA = "openagents.effectiveness_suite.v1";
|
|
43
|
+ |
|
|
44
|
+ |
/**
|
|
45
|
+ |
* The twelve Terminal-Bench 2.0 tasks selected in
|
|
46
|
+ |
* `bench/suites/tb2-cross-section.md`, which records why each slot exists.
|
|
47
|
+ |
* Kept as a name list here because the pins come from the registry.
|
|
48
|
+ |
*/
|
|
49
|
+ |
const TB2_CROSS_SECTION = [
|
|
50
|
+ |
"git-leak-recovery",
|
|
51
|
+ |
"sanitize-git-repo",
|
|
52
|
+ |
"merge-diff-arc-agi-task",
|
|
53
|
+ |
"build-cython-ext",
|
|
54
|
+ |
"sqlite-with-gcov",
|
|
55
|
+ |
"fix-code-vulnerability",
|
|
56
|
+ |
"regex-log",
|
|
57
|
+ |
"count-dataset-tokens",
|
|
58
|
+ |
"password-recovery",
|
|
59
|
+ |
"openssl-selfsigned-cert",
|
|
60
|
+ |
"nginx-request-logging",
|
|
61
|
+ |
"schemelike-metacircular-eval",
|
|
62
|
+ |
];
|
|
63
|
+ |
|
|
64
|
+ |
/**
|
|
65
|
+ |
* The bounded public SWE-bench subset: eight repositories, one instance each.
|
|
66
|
+ |
*
|
|
67
|
+ |
* Issue #34 names SWE-bench-lite as the candidate. Harbor's registry carries
|
|
68
|
+ |
* `swebench-verified@1.0` instead — the 500-instance human-validated subset —
|
|
69
|
+ |
* and that is the better half of the same idea: it is the subset whose task
|
|
70
|
+ |
* statements and tests a human confirmed are solvable and correctly graded,
|
|
71
|
+ |
* which is what a floor wants underneath it, and it runs through the same
|
|
72
|
+ |
* `harbor run --dataset` contract with no new harness code. Lite's
|
|
73
|
+ |
* distinguishing property is being small, and this suite is bounding itself.
|
|
74
|
+ |
*
|
|
75
|
+ |
* ONE INSTANCE PER REPOSITORY, CHOSEN BY RULE RATHER THAN BY HAND. The 500 are
|
|
76
|
+ |
* dominated by Django (231) and SymPy (75); a random eight would usually be
|
|
77
|
+ |
* five Djangos, and a score over it would mostly measure how well the coder
|
|
78
|
+ |
* knows one codebase. So: the eight repositories with the most instances, and
|
|
79
|
+ |
* from each the lexicographically first instance id.
|
|
80
|
+ |
*
|
|
81
|
+ |
* The second half of that rule matters more than it looks. Any hand-picked
|
|
82
|
+ |
* instance invites the question of whether it was picked because the coder does
|
|
83
|
+ |
* well on it, and there is no way to answer that question from the outside. A
|
|
84
|
+ |
* mechanical rule answers it in advance, survives a registry refresh, and can be
|
|
85
|
+ |
* re-derived by anyone with the registry — which is also what lets `--check`
|
|
86
|
+ |
* mean something.
|
|
87
|
+ |
*/
|
|
88
|
+ |
const SWEBENCH_REPOS = [
|
|
89
|
+ |
["astropy", "astropy: table and unit handling in a large scientific codebase"],
|
|
90
|
+ |
["django", "Django: the framework half of the benchmark, wide blast radius per change"],
|
|
91
|
+ |
["matplotlib", "matplotlib: rendering state, where the test is the only oracle"],
|
|
92
|
+ |
["pydata", "xarray: array semantics over pandas and numpy, dtype-sensitive"],
|
|
93
|
+ |
["pytest-dev", "pytest: the test runner itself, so a fix has to be reentrant"],
|
|
94
|
+ |
["scikit-learn", "scikit-learn: estimator API conformance, contract-shaped"],
|
|
95
|
+ |
["sphinx-doc", "Sphinx: documentation tooling, heavy on configuration surface"],
|
|
96
|
+ |
["sympy", "SymPy: symbolic evaluation, where a plausible-looking fix is usually wrong"],
|
|
97
|
+ |
];
|
|
98
|
+ |
|
|
99
|
+ |
const parseArguments = () => {
|
|
100
|
+ |
const argv = process.argv.slice(2);
|
|
101
|
+ |
let registry = resolve(REPO_ROOT, "../projects/repos/harbor/registry.json");
|
|
102
|
+ |
let issues = null;
|
|
103
|
+ |
let check = false;
|
|
104
|
+ |
for (let index = 0; index < argv.length; index += 1) {
|
|
105
|
+ |
const argument = argv[index];
|
|
106
|
+ |
if (argument === "--check") check = true;
|
|
107
|
+ |
else if (argument === "--registry") registry = argv[(index += 1)];
|
|
108
|
+ |
else if (argument === "--issues") issues = argv[(index += 1)];
|
|
109
|
+ |
else throw new Error(`unknown option: ${argument}`);
|
|
110
|
+ |
}
|
|
111
|
+ |
return { registry, issues, check };
|
|
112
|
+ |
};
|
|
113
|
+ |
|
|
114
|
+ |
/** Index a Harbor registry into `dataset@version -> { taskName -> pin }`. */
|
|
115
|
+ |
const readRegistry = (path) => {
|
|
116
|
+ |
const datasets = new Map();
|
|
117
|
+ |
for (const entry of JSON.parse(readFileSync(path, "utf8"))) {
|
|
118
|
+ |
const tasks = new Map();
|
|
119
|
+ |
for (const task of entry.tasks ?? []) {
|
|
120
|
+ |
tasks.set(task.name, {
|
|
121
|
+ |
gitUrl: task.git_url,
|
|
122
|
+ |
commit: task.git_commit_id,
|
|
123
|
+ |
path: task.path,
|
|
124
|
+ |
});
|
|
125
|
+ |
}
|
|
126
|
+ |
datasets.set(`${entry.name}@${entry.version}`, tasks);
|
|
127
|
+ |
}
|
|
128
|
+ |
return datasets;
|
|
129
|
+ |
};
|
|
130
|
+ |
|
|
131
|
+ |
/**
|
|
132
|
+ |
* Resolve a task name in a dataset to its pin.
|
|
133
|
+ |
*
|
|
134
|
+ |
* A missing name throws rather than being skipped. A suite that quietly dropped
|
|
135
|
+ |
* a task it could not resolve would still build, still digest, and still
|
|
136
|
+ |
* classify a run over the remaining tasks as full coverage — which is the
|
|
137
|
+ |
* failure the digest exists to prevent, arriving through the front door.
|
|
138
|
+ |
*/
|
|
139
|
+ |
const pinFor = (datasets, dataset, name) => {
|
|
140
|
+ |
const tasks = datasets.get(dataset);
|
|
141
|
+ |
if (tasks === undefined) {
|
|
142
|
+ |
throw new Error(`registry has no dataset ${dataset}`);
|
|
143
|
+ |
}
|
|
144
|
+ |
const pin = tasks.get(name);
|
|
145
|
+ |
if (pin === undefined) {
|
|
146
|
+ |
throw new Error(`registry dataset ${dataset} has no task ${name}`);
|
|
147
|
+ |
}
|
|
148
|
+ |
if (pin.commit === "HEAD") {
|
|
149
|
+ |
throw new Error(
|
|
150
|
+ |
`registry dataset ${dataset} pins ${name} at HEAD, which names a moving target rather than a commit; this suite cannot pin it`,
|
|
151
|
+ |
);
|
|
152
|
+ |
}
|
|
153
|
+ |
return { kind: "harbor-registry", dataset, ...pin };
|
|
154
|
+ |
};
|
|
155
|
+ |
|
|
156
|
+ |
/**
|
|
157
|
+ |
* The lexicographically first instance id a repository contributes to a
|
|
158
|
+ |
* dataset. SWE-bench instance ids are `<org>__<repo>-<pr-number>`, so the
|
|
159
|
+ |
* repository is the part before the double underscore.
|
|
160
|
+ |
*/
|
|
161
|
+ |
const firstInstanceOf = (datasets, dataset, repo) => {
|
|
162
|
+ |
const tasks = datasets.get(dataset);
|
|
163
|
+ |
if (tasks === undefined) throw new Error(`registry has no dataset ${dataset}`);
|
|
164
|
+ |
const names = [...tasks.keys()].filter((name) => name.split("__")[0] === repo).toSorted();
|
|
165
|
+ |
if (names.length === 0) {
|
|
166
|
+ |
throw new Error(`registry dataset ${dataset} holds no instance from ${repo}`);
|
|
167
|
+ |
}
|
|
168
|
+ |
return names[0];
|
|
169
|
+ |
};
|
|
170
|
+ |
|
|
171
|
+ |
const registryTask = (datasets, dataset, name, rationale, environmentProven) => ({
|
|
172
|
+ |
id: name,
|
|
173
|
+ |
pin: pinFor(datasets, dataset, name),
|
|
174
|
+ |
environmentProven,
|
|
175
|
+ |
...(rationale === undefined ? {} : { rationale }),
|
|
176
|
+ |
});
|
|
177
|
+ |
|
|
178
|
+ |
/**
|
|
179
|
+ |
* Owned tasks: closed issues in this tracker that carry a closing commit.
|
|
180
|
+ |
*
|
|
181
|
+ |
* The forge records a closing reference as evidence on the issue, so the
|
|
182
|
+ |
* accepted outcome is a fact the tracker already holds rather than a judgement
|
|
183
|
+ |
* this script makes. An issue with no such evidence is skipped and named: it may
|
|
184
|
+ |
* be perfectly well closed, but without a commit there is nothing to grade
|
|
185
|
+ |
* against.
|
|
186
|
+ |
*
|
|
187
|
+ |
* Every one of these is `environmentProven: false` today. The pin is real — the
|
|
188
|
+ |
* issue, its instruction, and the commit that satisfied it — and no container
|
|
189
|
+ |
* has been built that can grade it, so `parseSuiteManifest` will refuse to let
|
|
190
|
+ |
* them into a score-tier suite until one has. See `bench/tasks/owned/README.md`.
|
|
191
|
+ |
*/
|
|
192
|
+ |
const ownedTasks = (issuesPath) => {
|
|
193
|
+ |
const body = JSON.parse(readFileSync(issuesPath, "utf8"));
|
|
194
|
+ |
const issues = Array.isArray(body) ? body : (body.issues ?? []);
|
|
195
|
+ |
const tasks = [];
|
|
196
|
+ |
const skipped = [];
|
|
197
|
+ |
for (const issue of issues) {
|
|
198
|
+ |
const commits = (issue.openagents?.evidence ?? [])
|
|
199
|
+ |
.filter((entry) => entry.source === "closing_reference" && typeof entry.commit === "string")
|
|
200
|
+ |
.map((entry) => entry.commit);
|
|
201
|
+ |
if (commits.length === 0) {
|
|
202
|
+ |
skipped.push(issue.number);
|
|
203
|
+ |
continue;
|
|
204
|
+ |
}
|
|
205
|
+ |
// Several closing references means the work landed over more than one push.
|
|
206
|
+ |
// The last one is the state the issue was closed in, so that is the pin.
|
|
207
|
+ |
const acceptedCommit = commits.at(-1);
|
|
208
|
+ |
tasks.push({
|
|
209
|
+ |
id: `owned-issue-${issue.number}`,
|
|
210
|
+ |
pin: {
|
|
211
|
+ |
kind: "tracker-closed-issue",
|
|
212
|
+ |
repo: "OpenAgentsInc/openagents",
|
|
213
|
+ |
issue: issue.number,
|
|
214
|
+ |
acceptedCommit,
|
|
215
|
+ |
},
|
|
216
|
+ |
environmentProven: false,
|
|
217
|
+ |
rationale: issue.title,
|
|
218
|
+ |
});
|
|
219
|
+ |
}
|
|
220
|
+ |
tasks.sort((left, right) => left.pin.issue - right.pin.issue);
|
|
221
|
+ |
return { tasks, skipped };
|
|
222
|
+ |
};
|
|
223
|
+ |
|
|
224
|
+ |
const manifest = (id, tier, description, tasks) => ({
|
|
225
|
+ |
schema: SCHEMA,
|
|
226
|
+ |
id,
|
|
227
|
+ |
tier,
|
|
228
|
+ |
description,
|
|
229
|
+ |
tasks,
|
|
230
|
+ |
});
|
|
231
|
+ |
|
|
232
|
+ |
const write = (name, value, check) => {
|
|
233
|
+ |
const path = join(SUITES_DIR, name);
|
|
234
|
+ |
const text = `${JSON.stringify(value, null, 2)}\n`;
|
|
235
|
+ |
if (check) {
|
|
236
|
+ |
const existing = readFileSync(path, "utf8");
|
|
237
|
+ |
if (existing !== text) {
|
|
238
|
+ |
throw new Error(
|
|
239
|
+ |
`${name} is out of date with its sources. Run bench/build-suites.mjs without --check and commit the result.`,
|
|
240
|
+ |
);
|
|
241
|
+ |
}
|
|
242
|
+ |
process.stdout.write(`ok ${name}\n`);
|
|
243
|
+ |
return;
|
|
244
|
+ |
}
|
|
245
|
+ |
writeFileSync(path, text, "utf8");
|
|
246
|
+ |
process.stdout.write(`wrote ${name} (${String(value.tasks.length)} tasks)\n`);
|
|
247
|
+ |
};
|
|
248
|
+ |
|
|
249
|
+ |
const main = () => {
|
|
250
|
+ |
const { registry, issues, check } = parseArguments();
|
|
251
|
+ |
const datasets = readRegistry(registry);
|
|
252
|
+ |
|
|
253
|
+ |
const tb2 = TB2_CROSS_SECTION.map((name) =>
|
|
254
|
+ |
registryTask(datasets, "terminal-bench@2.0", name, undefined, true),
|
|
255
|
+ |
);
|
|
256
|
+ |
const swe = SWEBENCH_REPOS.map(([repo, rationale]) =>
|
|
257
|
+ |
registryTask(
|
|
258
|
+ |
datasets,
|
|
259
|
+ |
"swebench-verified@1.0",
|
|
260
|
+ |
firstInstanceOf(datasets, "swebench-verified@1.0", repo),
|
|
261
|
+ |
rationale,
|
|
262
|
+ |
true,
|
|
263
|
+ |
),
|
|
264
|
+ |
);
|
|
265
|
+ |
|
|
266
|
+ |
write(
|
|
267
|
+ |
"tb2-cross-section.suite.json",
|
|
268
|
+ |
manifest(
|
|
269
|
+ |
"tb2-cross-section",
|
|
270
|
+ |
"score",
|
|
271
|
+ |
"Twelve Terminal-Bench 2.0 tasks across git forensics, builds, C extensions, coverage, security fixes, log parsing, tokenisation, certificates, web-server configuration, and an interpreter. Selection rationale in tb2-cross-section.md.",
|
|
272
|
+ |
tb2,
|
|
273
|
+ |
),
|
|
274
|
+ |
check,
|
|
275
|
+ |
);
|
|
276
|
+ |
|
|
277
|
+ |
write(
|
|
278
|
+ |
"swebench-verified-subset.suite.json",
|
|
279
|
+ |
manifest(
|
|
280
|
+ |
"swebench-verified-subset",
|
|
281
|
+ |
"score",
|
|
282
|
+ |
"The bounded public subset issue #34 asks for: eight swebench-verified@1.0 instances, one per repository, so no single project's idioms dominate the score.",
|
|
283
|
+ |
swe,
|
|
284
|
+ |
),
|
|
285
|
+ |
check,
|
|
286
|
+ |
);
|
|
287
|
+ |
|
|
288
|
+ |
/**
|
|
289
|
+ |
* The two quickest tasks in the cross-section, declared `score` rather than
|
|
290
|
+ |
* `smoke` and floored accordingly.
|
|
291
|
+ |
*
|
|
292
|
+ |
* This is not a fast lane wearing a score badge, and the distinction is worth
|
|
293
|
+ |
* being precise about because the whole smoke rule depends on it. `smoke` is
|
|
294
|
+ |
* for a suite whose result should never be published — a liveness check.
|
|
295
|
+ |
* `tb2-quick` is a real, if narrow, measurement: two tasks the coder is
|
|
296
|
+ |
* genuinely expected to solve, always run to completion, scored against
|
|
297
|
+ |
* floors set for two tasks rather than for twelve. It exists because a suite
|
|
298
|
+ |
* you can run three times in an hour is the only kind you can prove a
|
|
299
|
+ |
* regression with on one machine, and because the headline suite's floors are
|
|
300
|
+ |
* useless if nobody ever runs anything against them.
|
|
301
|
+ |
*
|
|
302
|
+ |
* What it cannot do is stand in for the headline number. It shares no suite
|
|
303
|
+ |
* key with `coder-effectiveness-v1`, so no comparison will ever place its rows
|
|
304
|
+ |
* beside that suite's — which is the property that makes shipping a narrow
|
|
305
|
+ |
* score suite safe rather than a slow leak.
|
|
306
|
+ |
*/
|
|
307
|
+ |
write(
|
|
308
|
+ |
"tb2-quick.suite.json",
|
|
309
|
+ |
manifest(
|
|
310
|
+ |
"tb2-quick",
|
|
311
|
+ |
"score",
|
|
312
|
+ |
"Two quick Terminal-Bench 2.0 tasks: a narrow but real score suite, small enough to run repeatedly on one machine, floored for its own size. Never comparable to coder-effectiveness-v1 — different suite key.",
|
|
313
|
+ |
[
|
|
314
|
+ |
[
|
|
315
|
+ |
"regex-log",
|
|
316
|
+ |
"near-zero tool surface: the suite's test of whether the agent can just answer",
|
|
317
|
+ |
],
|
|
318
|
+ |
[
|
|
319
|
+ |
"openssl-selfsigned-cert",
|
|
320
|
+ |
"a fully specified checklist: every command is known upfront, so round count is a tool habit rather than a reasoning result",
|
|
321
|
+ |
],
|
|
322
|
+ |
].map(([name, rationale]) =>
|
|
323
|
+ |
registryTask(datasets, "terminal-bench@2.0", name, rationale, true),
|
|
324
|
+ |
),
|
|
325
|
+ |
),
|
|
326
|
+ |
check,
|
|
327
|
+ |
);
|
|
328
|
+ |
|
|
329
|
+ |
write(
|
|
330
|
+ |
"smoke.suite.json",
|
|
331
|
+ |
manifest(
|
|
332
|
+ |
"smoke",
|
|
333
|
+ |
"smoke",
|
|
334
|
+ |
"The fast lane: two quick Terminal-Bench tasks for checking that the harness, the adapter, and the lane are alive. Declared smoke, so its result is never a published score however completely it runs.",
|
|
335
|
+ |
TB2_CROSS_SECTION.filter(
|
|
336
|
+ |
(name) => name === "regex-log" || name === "fix-code-vulnerability",
|
|
337
|
+ |
).map((name) => registryTask(datasets, "terminal-bench@2.0", name, "quick-shaped", true)),
|
|
338
|
+ |
),
|
|
339
|
+ |
check,
|
|
340
|
+ |
);
|
|
341
|
+ |
|
|
342
|
+ |
if (issues !== null) {
|
|
343
|
+ |
const owned = ownedTasks(issues);
|
|
344
|
+ |
write(
|
|
345
|
+ |
"owned-closed-issues.suite.json",
|
|
346
|
+ |
manifest(
|
|
347
|
+ |
"owned-closed-issues",
|
|
348
|
+ |
"smoke",
|
|
349
|
+ |
"The owned half of issue #34's suite: closed issues in this tracker whose forge evidence carries a closing commit, so the accepted outcome is recorded rather than assumed. Tier smoke until the environments are built — see bench/tasks/owned/README.md.",
|
|
350
|
+ |
owned.tasks,
|
|
351
|
+ |
),
|
|
352
|
+ |
check,
|
|
353
|
+ |
);
|
|
354
|
+ |
process.stdout.write(
|
|
355
|
+ |
` ${String(owned.skipped.length)} closed issue(s) carry no closing-reference commit and are not tasks: ${owned.skipped.join(", ")}\n`,
|
|
356
|
+ |
);
|
|
357
|
+ |
|
|
358
|
+ |
write(
|
|
359
|
+ |
"coder-effectiveness-v1.suite.json",
|
|
360
|
+ |
manifest(
|
|
361
|
+ |
"coder-effectiveness-v1",
|
|
362
|
+ |
"score",
|
|
363
|
+ |
"Issue #34's headline suite: the twelve-task Terminal-Bench cross-section plus the eight-instance swebench-verified subset. The owned closed-issue tasks join it once their environments are proven; until then a score suite cannot hold them.",
|
|
364
|
+ |
[...tb2, ...swe],
|
|
365
|
+ |
),
|
|
366
|
+ |
check,
|
|
367
|
+ |
);
|
|
368
|
+ |
}
|
|
369
|
+ |
};
|
|
370
|
+ |
|
|
371
|
+ |
try {
|
|
372
|
+ |
main();
|
|
373
|
+ |
} catch (error) {
|
|
374
|
+ |
process.stderr.write(`build-suites: ${error.message}\n`);
|
|
375
|
+ |
process.exitCode = 1;
|
|
376
|
+ |
}
|