|
1
|
+ |
/**
|
|
2
|
+ |
* Read one completed Harbor job directory into typed, graded trials.
|
|
3
|
+ |
*
|
|
4
|
+ |
* The input is exactly what `bench/run-suite.sh` already leaves on disk after
|
|
5
|
+ |
* `harbor run --dataset terminal-bench@2.0 --agent-import-path
|
|
6
|
+ |
* adapters.openagents_coder:OpenAgentsCoder`:
|
|
7
|
+ |
*
|
|
8
|
+ |
* <job-dir>/result.json the job envelope
|
|
9
|
+ |
* <job-dir>/config.json the pinned recipe
|
|
10
|
+ |
* <job-dir>/<task>__<uuid>/result.json the trial and its verifier result
|
|
11
|
+ |
* <job-dir>/<task>__<uuid>/agent/trajectory.json the coder's ATIF export
|
|
12
|
+ |
* <job-dir>/<task>__<uuid>/agent/coder.txt the captured session
|
|
13
|
+ |
*
|
|
14
|
+ |
* Nothing new is produced by the coder for this suite to work. The field names
|
|
15
|
+ |
* read here are the same ones `bench/post_gym_run.py` reads, so the two agree
|
|
16
|
+ |
* about what a pass is.
|
|
17
|
+ |
*
|
|
18
|
+ |
* THE GRADING RULE. A trial is `accepted` only when a verifier RAN and
|
|
19
|
+ |
* returned a positive reward. A trial whose verifier never ran is `ungraded` —
|
|
20
|
+ |
* not a failure, and not a pass. The Terminal-Bench images are amd64 and their
|
|
21
|
+ |
* verifier segfaults under qemu on Apple Silicon, so a crashed grader is a
|
|
22
|
+ |
* routine local outcome rather than a rare one, and folding it into either
|
|
23
|
+ |
* bucket would move the headline number for a reason that has nothing to do
|
|
24
|
+ |
* with the coder. `ungraded` trials are counted, reported, and kept out of the
|
|
25
|
+ |
* success-rate denominator, and a threshold can cap how many of them a run may
|
|
26
|
+ |
* contain before the run stops being worth reading.
|
|
27
|
+ |
*/
|
|
28
|
+ |
|
|
29
|
+ |
import { createHash } from "node:crypto";
|
|
30
|
+ |
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
31
|
+ |
import { join } from "node:path";
|
|
32
|
+ |
|
|
33
|
+ |
/** The coder's `--plain` thread announcement, the same contract bench parses. */
|
|
34
|
+ |
const THREAD_LINE = /\[oa:thread ([0-9a-fA-F-]{36})\]/u;
|
|
35
|
+ |
|
|
36
|
+ |
/** What a verifier decided about one trial. */
|
|
37
|
+ |
export type TrialOutcome = "accepted" | "rejected" | "ungraded";
|
|
38
|
+ |
|
|
39
|
+ |
export interface TrialRecord {
|
|
40
|
+ |
/** The task half of Harbor's `<task>__<shortuuid>` directory name. */
|
|
41
|
+ |
readonly task: string;
|
|
42
|
+ |
readonly outcome: TrialOutcome;
|
|
43
|
+ |
readonly modelId: string | null;
|
|
44
|
+ |
readonly agentVersion: string | null;
|
|
45
|
+ |
readonly promptTokens: number | null;
|
|
46
|
+ |
readonly completionTokens: number | null;
|
|
47
|
+ |
/** Cached-read input tokens, summed from the ATIF steps. */
|
|
48
|
+ |
readonly cachedInputTokens: number;
|
|
49
|
+ |
/** Tool calls the trajectory records, or `null` when it records no steps. */
|
|
50
|
+ |
readonly toolCalls: number | null;
|
|
51
|
+ |
readonly wallClockSeconds: number | null;
|
|
52
|
+ |
/** The forge thread the trial ran in, when the coder announced one. */
|
|
53
|
+ |
readonly threadId: string | null;
|
|
54
|
+ |
/** The typed error Harbor classified, when the trial raised one. */
|
|
55
|
+ |
readonly exception: string | null;
|
|
56
|
+ |
}
|
|
57
|
+ |
|
|
58
|
+ |
export interface GradedRun {
|
|
59
|
+ |
readonly jobId: string | null;
|
|
60
|
+ |
readonly suite: string;
|
|
61
|
+ |
readonly lane: string;
|
|
62
|
+ |
/**
|
|
63
|
+ |
* A digest over what was actually run: the suite, the lane, the sorted task
|
|
64
|
+ |
* list, the CLI version, the model, and the rate catalog version. Two runs
|
|
65
|
+ |
* that share a digest are comparable rows; two that do not are not, and the
|
|
66
|
+ |
* report says so rather than letting a reader assume.
|
|
67
|
+ |
*/
|
|
68
|
+ |
readonly runDigest: string;
|
|
69
|
+ |
readonly trials: ReadonlyArray<TrialRecord>;
|
|
70
|
+ |
}
|
|
71
|
+ |
|
|
72
|
+ |
export interface ReadHarborJobOptions {
|
|
73
|
+ |
readonly suite: string;
|
|
74
|
+ |
readonly lane: string;
|
|
75
|
+ |
/** Mixed into the run digest so a re-scored run is not mistaken for a re-run. */
|
|
76
|
+ |
readonly rateCatalogVersion: string;
|
|
77
|
+ |
}
|
|
78
|
+ |
|
|
79
|
+ |
/**
|
|
80
|
+ |
* Read a Harbor job directory. Throws when the directory is not a Harbor job,
|
|
81
|
+ |
* because scoring a directory that holds no result is not a zero-score run.
|
|
82
|
+ |
*/
|
|
83
|
+ |
export const readHarborJob = (jobDir: string, options: ReadHarborJobOptions): GradedRun => {
|
|
84
|
+ |
const jobResult = readJson(join(jobDir, "result.json"));
|
|
85
|
+ |
if (jobResult === undefined) {
|
|
86
|
+ |
throw new Error(
|
|
87
|
+ |
`not a Harbor job directory (no result.json): ${jobDir}. Point --job-dir at the directory harbor run created under its --jobs-dir.`,
|
|
88
|
+ |
);
|
|
89
|
+ |
}
|
|
90
|
+ |
|
|
91
|
+ |
const trials: Array<TrialRecord> = [];
|
|
92
|
+ |
for (const entry of readdirSync(jobDir).toSorted()) {
|
|
93
|
+ |
const trialDir = join(jobDir, entry);
|
|
94
|
+ |
if (!statSync(trialDir).isDirectory()) continue;
|
|
95
|
+ |
const trialResult = readJson(join(trialDir, "result.json"));
|
|
96
|
+ |
if (trialResult === undefined) continue;
|
|
97
|
+ |
trials.push(readTrial(entry, trialDir, trialResult));
|
|
98
|
+ |
}
|
|
99
|
+ |
|
|
100
|
+ |
return {
|
|
101
|
+ |
jobId: readString(readField(jobResult, "id")),
|
|
102
|
+ |
suite: options.suite,
|
|
103
|
+ |
lane: options.lane,
|
|
104
|
+ |
runDigest: runDigestOf(trials, options),
|
|
105
|
+ |
trials,
|
|
106
|
+ |
};
|
|
107
|
+ |
};
|
|
108
|
+ |
|
|
109
|
+ |
const readTrial = (dirName: string, trialDir: string, trialResult: unknown): TrialRecord => {
|
|
110
|
+ |
const trajectory = readJson(join(trialDir, "agent", "trajectory.json"));
|
|
111
|
+ |
const usage = readTrajectoryUsage(trajectory);
|
|
112
|
+ |
const agent = readField(trajectory, "agent");
|
|
113
|
+ |
|
|
114
|
+ |
return {
|
|
115
|
+ |
task: dirName.includes("__") ? dirName.slice(0, dirName.lastIndexOf("__")) : dirName,
|
|
116
|
+ |
outcome: outcomeOf(trialResult),
|
|
117
|
+ |
modelId: readString(readField(agent, "model_name")),
|
|
118
|
+ |
agentVersion: readString(readField(agent, "version")),
|
|
119
|
+ |
promptTokens: usage.promptTokens,
|
|
120
|
+ |
completionTokens: usage.completionTokens,
|
|
121
|
+ |
cachedInputTokens: usage.cachedInputTokens,
|
|
122
|
+ |
toolCalls: usage.toolCalls,
|
|
123
|
+ |
wallClockSeconds: wallClockOf(trialResult),
|
|
124
|
+ |
threadId: threadIdOf(trialDir),
|
|
125
|
+ |
exception: readString(readField(readField(trialResult, "exception_info"), "exception_type")),
|
|
126
|
+ |
};
|
|
127
|
+ |
};
|
|
128
|
+ |
|
|
129
|
+ |
/**
|
|
130
|
+ |
* A verifier that never ran leaves no `verifier_result`. That is `ungraded`.
|
|
131
|
+ |
* Where one ran, any positive reward is an accepted outcome; Harbor writes the
|
|
132
|
+ |
* reward either as `rewards.reward` or as the sole entry of a `rewards` map.
|
|
133
|
+ |
*/
|
|
134
|
+ |
const outcomeOf = (trialResult: unknown): TrialOutcome => {
|
|
135
|
+ |
const verifier = readField(trialResult, "verifier_result");
|
|
136
|
+ |
if (verifier === undefined || verifier === null) return "ungraded";
|
|
137
|
+ |
const rewards = readField(verifier, "rewards") ?? readField(trialResult, "rewards");
|
|
138
|
+ |
const reward = rewardValue(rewards);
|
|
139
|
+ |
return reward !== null && reward > 0 ? "accepted" : "rejected";
|
|
140
|
+ |
};
|
|
141
|
+ |
|
|
142
|
+ |
const rewardValue = (rewards: unknown): number | null => {
|
|
143
|
+ |
const direct = readNumber(readField(rewards, "reward"));
|
|
144
|
+ |
if (direct !== null) return direct;
|
|
145
|
+ |
if (typeof rewards !== "object" || rewards === null) return null;
|
|
146
|
+ |
const values = Object.values(rewards as Record<string, unknown>);
|
|
147
|
+ |
const first = values.length === 1 ? readNumber(values[0]) : null;
|
|
148
|
+ |
return first;
|
|
149
|
+ |
};
|
|
150
|
+ |
|
|
151
|
+ |
const wallClockOf = (trialResult: unknown): number | null => {
|
|
152
|
+ |
const execution = readField(trialResult, "agent_execution");
|
|
153
|
+ |
const started = readString(readField(execution, "started_at"));
|
|
154
|
+ |
const finished = readString(readField(execution, "finished_at"));
|
|
155
|
+ |
if (started === null || finished === null) return null;
|
|
156
|
+ |
const span = Date.parse(finished) - Date.parse(started);
|
|
157
|
+ |
return Number.isFinite(span) ? span / 1000 : null;
|
|
158
|
+ |
};
|
|
159
|
+ |
|
|
160
|
+ |
interface TrajectoryUsage {
|
|
161
|
+ |
readonly promptTokens: number | null;
|
|
162
|
+ |
readonly completionTokens: number | null;
|
|
163
|
+ |
readonly cachedInputTokens: number;
|
|
164
|
+ |
readonly toolCalls: number | null;
|
|
165
|
+ |
}
|
|
166
|
+ |
|
|
167
|
+ |
/**
|
|
168
|
+ |
* Pull usage out of the coder's ATIF export.
|
|
169
|
+ |
*
|
|
170
|
+ |
* Totals come from `final_metrics`, which the coder writes only when at least
|
|
171
|
+ |
* one turn reported usage — an absent total is unknown, so it stays `null`.
|
|
172
|
+ |
* Cached reads are NOT in `final_metrics`: the exporter carries them per step
|
|
173
|
+ |
* as `metrics.extra.cache_read_input_tokens` and never totals them, so this
|
|
174
|
+ |
* sums the steps. A trajectory with no cached figure anywhere reports 0 cached
|
|
175
|
+ |
* tokens, which is the honest reading — the coder omits the field when the
|
|
176
|
+ |
* turn reported no cache reads, not when it failed to measure them.
|
|
177
|
+ |
*/
|
|
178
|
+ |
const readTrajectoryUsage = (trajectory: unknown): TrajectoryUsage => {
|
|
179
|
+ |
const finalMetrics = readField(trajectory, "final_metrics");
|
|
180
|
+ |
const steps = readArray(readField(trajectory, "steps"));
|
|
181
|
+ |
|
|
182
|
+ |
let cachedInputTokens = 0;
|
|
183
|
+ |
let toolCalls = 0;
|
|
184
|
+ |
for (const step of steps) {
|
|
185
|
+ |
const extra = readField(readField(step, "metrics"), "extra");
|
|
186
|
+ |
cachedInputTokens += readNumber(readField(extra, "cache_read_input_tokens")) ?? 0;
|
|
187
|
+ |
toolCalls += readArray(readField(step, "tool_calls")).length;
|
|
188
|
+ |
}
|
|
189
|
+ |
|
|
190
|
+ |
return {
|
|
191
|
+ |
promptTokens: readNumber(readField(finalMetrics, "total_prompt_tokens")),
|
|
192
|
+ |
completionTokens: readNumber(readField(finalMetrics, "total_completion_tokens")),
|
|
193
|
+ |
cachedInputTokens,
|
|
194
|
+ |
toolCalls: steps.length === 0 ? null : toolCalls,
|
|
195
|
+ |
};
|
|
196
|
+ |
};
|
|
197
|
+ |
|
|
198
|
+ |
const threadIdOf = (trialDir: string): string | null => {
|
|
199
|
+ |
const path = join(trialDir, "agent", "coder.txt");
|
|
200
|
+ |
if (!existsSync(path)) return null;
|
|
201
|
+ |
const match = THREAD_LINE.exec(readFileSync(path, "utf8"));
|
|
202
|
+ |
return match?.[1] ?? null;
|
|
203
|
+ |
};
|
|
204
|
+ |
|
|
205
|
+ |
/**
|
|
206
|
+ |
* The recipe pin. Deliberately independent of the `harbor:` digest
|
|
207
|
+ |
* `bench/post_gym_run.py` computes: that one hashes a Python-serialised config
|
|
208
|
+ |
* and this one hashes an explicit list of the facts that make two runs
|
|
209
|
+ |
* comparable, so claiming they agree would be a claim neither can keep.
|
|
210
|
+ |
*/
|
|
211
|
+ |
const runDigestOf = (trials: ReadonlyArray<TrialRecord>, options: ReadHarborJobOptions): string => {
|
|
212
|
+ |
const source = JSON.stringify({
|
|
213
|
+ |
suite: options.suite,
|
|
214
|
+ |
lane: options.lane,
|
|
215
|
+ |
rateCatalogVersion: options.rateCatalogVersion,
|
|
216
|
+ |
tasks: trials.map((trial) => trial.task).toSorted(),
|
|
217
|
+ |
agentVersions: distinct(trials.map((trial) => trial.agentVersion)),
|
|
218
|
+ |
models: distinct(trials.map((trial) => trial.modelId)),
|
|
219
|
+ |
});
|
|
220
|
+ |
return `effectiveness:${createHash("sha256").update(source).digest("hex")}`;
|
|
221
|
+ |
};
|
|
222
|
+ |
|
|
223
|
+ |
const distinct = (values: ReadonlyArray<string | null>): ReadonlyArray<string> =>
|
|
224
|
+ |
[...new Set(values.filter((value): value is string => value !== null))].toSorted();
|
|
225
|
+ |
|
|
226
|
+ |
const readJson = (path: string): unknown => {
|
|
227
|
+ |
if (!existsSync(path)) return undefined;
|
|
228
|
+ |
try {
|
|
229
|
+ |
return JSON.parse(readFileSync(path, "utf8")) as unknown;
|
|
230
|
+ |
} catch {
|
|
231
|
+ |
return undefined;
|
|
232
|
+ |
}
|
|
233
|
+ |
};
|
|
234
|
+ |
|
|
235
|
+ |
const readField = (value: unknown, key: string): unknown =>
|
|
236
|
+ |
typeof value === "object" && value !== null ? (value as Record<string, unknown>)[key] : undefined;
|
|
237
|
+ |
|
|
238
|
+ |
const readArray = (value: unknown): ReadonlyArray<unknown> => (Array.isArray(value) ? value : []);
|
|
239
|
+ |
|
|
240
|
+ |
const readString = (value: unknown): string | null =>
|
|
241
|
+ |
typeof value === "string" && value !== "" ? value : null;
|
|
242
|
+ |
|
|
243
|
+ |
const readNumber = (value: unknown): number | null =>
|
|
244
|
+ |
typeof value === "number" && Number.isFinite(value) ? value : null;
|