|
1
|
+ |
/**
|
|
2
|
+ |
* What one tool result may cost, per model family.
|
|
3
|
+ |
*
|
|
4
|
+ |
* A tool result is not paid for once. Every later round of the turn re-sends
|
|
5
|
+ |
* the whole transcript, so the result a command printed on round two is bought
|
|
6
|
+ |
* again on rounds three through fifteen. The measured bill for that is in the
|
|
7
|
+ |
* first graded Gym runs: a session that read the issue boards accumulated 82 KB
|
|
8
|
+ |
* of tool output and re-sent it 25 times, and 91% of everything it sent each
|
|
9
|
+ |
* round was output the model had already read
|
|
10
|
+ |
* (`openagents.com` docs/terminalbench/2026-08-24-fix-git-run-analysis.md).
|
|
11
|
+ |
*
|
|
12
|
+ |
* The bound on that used to be one number — 4,000 characters — written twice,
|
|
13
|
+ |
* once in the thread lane and once in the local lane, and applied to a 32k
|
|
14
|
+ |
* local model and to a million-token hosted one alike. A single number cannot
|
|
15
|
+ |
* be right for both: it is either most of a small model's window or a rounding
|
|
16
|
+ |
* error in a large one, and in neither case does it reflect what the family's
|
|
17
|
+ |
* tokenizer charges for those characters. So the budget is per family, it is
|
|
18
|
+ |
* stated in tokens, and it is converted to characters through that family's own
|
|
19
|
+ |
* density.
|
|
20
|
+ |
*
|
|
21
|
+ |
* ## The density figures are approximations, and say so
|
|
22
|
+ |
*
|
|
23
|
+ |
* Nothing here tokenizes. A real tokenizer per family is a dependency and a
|
|
24
|
+ |
* download for a decision that is a ceiling, not an accounting entry — the
|
|
25
|
+ |
* result is cut at a character count either way. What each family's figure is
|
|
26
|
+ |
* for is that tokenizers differ enough to matter: byte-pair vocabularies
|
|
27
|
+ |
* trained mostly on English prose land near four characters a token on prose
|
|
28
|
+ |
* and lower on the shell output, paths, and diffs a coding session actually
|
|
29
|
+ |
* reads. The numbers below are documented approximations, held low rather than
|
|
30
|
+ |
* high, so an error spends less of the window than the budget says rather than
|
|
31
|
+ |
* more. Replace one when a family is measured; do not read it as a measurement.
|
|
32
|
+ |
*
|
|
33
|
+ |
* ## Cutting is reported, never silent
|
|
34
|
+ |
*
|
|
35
|
+ |
* The caller of a tool is a model, and a model handed a quietly shortened
|
|
36
|
+ |
* `git log` will summarize it as though it were the whole log. So a cut result
|
|
37
|
+ |
* says it was cut, by how much, out of what, and against which family's budget
|
|
38
|
+ |
* — the fail-closed limit discipline in `INVARIANTS.md`: a cap may drop
|
|
39
|
+ |
* coverage, but it may never let an incomplete result read as complete.
|
|
40
|
+ |
*/
|
|
41
|
+ |
|
|
42
|
+ |
import type { ToolFamily } from "./coder-tool-families.js";
|
|
43
|
+ |
|
|
44
|
+ |
/** The figures a family's budget is derived from. */
|
|
45
|
+ |
interface FamilyBudget {
|
|
46
|
+ |
/** The family's context window, in tokens, as the lane advertises it. */
|
|
47
|
+ |
readonly contextWindowTokens: number;
|
|
48
|
+ |
/** What one tool result may take of that window, in tokens. */
|
|
49
|
+ |
readonly resultTokens: number;
|
|
50
|
+ |
/** Approximate characters per token for this family's tokenizer. */
|
|
51
|
+ |
readonly charactersPerToken: number;
|
|
52
|
+ |
/** Why this family's result allowance is what it is. */
|
|
53
|
+ |
readonly because: string;
|
|
54
|
+ |
}
|
|
55
|
+ |
|
|
56
|
+ |
/**
|
|
57
|
+ |
* The budgets, as data.
|
|
58
|
+ |
*
|
|
59
|
+ |
* Exhaustive over `ToolFamily` on purpose: adding a family to that union
|
|
60
|
+ |
* without deciding what it may spend does not compile. Each row carries the
|
|
61
|
+ |
* reason it holds the figure it does, because a number nobody can argue with
|
|
62
|
+ |
* is a number nobody will ever correct.
|
|
63
|
+ |
*/
|
|
64
|
+ |
const BUDGETS: Record<ToolFamily, FamilyBudget> = {
|
|
65
|
+ |
default: {
|
|
66
|
+ |
contextWindowTokens: 200_000,
|
|
67
|
+ |
resultTokens: 1_100,
|
|
68
|
+ |
// Byte-pair vocabularies of the o200k shape run near four characters a
|
|
69
|
+ |
// token on prose and lower on the command output a session reads. Held at
|
|
70
|
+ |
// 3.6 so the estimate errs toward spending less.
|
|
71
|
+ |
charactersPerToken: 3.6,
|
|
72
|
+ |
because:
|
|
73
|
+ |
"the hosted general lanes carry a large window, and 1,100 tokens is the " +
|
|
74
|
+ |
"measured allowance the shipped 4,000-character bound already amounted to",
|
|
75
|
+ |
},
|
|
76
|
+ |
gemini: {
|
|
77
|
+ |
contextWindowTokens: 1_000_000,
|
|
78
|
+ |
resultTokens: 700,
|
|
79
|
+ |
// Google documents roughly four characters a token for English on the
|
|
80
|
+ |
// SentencePiece vocabulary these lanes use.
|
|
81
|
+ |
charactersPerToken: 4,
|
|
82
|
+ |
because:
|
|
83
|
+ |
"the window is the largest of any lane and the round count is what costs: " +
|
|
84
|
+ |
"this family issued fifteen tool rounds where another issued six and spent " +
|
|
85
|
+ |
"three times the input tokens replaying whole-file dumps, so its results " +
|
|
86
|
+ |
"are cut sooner to make a narrower second read the cheaper move",
|
|
87
|
+ |
},
|
|
88
|
+ |
local: {
|
|
89
|
+ |
contextWindowTokens: 32_768,
|
|
90
|
+ |
resultTokens: 500,
|
|
91
|
+ |
// Qwen-shaped byte-pair vocabularies, again held low for code and paths.
|
|
92
|
+ |
charactersPerToken: 3.5,
|
|
93
|
+ |
because:
|
|
94
|
+ |
"the window is a fraction of a hosted one and generation is slow on one " +
|
|
95
|
+ |
"machine, so every re-sent character is paid in wall clock rather than in " +
|
|
96
|
+ |
"money",
|
|
97
|
+ |
},
|
|
98
|
+ |
};
|
|
99
|
+ |
|
|
100
|
+ |
/** The smallest budget on the table: what an unrecognized family is given. */
|
|
101
|
+ |
const mostConservative = (): ToolFamily => {
|
|
102
|
+ |
const families = Object.keys(BUDGETS) as ReadonlyArray<ToolFamily>;
|
|
103
|
+ |
let smallest: ToolFamily = "local";
|
|
104
|
+ |
for (const family of families) {
|
|
105
|
+ |
if (charactersOf(BUDGETS[family]) < charactersOf(BUDGETS[smallest])) smallest = family;
|
|
106
|
+ |
}
|
|
107
|
+ |
return smallest;
|
|
108
|
+ |
};
|
|
109
|
+ |
|
|
110
|
+ |
const charactersOf = (budget: FamilyBudget): number =>
|
|
111
|
+ |
Math.floor(budget.resultTokens * budget.charactersPerToken);
|
|
112
|
+ |
|
|
113
|
+ |
/** What one tool result may spend, resolved for one family. */
|
|
114
|
+ |
export interface ToolResultBudget {
|
|
115
|
+ |
/** The family the budget was asked for. */
|
|
116
|
+ |
readonly family: ToolFamily;
|
|
117
|
+ |
/** The ceiling the result is cut to. */
|
|
118
|
+ |
readonly characters: number;
|
|
119
|
+ |
/** What those characters are believed to cost. */
|
|
120
|
+ |
readonly tokens: number;
|
|
121
|
+ |
/** The approximation the two are related by. */
|
|
122
|
+ |
readonly charactersPerToken: number;
|
|
123
|
+ |
readonly contextWindowTokens: number;
|
|
124
|
+ |
/**
|
|
125
|
+ |
* True when the family had no row and the smallest budget was substituted.
|
|
126
|
+ |
*
|
|
127
|
+ |
* Carried rather than hidden: the notice on a cut result says so, because a
|
|
128
|
+ |
* budget that is a guess and a budget that is a decision are not the same
|
|
129
|
+ |
* claim.
|
|
130
|
+ |
*/
|
|
131
|
+ |
readonly substituted: boolean;
|
|
132
|
+ |
}
|
|
133
|
+ |
|
|
134
|
+ |
/**
|
|
135
|
+ |
* The budget for a family.
|
|
136
|
+ |
*
|
|
137
|
+ |
* A family with no row falls back to the smallest budget on the table rather
|
|
138
|
+ |
* than to a generous default. A family name reaches this from data — a lane
|
|
139
|
+ |
* name derived from a server catalog, a resumed session's record — so the case
|
|
140
|
+ |
* is reachable at runtime even though the union is exhaustive at compile time,
|
|
141
|
+ |
* and guessing high is the failure that is expensive.
|
|
142
|
+ |
*/
|
|
143
|
+ |
export const toolResultBudget = (family: ToolFamily): ToolResultBudget => {
|
|
144
|
+ |
const held = BUDGETS[family] as FamilyBudget | undefined;
|
|
145
|
+ |
const substituted = held === undefined;
|
|
146
|
+ |
const budget = held ?? BUDGETS[mostConservative()];
|
|
147
|
+ |
return {
|
|
148
|
+ |
family,
|
|
149
|
+ |
characters: charactersOf(budget),
|
|
150
|
+ |
tokens: budget.resultTokens,
|
|
151
|
+ |
charactersPerToken: budget.charactersPerToken,
|
|
152
|
+ |
contextWindowTokens: budget.contextWindowTokens,
|
|
153
|
+ |
substituted,
|
|
154
|
+ |
};
|
|
155
|
+ |
};
|
|
156
|
+ |
|
|
157
|
+ |
/** One line naming a family's allowance, for a reader asking what it is. */
|
|
158
|
+ |
export const describeBudget = (budget: ToolResultBudget): string =>
|
|
159
|
+ |
`Tool results are cut to ${String(budget.characters)} characters for the ` +
|
|
160
|
+ |
`${budget.family} model family: about ${String(budget.tokens)} tokens at an ` +
|
|
161
|
+ |
`approximate ${String(budget.charactersPerToken)} characters per token, against a ` +
|
|
162
|
+ |
`${String(budget.contextWindowTokens)}-token window` +
|
|
163
|
+ |
(budget.substituted
|
|
164
|
+ |
? " — the smallest budget on the table, substituted because that family has none of its own."
|
|
165
|
+ |
: ".");
|
|
166
|
+ |
|
|
167
|
+ |
/**
|
|
168
|
+ |
* One tool result as the model transcript carries it.
|
|
169
|
+ |
*
|
|
170
|
+ |
* Kept at both ends, which is what a long output is read for: a command's
|
|
171
|
+ |
* first lines say what it did and its last lines say how it ended, and the
|
|
172
|
+ |
* middle is the part a second, narrower run can recover. The notice in place of
|
|
173
|
+ |
* the middle is written for the model that has to decide what to do next, so it
|
|
174
|
+ |
* carries the arithmetic rather than the word "truncated".
|
|
175
|
+ |
*/
|
|
176
|
+ |
export const budgetedResult = (output: string, family: ToolFamily): string => {
|
|
177
|
+ |
const budget = toolResultBudget(family);
|
|
178
|
+ |
if (output.length <= budget.characters) return output;
|
|
179
|
+ |
|
|
180
|
+ |
const half = Math.floor(budget.characters / 2);
|
|
181
|
+ |
const omitted = output.length - budget.characters;
|
|
182
|
+ |
const notice =
|
|
183
|
+ |
`[${String(omitted)} of ${String(output.length)} characters omitted from the middle. ` +
|
|
184
|
+ |
`${describeBudget(budget)} ` +
|
|
185
|
+ |
"Every later round of this turn re-sends what you are reading, which is why the " +
|
|
186
|
+ |
"budget exists. What you have is incomplete and must not be summarized as if it " +
|
|
187
|
+ |
"were the whole answer: run it again more narrowly — a range, a filter, a count — " +
|
|
188
|
+ |
"if you need what is missing.]";
|
|
189
|
+ |
return `${output.slice(0, half)}\n\n${notice}\n\n${output.slice(-half)}`;
|
|
190
|
+ |
};
|