|
1
|
+ |
/**
|
|
2
|
+ |
* A reply source backed by a thread of the caller's own, and the grant that
|
|
3
|
+ |
* thread mints.
|
|
4
|
+ |
*
|
|
5
|
+ |
* `openagents coder` used to submit through `POST /api/v3/chat/turns` and poll
|
|
6
|
+ |
* `GET /api/v3/chat/events`. That was the only route a user token could reach a
|
|
7
|
+ |
* model through, and the server records one conversation per account, so every
|
|
8
|
+ |
* prompt a person typed in a terminal landed in the same conversation `/chat`
|
|
9
|
+ |
* reads, contended for the one streaming slot that conversation admits, and
|
|
10
|
+ |
* became provider context for the next question asked in the browser. This
|
|
11
|
+ |
* replaces both paths.
|
|
12
|
+ |
*
|
|
13
|
+ |
* `POST /api/v3/threads` opens a thread and returns a grant. The grant is the
|
|
14
|
+ |
* bearer for `POST /api/inference/proxy`, an OpenAI-compatible
|
|
15
|
+ |
* `/chat/completions` surface that meters against the thread's own budget and
|
|
16
|
+ |
* keeps the provider credential on the server, so the CLI still holds no
|
|
17
|
+ |
* provider key. `DELETE /api/v3/threads/{id}` revokes the thread on exit, which
|
|
18
|
+ |
* matters because an account may hold only eight open threads at once and a
|
|
19
|
+ |
* closed terminal would otherwise hold a slot until the authority expired.
|
|
20
|
+ |
*
|
|
21
|
+ |
* Three properties of that proxy shape this file, and each is a real loss
|
|
22
|
+ |
* against the event log this replaces:
|
|
23
|
+ |
*
|
|
24
|
+ |
* - **It answers in one piece.** The proxy builds the whole SSE body and sends
|
|
25
|
+ |
* it once, so the frames below all arrive together. The parser is written
|
|
26
|
+ |
* against the stream anyway rather than against `await response.text()`, so
|
|
27
|
+ |
* chunked delivery becomes visible here the day the server sends it, with no
|
|
28
|
+ |
* change on this side.
|
|
29
|
+ |
* - **It carries no reasoning.** `OpenAgents.Providers.ProviderEvent` has no
|
|
30
|
+ |
* reasoning member at all — the union is `response_started`, `text_delta`,
|
|
31
|
+ |
* `tool_call`, `usage`, `response_completed`, `failed`, `cancelled` — and the
|
|
32
|
+ |
* proxy drops everything it cannot name. The chat event log had
|
|
33
|
+ |
* `reasoning_delta`; nothing on this path does. So no `reasoning` chunk is
|
|
34
|
+ |
* ever produced here, and the interface's dim-italic reasoning entry, which
|
|
35
|
+ |
* the stand-in behind `--offline` still exercises, never appears against a
|
|
36
|
+ |
* live model.
|
|
37
|
+ |
* - **No tool runs.** The chat lane ran tools on the server and reported each
|
|
38
|
+ |
* one. The proxy is a bare completions surface: it forwards the `tools` a
|
|
39
|
+ |
* caller declares and returns the calls the model asks for, and the caller
|
|
40
|
+ |
* executes them. This CLI declares none and has no tool runtime, so the
|
|
41
|
+ |
* `tool_calls` translation below is the honest mapping of a frame that does
|
|
42
|
+ |
* not arrive today. It is kept because the frame is part of the surface and
|
|
43
|
+ |
* the alternative is discovering the mapping is missing on the day tools land.
|
|
44
|
+ |
*
|
|
45
|
+ |
* Nothing here announces any of that on screen. `2c15c6ed20` removed the
|
|
46
|
+ |
* `scopeNotice` seam with the reasoning that a session private to its own
|
|
47
|
+ |
* thread has nothing to announce, and a banner at the top of every session
|
|
48
|
+ |
* teaches the constraint rather than the design. The losses above are real and
|
|
49
|
+ |
* belong in the issue that decides what to do about them, not in a permanent
|
|
50
|
+ |
* line above the first prompt.
|
|
51
|
+ |
*
|
|
52
|
+ |
* The grant is a bearer credential. It is held `Redacted` so that an accidental
|
|
53
|
+ |
* interpolation prints a placeholder, it never reaches the transcript, and it
|
|
54
|
+ |
* is never passed as an argument to anything this process spawns.
|
|
55
|
+ |
*/
|
|
56
|
+ |
|
|
57
|
+ |
import { Redacted } from "effect";
|
|
58
|
+ |
|
|
59
|
+ |
import type { ReplyChunk, ReplySource } from "./coder-session.js";
|
|
60
|
+ |
|
|
61
|
+ |
const THREADS_PATH = "/api/v3/threads";
|
|
62
|
+ |
|
|
63
|
+ |
/** What the thread may still spend, as the server last reported it. */
|
|
64
|
+ |
export interface ThreadBudget {
|
|
65
|
+ |
readonly calls: number;
|
|
66
|
+ |
readonly totalTokens: number;
|
|
67
|
+ |
readonly costMicrousd: number;
|
|
68
|
+ |
}
|
|
69
|
+ |
|
|
70
|
+ |
export interface ThreadOptions {
|
|
71
|
+ |
readonly origin: string;
|
|
72
|
+ |
/** The account token. Opens, reads, and revokes the thread; never spends it. */
|
|
73
|
+ |
readonly token: string;
|
|
74
|
+ |
/** What this body of work is for. The server requires one. */
|
|
75
|
+ |
readonly objective: string;
|
|
76
|
+ |
/** Recorded on the thread as its admitted execution shape. */
|
|
77
|
+ |
readonly reasoning?: string | undefined;
|
|
78
|
+ |
}
|
|
79
|
+ |
|
|
80
|
+ |
export class ThreadUnavailable extends Error {
|
|
81
|
+ |
constructor(
|
|
82
|
+ |
readonly code: string,
|
|
83
|
+ |
message: string,
|
|
84
|
+ |
/** The HTTP status behind the code, or 0 when the request never landed. */
|
|
85
|
+ |
readonly status = 0,
|
|
86
|
+ |
) {
|
|
87
|
+ |
super(message);
|
|
88
|
+ |
this.name = "ThreadUnavailable";
|
|
89
|
+ |
}
|
|
90
|
+ |
}
|
|
91
|
+ |
|
|
92
|
+ |
/**
|
|
93
|
+ |
* Open a thread and take its grant.
|
|
94
|
+ |
*
|
|
95
|
+ |
* A refusal here is reported with the server's own code and sentence. The
|
|
96
|
+ |
* account cap is the one a person meets: the ninth concurrent session is
|
|
97
|
+ |
* refused `thread_quota_reached` with a message naming the limit and how many
|
|
98
|
+ |
* threads the account is holding, which is what tells them to close one rather
|
|
99
|
+ |
* than to retry.
|
|
100
|
+ |
*/
|
|
101
|
+ |
export async function openThread(options: ThreadOptions): Promise<ThreadReplySource> {
|
|
102
|
+ |
const response = await fetch(new URL(THREADS_PATH, options.origin), {
|
|
103
|
+ |
method: "POST",
|
|
104
|
+ |
headers: {
|
|
105
|
+ |
authorization: `Bearer ${options.token}`,
|
|
106
|
+ |
"content-type": "application/json",
|
|
107
|
+ |
accept: "application/json",
|
|
108
|
+ |
},
|
|
109
|
+ |
body: JSON.stringify({
|
|
110
|
+ |
objective: options.objective,
|
|
111
|
+ |
...(options.reasoning === undefined ? {} : { reasoning: options.reasoning }),
|
|
112
|
+ |
}),
|
|
113
|
+ |
}).catch((cause: unknown) => {
|
|
114
|
+ |
throw new ThreadUnavailable(
|
|
115
|
+ |
"network_refused",
|
|
116
|
+ |
`The API at ${options.origin} could not be reached: ${String(cause)}`,
|
|
117
|
+ |
);
|
|
118
|
+ |
});
|
|
119
|
+ |
|
|
120
|
+ |
const body = (await response.json().catch(() => ({}))) as Record<string, unknown>;
|
|
121
|
+ |
|
|
122
|
+ |
if (response.status === 401 || response.status === 403) {
|
|
123
|
+ |
throw new ThreadUnavailable(
|
|
124
|
+ |
"scope_missing",
|
|
125
|
+ |
"This token cannot open a thread. Sign in again with the chat:account scope.",
|
|
126
|
+ |
response.status,
|
|
127
|
+ |
);
|
|
128
|
+ |
}
|
|
129
|
+ |
if (response.status < 200 || response.status >= 300) {
|
|
130
|
+ |
// The envelope names the code and the sentence. Passing both through is
|
|
131
|
+ |
// what turns a ninth session from an obscure failure into an instruction.
|
|
132
|
+ |
const code = typeof body["code"] === "string" ? body["code"] : `http_${response.status}`;
|
|
133
|
+ |
const message =
|
|
134
|
+ |
typeof body["message"] === "string"
|
|
135
|
+ |
? body["message"]
|
|
136
|
+ |
: `The server refused to open a thread (${code}).`;
|
|
137
|
+ |
throw new ThreadUnavailable(code, message, response.status);
|
|
138
|
+ |
}
|
|
139
|
+ |
|
|
140
|
+ |
const thread = record(body["thread"]);
|
|
141
|
+ |
const grant = record(body["grant"]);
|
|
142
|
+ |
const id = string(thread["id"]);
|
|
143
|
+ |
const token = string(grant["token"]);
|
|
144
|
+ |
const url = string(grant["url"]);
|
|
145
|
+ |
const model = string(grant["model"]);
|
|
146
|
+ |
|
|
147
|
+ |
if (id === undefined || token === undefined || url === undefined || model === undefined) {
|
|
148
|
+ |
throw new ThreadUnavailable(
|
|
149
|
+ |
"malformed_thread",
|
|
150
|
+ |
"The server opened a thread but did not return the grant needed to spend it.",
|
|
151
|
+ |
);
|
|
152
|
+ |
}
|
|
153
|
+ |
|
|
154
|
+ |
return new ThreadReplySource({
|
|
155
|
+ |
origin: options.origin,
|
|
156
|
+ |
accountToken: options.token,
|
|
157
|
+ |
threadId: id,
|
|
158
|
+ |
grantToken: Redacted.make(token),
|
|
159
|
+ |
proxyUrl: url,
|
|
160
|
+ |
model,
|
|
161
|
+ |
budget: budgetOf(record(grant["limits"]), record(grant["limits"])),
|
|
162
|
+ |
});
|
|
163
|
+ |
}
|
|
164
|
+ |
|
|
165
|
+ |
interface SourceState {
|
|
166
|
+ |
readonly origin: string;
|
|
167
|
+ |
readonly accountToken: string;
|
|
168
|
+ |
readonly threadId: string;
|
|
169
|
+ |
readonly grantToken: Redacted.Redacted<string>;
|
|
170
|
+ |
readonly proxyUrl: string;
|
|
171
|
+ |
readonly model: string;
|
|
172
|
+ |
readonly budget: ThreadBudget;
|
|
173
|
+ |
}
|
|
174
|
+ |
|
|
175
|
+ |
/** One chat-completions message, which is what the proxy takes as its input. */
|
|
176
|
+ |
interface WireMessage {
|
|
177
|
+ |
readonly role: "user" | "assistant";
|
|
178
|
+ |
readonly content: string;
|
|
179
|
+ |
}
|
|
180
|
+ |
|
|
181
|
+ |
export class ThreadReplySource implements ReplySource {
|
|
182
|
+ |
readonly threadId: string;
|
|
183
|
+ |
/**
|
|
184
|
+ |
* The thread's transcript, keyed on the thread by construction: this array
|
|
185
|
+ |
* exists only inside the source that holds that thread's grant, so the
|
|
186
|
+ |
* context a turn is answered against is the thread's and nothing else's.
|
|
187
|
+ |
* The account conversation is not read and not written.
|
|
188
|
+ |
*/
|
|
189
|
+ |
private readonly transcript: WireMessage[] = [];
|
|
190
|
+ |
private remaining: ThreadBudget;
|
|
191
|
+ |
|
|
192
|
+ |
constructor(private readonly state: SourceState) {
|
|
193
|
+ |
this.threadId = state.threadId;
|
|
194
|
+ |
this.remaining = state.budget;
|
|
195
|
+ |
}
|
|
196
|
+ |
|
|
197
|
+ |
/**
|
|
198
|
+ |
* The model the grant pins.
|
|
199
|
+ |
*
|
|
200
|
+ |
* Not a backend the client chose. The proxy takes the model from the grant so
|
|
201
|
+ |
* a request body cannot select another, and the thread route deliberately
|
|
202
|
+ |
* publishes no model parameter, so this is the one name that is true of the
|
|
203
|
+ |
* reply on screen.
|
|
204
|
+ |
*/
|
|
205
|
+ |
get model(): string {
|
|
206
|
+ |
return this.state.model;
|
|
207
|
+ |
}
|
|
208
|
+ |
|
|
209
|
+ |
/** What is left to spend, in the width a status line has for it. */
|
|
210
|
+ |
get budget(): string {
|
|
211
|
+ |
return formatBudget(this.remaining);
|
|
212
|
+ |
}
|
|
213
|
+ |
|
|
214
|
+ |
async *reply(prompt: string, signal: AbortSignal): AsyncIterable<ReplyChunk> {
|
|
215
|
+ |
this.transcript.push({ role: "user", content: prompt });
|
|
216
|
+ |
|
|
217
|
+ |
let assistant = "";
|
|
218
|
+ |
try {
|
|
219
|
+ |
for await (const chunk of this.stream(signal)) {
|
|
220
|
+ |
if (signal.aborted) break;
|
|
221
|
+ |
if (chunk.type === "text") assistant += chunk.value;
|
|
222
|
+ |
yield chunk;
|
|
223
|
+ |
}
|
|
224
|
+ |
} finally {
|
|
225
|
+ |
// Whatever the model said belongs to the thread even when the turn was
|
|
226
|
+ |
// interrupted, or the next turn answers a question it cannot see it
|
|
227
|
+ |
// half-answered.
|
|
228
|
+ |
if (assistant.length > 0) this.transcript.push({ role: "assistant", content: assistant });
|
|
229
|
+ |
// Read the budget on the way out of every turn, including an interrupted
|
|
230
|
+ |
// one. Interrupting is a client-side abort: the proxy had already bought
|
|
231
|
+ |
// the call and metered it, so a status line that kept the figure it
|
|
232
|
+ |
// opened with would under-report the spend by exactly the turns a reader
|
|
233
|
+ |
// cut short.
|
|
234
|
+ |
await this.refresh();
|
|
235
|
+ |
}
|
|
236
|
+ |
}
|
|
237
|
+ |
|
|
238
|
+ |
/**
|
|
239
|
+ |
* Revoke the thread and its grant.
|
|
240
|
+ |
*
|
|
241
|
+ |
* Best effort by design: this runs while the process is leaving, and a
|
|
242
|
+ |
* network failure on the way out must not turn a finished session into an
|
|
243
|
+ |
* error. The server retires elapsed authority on its own, so the worst case
|
|
244
|
+ |
* of a failed revoke is a slot held until the thread expires rather than one
|
|
245
|
+ |
* held forever.
|
|
246
|
+ |
*/
|
|
247
|
+ |
async revoke(): Promise<void> {
|
|
248
|
+ |
await fetch(new URL(`${THREADS_PATH}/${this.state.threadId}`, this.state.origin), {
|
|
249
|
+ |
method: "DELETE",
|
|
250
|
+ |
headers: {
|
|
251
|
+ |
authorization: `Bearer ${this.state.accountToken}`,
|
|
252
|
+ |
accept: "application/json",
|
|
253
|
+ |
},
|
|
254
|
+ |
}).catch(() => undefined);
|
|
255
|
+ |
}
|
|
256
|
+ |
|
|
257
|
+ |
/** Spend one call against the proxy and translate what comes back. */
|
|
258
|
+ |
private async *stream(signal: AbortSignal): AsyncIterable<ReplyChunk> {
|
|
259
|
+ |
const response = await fetch(this.state.proxyUrl, {
|
|
260
|
+ |
method: "POST",
|
|
261
|
+ |
signal,
|
|
262
|
+ |
headers: {
|
|
263
|
+ |
authorization: `Bearer ${Redacted.value(this.state.grantToken)}`,
|
|
264
|
+ |
"content-type": "application/json",
|
|
265
|
+ |
// The body is an event stream and the refusals are JSON, and both have
|
|
266
|
+ |
// to be acceptable: the `:api` pipeline negotiates on `json` and
|
|
267
|
+ |
// answers `406` to a request that will only take `text/event-stream`.
|
|
268
|
+ |
accept: "text/event-stream, application/json",
|
|
269
|
+ |
},
|
|
270
|
+ |
body: JSON.stringify({
|
|
271
|
+ |
model: this.state.model,
|
|
272
|
+ |
stream: true,
|
|
273
|
+ |
messages: this.transcript,
|
|
274
|
+ |
}),
|
|
275
|
+ |
}).catch((cause: unknown) => {
|
|
276
|
+ |
if (signal.aborted) return undefined;
|
|
277
|
+ |
throw new ThreadUnavailable(
|
|
278
|
+ |
"network_refused",
|
|
279
|
+ |
`The inference proxy could not be reached: ${String(cause)}`,
|
|
280
|
+ |
);
|
|
281
|
+ |
});
|
|
282
|
+ |
|
|
283
|
+ |
if (response === undefined || signal.aborted) return;
|
|
284
|
+ |
if (response.status < 200 || response.status >= 300) {
|
|
285
|
+ |
throw await proxyRefusal(response);
|
|
286
|
+ |
}
|
|
287
|
+ |
if (response.body === null) return;
|
|
288
|
+ |
|
|
289
|
+ |
/** Tool call fragments by their wire index, assembled as frames arrive. */
|
|
290
|
+ |
const calls = new Map<number, { id: string; name: string; args: string }>();
|
|
291
|
+ |
|
|
292
|
+ |
for await (const frame of frames(response.body, signal)) {
|
|
293
|
+ |
if (signal.aborted) return;
|
|
294
|
+ |
if (frame === "[DONE]") break;
|
|
295
|
+ |
|
|
296
|
+ |
const payload = parse(frame);
|
|
297
|
+ |
if (payload === undefined) continue;
|
|
298
|
+ |
|
|
299
|
+ |
const usage = record(payload["usage"]);
|
|
300
|
+ |
if (Object.keys(usage).length > 0) this.spend(usage);
|
|
301
|
+ |
|
|
302
|
+ |
const choices = payload["choices"];
|
|
303
|
+ |
if (!Array.isArray(choices)) continue;
|
|
304
|
+ |
|
|
305
|
+ |
for (const choice of choices) {
|
|
306
|
+ |
const delta = record(record(choice)["delta"]);
|
|
307
|
+ |
|
|
308
|
+ |
const content = delta["content"];
|
|
309
|
+ |
if (typeof content === "string" && content.length > 0) {
|
|
310
|
+ |
yield { type: "text", value: content };
|
|
311
|
+ |
}
|
|
312
|
+ |
|
|
313
|
+ |
const toolCalls = delta["tool_calls"];
|
|
314
|
+ |
if (Array.isArray(toolCalls)) accumulate(calls, toolCalls);
|
|
315
|
+ |
}
|
|
316
|
+ |
}
|
|
317
|
+ |
|
|
318
|
+ |
for (const call of calls.values()) {
|
|
319
|
+ |
yield { type: "tool_call", callId: call.id, name: call.name, arguments: call.args };
|
|
320
|
+ |
}
|
|
321
|
+ |
}
|
|
322
|
+ |
|
|
323
|
+ |
/**
|
|
324
|
+ |
* Take the turn's own usage off the budget immediately.
|
|
325
|
+ |
*
|
|
326
|
+ |
* The authoritative numbers come from the server a moment later, but a status
|
|
327
|
+ |
* line that only moves after a second request would show a stale budget for
|
|
328
|
+ |
* exactly as long as the reader is looking at the reply that spent it.
|
|
329
|
+ |
*/
|
|
330
|
+ |
private spend(usage: Record<string, unknown>): void {
|
|
331
|
+ |
const total = number(usage["total_tokens"]);
|
|
332
|
+ |
this.remaining = {
|
|
333
|
+ |
calls: Math.max(0, this.remaining.calls - 1),
|
|
334
|
+ |
totalTokens: Math.max(0, this.remaining.totalTokens - total),
|
|
335
|
+ |
costMicrousd: this.remaining.costMicrousd,
|
|
336
|
+ |
};
|
|
337
|
+ |
}
|
|
338
|
+ |
|
|
339
|
+ |
/** Read what the server says the thread has left. Failure keeps the estimate. */
|
|
340
|
+ |
private async refresh(): Promise<void> {
|
|
341
|
+ |
const response = await fetch(
|
|
342
|
+ |
new URL(`${THREADS_PATH}/${this.state.threadId}`, this.state.origin),
|
|
343
|
+ |
{
|
|
344
|
+ |
headers: {
|
|
345
|
+ |
authorization: `Bearer ${this.state.accountToken}`,
|
|
346
|
+ |
accept: "application/json",
|
|
347
|
+ |
},
|
|
348
|
+ |
},
|
|
349
|
+ |
).catch(() => undefined);
|
|
350
|
+ |
|
|
351
|
+ |
if (response === undefined || response.status < 200 || response.status >= 300) return;
|
|
352
|
+ |
const body = (await response.json().catch(() => ({}))) as Record<string, unknown>;
|
|
353
|
+ |
const grant = record(body["grant"]);
|
|
354
|
+ |
const remaining = record(grant["remaining"]);
|
|
355
|
+ |
if (Object.keys(remaining).length === 0) return;
|
|
356
|
+ |
this.remaining = budgetOf(remaining, record(grant["limits"]));
|
|
357
|
+ |
}
|
|
358
|
+ |
}
|
|
359
|
+ |
|
|
360
|
+ |
/** Frames of an SSE body, yielded as the body arrives rather than after it. */
|
|
361
|
+ |
async function* frames(
|
|
362
|
+ |
body: ReadableStream<Uint8Array>,
|
|
363
|
+ |
signal: AbortSignal,
|
|
364
|
+ |
): AsyncIterable<string> {
|
|
365
|
+ |
const reader = body.getReader();
|
|
366
|
+ |
const decoder = new TextDecoder();
|
|
367
|
+ |
let buffer = "";
|
|
368
|
+ |
|
|
369
|
+ |
try {
|
|
370
|
+ |
for (;;) {
|
|
371
|
+ |
// A stream is read in order and each read depends on the one before it,
|
|
372
|
+ |
// so there is no set of promises here to run together.
|
|
373
|
+ |
// eslint-disable-next-line no-await-in-loop
|
|
374
|
+ |
const { done, value } = await reader.read();
|
|
375
|
+ |
if (done || signal.aborted) break;
|
|
376
|
+ |
buffer += decoder.decode(value, { stream: true });
|
|
377
|
+ |
|
|
378
|
+ |
for (;;) {
|
|
379
|
+ |
const boundary = buffer.indexOf("\n\n");
|
|
380
|
+ |
if (boundary < 0) break;
|
|
381
|
+ |
const frame = buffer.slice(0, boundary);
|
|
382
|
+ |
buffer = buffer.slice(boundary + 2);
|
|
383
|
+ |
const data = dataOf(frame);
|
|
384
|
+ |
if (data !== undefined) yield data;
|
|
385
|
+ |
}
|
|
386
|
+ |
}
|
|
387
|
+ |
} finally {
|
|
388
|
+ |
reader.releaseLock();
|
|
389
|
+ |
}
|
|
390
|
+ |
}
|
|
391
|
+ |
|
|
392
|
+ |
/** The `data:` payload of one frame, or nothing for a comment or a keep-alive. */
|
|
393
|
+ |
function dataOf(frame: string): string | undefined {
|
|
394
|
+ |
const lines = frame.split("\n");
|
|
395
|
+ |
const parts: string[] = [];
|
|
396
|
+ |
for (const line of lines) {
|
|
397
|
+ |
const trimmed = line.endsWith("\r") ? line.slice(0, -1) : line;
|
|
398
|
+ |
if (!trimmed.startsWith("data:")) continue;
|
|
399
|
+ |
parts.push(trimmed.slice(5).trimStart());
|
|
400
|
+ |
}
|
|
401
|
+ |
return parts.length === 0 ? undefined : parts.join("\n");
|
|
402
|
+ |
}
|
|
403
|
+ |
|
|
404
|
+ |
function parse(frame: string): Record<string, unknown> | undefined {
|
|
405
|
+ |
try {
|
|
406
|
+ |
const value: unknown = JSON.parse(frame);
|
|
407
|
+ |
return typeof value === "object" && value !== null
|
|
408
|
+ |
? (value as Record<string, unknown>)
|
|
409
|
+ |
: undefined;
|
|
410
|
+ |
} catch {
|
|
411
|
+ |
return undefined;
|
|
412
|
+ |
}
|
|
413
|
+ |
}
|
|
414
|
+ |
|
|
415
|
+ |
/**
|
|
416
|
+ |
* Fold `tool_calls` fragments into whole calls.
|
|
417
|
+ |
*
|
|
418
|
+ |
* Chat-completions splits one call across frames and identifies the pieces by
|
|
419
|
+ |
* `index`, so a name and its arguments can arrive separately.
|
|
420
|
+ |
*/
|
|
421
|
+ |
function accumulate(
|
|
422
|
+ |
calls: Map<number, { id: string; name: string; args: string }>,
|
|
423
|
+ |
fragments: ReadonlyArray<unknown>,
|
|
424
|
+ |
): void {
|
|
425
|
+ |
for (const fragment of fragments) {
|
|
426
|
+ |
const piece = record(fragment);
|
|
427
|
+ |
const index = number(piece["index"]);
|
|
428
|
+ |
const current = calls.get(index) ?? { id: "", name: "tool", args: "" };
|
|
429
|
+ |
const fn = record(piece["function"]);
|
|
430
|
+ |
|
|
431
|
+ |
calls.set(index, {
|
|
432
|
+ |
id: string(piece["id"]) ?? current.id,
|
|
433
|
+ |
name: string(fn["name"]) ?? current.name,
|
|
434
|
+ |
args: current.args + (string(fn["arguments"]) ?? ""),
|
|
435
|
+ |
});
|
|
436
|
+ |
}
|
|
437
|
+ |
}
|
|
438
|
+ |
|
|
439
|
+ |
/** The proxy's typed refusal, turned into a sentence a reader can act on. */
|
|
440
|
+ |
async function proxyRefusal(response: Response): Promise<ThreadUnavailable> {
|
|
441
|
+ |
const body = (await response.json().catch(() => ({}))) as Record<string, unknown>;
|
|
442
|
+ |
const code = string(record(body["error"])["code"]) ?? `http_${response.status}`;
|
|
443
|
+ |
|
|
444
|
+ |
const sentences: Record<string, string> = {
|
|
445
|
+ |
grant_revoked: "This thread was revoked. Start a new session to open another.",
|
|
446
|
+ |
grant_expired: "This thread's authority expired. Start a new session to open another.",
|
|
447
|
+ |
grant_exhausted: "This thread spent its budget. Start a new session to open another.",
|
|
448
|
+ |
grant_budget_reached: "This thread reached its budget ceiling and cannot buy another call.",
|
|
449
|
+ |
invalid_grant: "The inference proxy did not recognize this thread's grant.",
|
|
450
|
+ |
provider_failed: "The model provider failed. The call was not completed.",
|
|
451
|
+ |
};
|
|
452
|
+ |
|
|
453
|
+ |
return new ThreadUnavailable(
|
|
454
|
+ |
code,
|
|
455
|
+ |
sentences[code] ?? `The inference proxy refused the call (${code}).`,
|
|
456
|
+ |
response.status,
|
|
457
|
+ |
);
|
|
458
|
+ |
}
|
|
459
|
+ |
|
|
460
|
+ |
/**
|
|
461
|
+ |
* The budget, read from `remaining` when the server has reported one and from
|
|
462
|
+ |
* `limits` at the moment of minting, when nothing has been spent yet.
|
|
463
|
+ |
*/
|
|
464
|
+ |
function budgetOf(
|
|
465
|
+ |
remaining: Record<string, unknown>,
|
|
466
|
+ |
limits: Record<string, unknown>,
|
|
467
|
+ |
): ThreadBudget {
|
|
468
|
+ |
return {
|
|
469
|
+ |
calls: number(remaining["calls"] ?? limits["max_calls"]),
|
|
470
|
+ |
totalTokens: number(remaining["total_tokens"] ?? limits["max_total_tokens"]),
|
|
471
|
+ |
costMicrousd: number(remaining["cost_microusd"] ?? limits["max_cost_microusd"]),
|
|
472
|
+ |
};
|
|
473
|
+ |
}
|
|
474
|
+ |
|
|
475
|
+ |
/**
|
|
476
|
+ |
* The budget in the width a status line has for it.
|
|
477
|
+ |
*
|
|
478
|
+ |
* An agent that exhausts its budget mid-edit without ever having shown one is
|
|
479
|
+ |
* an agent that lost the work, so all three ceilings are named: the call count
|
|
480
|
+ |
* is what usually runs out first, and the other two are what a long turn or an
|
|
481
|
+ |
* expensive model runs into instead.
|
|
482
|
+ |
*/
|
|
483
|
+ |
export function formatBudget(budget: ThreadBudget): string {
|
|
484
|
+ |
return `${budget.calls} calls · ${compact(budget.totalTokens)} tok · ${dollars(budget.costMicrousd)}`;
|
|
485
|
+ |
}
|
|
486
|
+ |
|
|
487
|
+ |
function compact(tokens: number): string {
|
|
488
|
+ |
// The threshold is where the K form would round to four digits, so a ceiling
|
|
489
|
+ |
// of a million reads `1.0M` before a turn is spent and `1.0M` after it.
|
|
490
|
+ |
if (tokens >= 999_500) return `${(tokens / 1_000_000).toFixed(1)}M`;
|
|
491
|
+ |
if (tokens >= 1_000) return `${Math.round(tokens / 1_000)}K`;
|
|
492
|
+ |
return `${tokens}`;
|
|
493
|
+ |
}
|
|
494
|
+ |
|
|
495
|
+ |
function dollars(microusd: number): string {
|
|
496
|
+ |
return `$${(microusd / 1_000_000).toFixed(2)}`;
|
|
497
|
+ |
}
|
|
498
|
+ |
|
|
499
|
+ |
function record(value: unknown): Record<string, unknown> {
|
|
500
|
+ |
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
501
|
+ |
? (value as Record<string, unknown>)
|
|
502
|
+ |
: {};
|
|
503
|
+ |
}
|
|
504
|
+ |
|
|
505
|
+ |
function string(value: unknown): string | undefined {
|
|
506
|
+ |
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
507
|
+ |
}
|
|
508
|
+ |
|
|
509
|
+ |
function number(value: unknown): number {
|
|
510
|
+ |
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
511
|
+ |
}
|