|
1
|
+ |
/**
|
|
2
|
+ |
* A local OpenAI-compatible endpoint that lends the session's thread grant to
|
|
3
|
+ |
* child coding agents.
|
|
4
|
+ |
*
|
|
5
|
+ |
* Delegation used to require the reader to name a child model and to hold a
|
|
6
|
+ |
* provider credential of their own, which meant the fleet was off by default
|
|
7
|
+ |
* and `/delegate` in a fresh session did nothing. That was backwards: the
|
|
8
|
+ |
* session already holds a grant the server minted for it, the grant is already
|
|
9
|
+ |
* metered against the thread's own budget, and a child is the same kind of
|
|
10
|
+ |
* spend as a reply. So children run on the session's grant, and delegation
|
|
11
|
+ |
* needs no flags, no key, and no second account.
|
|
12
|
+ |
*
|
|
13
|
+ |
* A harness cannot talk to `POST /api/inference/proxy` directly, for two
|
|
14
|
+ |
* reasons:
|
|
15
|
+ |
*
|
|
16
|
+ |
* - **Path.** An OpenAI-compatible client appends `/chat/completions` to its
|
|
17
|
+ |
* base URL, and the proxy is one fixed path that answers nothing else.
|
|
18
|
+ |
* - **Tool history.** The proxy maps a `tool` message to a
|
|
19
|
+ |
* `function_call_output` item and sends it *alone*, which the provider
|
|
20
|
+ |
* refuses without the `function_call` that preceded it — and the response id
|
|
21
|
+ |
* that would link them is never given to a client. A coding agent calls a
|
|
22
|
+ |
* tool on nearly every step, so the second step of every child would fail.
|
|
23
|
+ |
* This gateway therefore flattens a tool exchange into plain turns: the
|
|
24
|
+ |
* assistant's call becomes assistant text naming the call, and the result
|
|
25
|
+ |
* becomes a user turn carrying the output. The harness keeps its own state
|
|
26
|
+ |
* machine and only needs the next call from the model, so a flattened history
|
|
27
|
+ |
* costs prompt-shape fidelity and nothing else.
|
|
28
|
+ |
*
|
|
29
|
+ |
* The grant never reaches the harness: it is held here, the child is told only
|
|
30
|
+ |
* a loopback URL, and the placeholder key it sends back is ignored. That is why
|
|
31
|
+ |
* the gateway binds to `127.0.0.1` and to a port the operating system picks —
|
|
32
|
+ |
* anything on the box could otherwise spend the thread's budget, and a fixed
|
|
33
|
+ |
* port would collide between two sessions.
|
|
34
|
+ |
*/
|
|
35
|
+ |
|
|
36
|
+ |
import { createServer } from "node:http";
|
|
37
|
+ |
import type { AddressInfo } from "node:net";
|
|
38
|
+ |
import { Redacted } from "effect";
|
|
39
|
+ |
|
|
40
|
+ |
/** What the child gateway needs in order to spend a thread's grant. */
|
|
41
|
+ |
export interface ChildGrant {
|
|
42
|
+ |
/** The proxy URL the grant was minted for. */
|
|
43
|
+ |
readonly proxyUrl: string;
|
|
44
|
+ |
readonly token: Redacted.Redacted<string>;
|
|
45
|
+ |
/** The model the grant pins. A request body cannot select another. */
|
|
46
|
+ |
readonly model: string;
|
|
47
|
+ |
}
|
|
48
|
+ |
|
|
49
|
+ |
/** One chat-completions message as a harness sends it. */
|
|
50
|
+ |
interface ClientMessage {
|
|
51
|
+ |
readonly role?: unknown;
|
|
52
|
+ |
readonly content?: unknown;
|
|
53
|
+ |
readonly tool_calls?: unknown;
|
|
54
|
+ |
readonly tool_call_id?: unknown;
|
|
55
|
+ |
}
|
|
56
|
+ |
|
|
57
|
+ |
/** A turn the proxy accepts: a role and text, and nothing else. */
|
|
58
|
+ |
interface FlatMessage {
|
|
59
|
+ |
readonly role: "system" | "user" | "assistant";
|
|
60
|
+ |
readonly content: string;
|
|
61
|
+ |
}
|
|
62
|
+ |
|
|
63
|
+ |
/**
|
|
64
|
+ |
* Flatten a harness's message list into turns the proxy accepts.
|
|
65
|
+ |
*
|
|
66
|
+ |
* Exported for the tests, which is the only way to check the mapping without
|
|
67
|
+ |
* standing up a server and a child.
|
|
68
|
+ |
*/
|
|
69
|
+ |
export function flattenForProxy(messages: ReadonlyArray<unknown>): ReadonlyArray<FlatMessage> {
|
|
70
|
+ |
const flat: FlatMessage[] = [];
|
|
71
|
+ |
|
|
72
|
+ |
for (const raw of messages) {
|
|
73
|
+ |
if (typeof raw !== "object" || raw === null) continue;
|
|
74
|
+ |
const message = raw as ClientMessage;
|
|
75
|
+ |
const text = textOf(message.content);
|
|
76
|
+ |
const role = typeof message.role === "string" ? message.role : "user";
|
|
77
|
+ |
|
|
78
|
+ |
if (role === "tool") {
|
|
79
|
+ |
const id = typeof message.tool_call_id === "string" ? message.tool_call_id : "";
|
|
80
|
+ |
// A result is put on a user turn because that is the only role the proxy
|
|
81
|
+ |
// will carry text on that the model has not already spoken.
|
|
82
|
+ |
flat.push({ role: "user", content: `[tool result${id === "" ? "" : ` ${id}`}]\n${text}` });
|
|
83
|
+ |
continue;
|
|
84
|
+ |
}
|
|
85
|
+ |
|
|
86
|
+ |
const calls = describeCalls(message.tool_calls);
|
|
87
|
+ |
if (role === "assistant" && calls !== undefined) {
|
|
88
|
+ |
flat.push({ role: "assistant", content: `${text}\n[tool call]\n${calls}`.trim() });
|
|
89
|
+ |
continue;
|
|
90
|
+ |
}
|
|
91
|
+ |
|
|
92
|
+ |
// An empty turn is dropped rather than sent: the proxy refuses a request
|
|
93
|
+ |
// whose input is empty, and a blank assistant turn is what a harness emits
|
|
94
|
+ |
// around a call it has already described.
|
|
95
|
+ |
if (text.trim().length === 0) continue;
|
|
96
|
+ |
flat.push({
|
|
97
|
+ |
role: role === "system" ? "system" : role === "assistant" ? "assistant" : "user",
|
|
98
|
+ |
content: text,
|
|
99
|
+ |
});
|
|
100
|
+ |
}
|
|
101
|
+ |
|
|
102
|
+ |
return flat;
|
|
103
|
+ |
}
|
|
104
|
+ |
|
|
105
|
+ |
/** The text of a message whose content may be a string or a part array. */
|
|
106
|
+ |
function textOf(content: unknown): string {
|
|
107
|
+ |
if (typeof content === "string") return content;
|
|
108
|
+ |
if (!Array.isArray(content)) return "";
|
|
109
|
+ |
return content
|
|
110
|
+ |
.map((part) => {
|
|
111
|
+ |
if (typeof part === "string") return part;
|
|
112
|
+ |
if (typeof part === "object" && part !== null) {
|
|
113
|
+ |
const text = (part as { text?: unknown }).text;
|
|
114
|
+ |
return typeof text === "string" ? text : "";
|
|
115
|
+ |
}
|
|
116
|
+ |
return "";
|
|
117
|
+ |
})
|
|
118
|
+ |
.join("");
|
|
119
|
+ |
}
|
|
120
|
+ |
|
|
121
|
+ |
/** The calls on an assistant message, as one line each, or nothing. */
|
|
122
|
+ |
function describeCalls(toolCalls: unknown): string | undefined {
|
|
123
|
+ |
if (!Array.isArray(toolCalls) || toolCalls.length === 0) return undefined;
|
|
124
|
+ |
const lines: string[] = [];
|
|
125
|
+ |
for (const raw of toolCalls) {
|
|
126
|
+ |
if (typeof raw !== "object" || raw === null) continue;
|
|
127
|
+ |
const fn = (raw as { function?: { name?: unknown; arguments?: unknown } }).function ?? {};
|
|
128
|
+ |
const id = (raw as { id?: unknown }).id;
|
|
129
|
+ |
const name = typeof fn.name === "string" ? fn.name : "tool";
|
|
130
|
+ |
const args = typeof fn.arguments === "string" ? fn.arguments : "";
|
|
131
|
+ |
const suffix = typeof id === "string" && id.length > 0 ? ` id=${id}` : "";
|
|
132
|
+ |
lines.push(`${name}(${args})${suffix}`);
|
|
133
|
+ |
}
|
|
134
|
+ |
return lines.length === 0 ? undefined : lines.join("\n");
|
|
135
|
+ |
}
|
|
136
|
+ |
|
|
137
|
+ |
/** A running gateway: where children should send their calls, and how to stop. */
|
|
138
|
+ |
export interface ChildGateway {
|
|
139
|
+ |
/** The base URL a harness config points at, without a trailing slash. */
|
|
140
|
+ |
readonly baseUrl: string;
|
|
141
|
+ |
/** The model id, as the harness must name it: `provider/model`. */
|
|
142
|
+ |
readonly modelId: string;
|
|
143
|
+ |
close(): Promise<void>;
|
|
144
|
+ |
}
|
|
145
|
+ |
|
|
146
|
+ |
/** The provider name children see. Part of the model id they are given. */
|
|
147
|
+ |
export const CHILD_PROVIDER = "openagents";
|
|
148
|
+ |
|
|
149
|
+ |
/**
|
|
150
|
+ |
* Start the gateway on a loopback port of the operating system's choosing.
|
|
151
|
+ |
*
|
|
152
|
+ |
* Resolves once it is listening, because a child launched against a port that
|
|
153
|
+ |
* is not up yet fails on its first call and reports it as a provider error.
|
|
154
|
+ |
*/
|
|
155
|
+ |
export async function startChildGateway(grant: ChildGrant): Promise<ChildGateway> {
|
|
156
|
+ |
const server = createServer((request, response) => {
|
|
157
|
+ |
const chunks: Buffer[] = [];
|
|
158
|
+ |
request.on("data", (chunk: Buffer) => chunks.push(chunk));
|
|
159
|
+ |
request.on("end", () => {
|
|
160
|
+ |
void forward(Buffer.concat(chunks).toString("utf8"), grant, response);
|
|
161
|
+ |
});
|
|
162
|
+ |
});
|
|
163
|
+ |
|
|
164
|
+ |
// A child that hangs must not hold the console open on the way out.
|
|
165
|
+ |
server.unref();
|
|
166
|
+ |
|
|
167
|
+ |
await new Promise<void>((resolve, reject) => {
|
|
168
|
+ |
server.once("error", reject);
|
|
169
|
+ |
server.listen(0, "127.0.0.1", resolve);
|
|
170
|
+ |
});
|
|
171
|
+ |
|
|
172
|
+ |
const address = server.address() as AddressInfo;
|
|
173
|
+ |
return {
|
|
174
|
+ |
baseUrl: `http://127.0.0.1:${String(address.port)}/v1`,
|
|
175
|
+ |
modelId: `${CHILD_PROVIDER}/${grant.model}`,
|
|
176
|
+ |
close: () =>
|
|
177
|
+ |
new Promise<void>((resolve) => {
|
|
178
|
+ |
server.close(() => resolve());
|
|
179
|
+ |
}),
|
|
180
|
+ |
};
|
|
181
|
+ |
}
|
|
182
|
+ |
|
|
183
|
+ |
/** Spend the grant on one child call and stream the answer back verbatim. */
|
|
184
|
+ |
async function forward(
|
|
185
|
+ |
body: string,
|
|
186
|
+ |
grant: ChildGrant,
|
|
187
|
+ |
response: import("node:http").ServerResponse,
|
|
188
|
+ |
): Promise<void> {
|
|
189
|
+ |
let payload: Record<string, unknown> = {};
|
|
190
|
+ |
try {
|
|
191
|
+ |
const parsed: unknown = JSON.parse(body === "" ? "{}" : body);
|
|
192
|
+ |
if (typeof parsed === "object" && parsed !== null) payload = parsed as Record<string, unknown>;
|
|
193
|
+ |
} catch {
|
|
194
|
+ |
// An unparseable body is treated as an empty one, and the proxy's own
|
|
195
|
+ |
// refusal is what the child then reports.
|
|
196
|
+ |
}
|
|
197
|
+ |
|
|
198
|
+ |
const messages = Array.isArray(payload["messages"]) ? payload["messages"] : [];
|
|
199
|
+ |
const tools = payload["tools"];
|
|
200
|
+ |
|
|
201
|
+ |
const upstream = await fetch(grant.proxyUrl, {
|
|
202
|
+ |
method: "POST",
|
|
203
|
+ |
headers: {
|
|
204
|
+ |
authorization: `Bearer ${Redacted.value(grant.token)}`,
|
|
205
|
+ |
"content-type": "application/json",
|
|
206
|
+ |
accept: "text/event-stream, application/json",
|
|
207
|
+ |
},
|
|
208
|
+ |
body: JSON.stringify({
|
|
209
|
+ |
// The grant pins the model, so whatever the child named is ignored here
|
|
210
|
+ |
// rather than passed through and refused.
|
|
211
|
+ |
model: grant.model,
|
|
212
|
+ |
stream: true,
|
|
213
|
+ |
messages: flattenForProxy(messages),
|
|
214
|
+ |
...(Array.isArray(tools) && tools.length > 0 ? { tools } : {}),
|
|
215
|
+ |
}),
|
|
216
|
+ |
}).catch((cause: unknown) => {
|
|
217
|
+ |
response.writeHead(502, { "content-type": "application/json" });
|
|
218
|
+ |
response.end(
|
|
219
|
+ |
JSON.stringify({
|
|
220
|
+ |
error: { message: `The inference proxy could not be reached: ${String(cause)}` },
|
|
221
|
+ |
}),
|
|
222
|
+ |
);
|
|
223
|
+ |
return undefined;
|
|
224
|
+ |
});
|
|
225
|
+ |
|
|
226
|
+ |
if (upstream === undefined) return;
|
|
227
|
+ |
|
|
228
|
+ |
if (!upstream.ok) {
|
|
229
|
+ |
// The proxy answers a refusal as `{"error":{"code":"…"}}`, which an
|
|
230
|
+ |
// OpenAI-compatible client reports as an unexplained server error because
|
|
231
|
+ |
// it looks for `error.message`. Putting the status and the body in a
|
|
232
|
+ |
// message is the difference between a child that says
|
|
233
|
+ |
// `budget_exhausted` and three children that say `exited with code 1`.
|
|
234
|
+ |
const detail = (await upstream.text().catch(() => "")).slice(0, 400);
|
|
235
|
+ |
response.writeHead(upstream.status, { "content-type": "application/json" });
|
|
236
|
+ |
response.end(
|
|
237
|
+ |
JSON.stringify({
|
|
238
|
+ |
error: {
|
|
239
|
+ |
type: "openagents_proxy",
|
|
240
|
+ |
message:
|
|
241
|
+ |
`The OpenAgents inference proxy refused this call with HTTP ${String(upstream.status)}` +
|
|
242
|
+ |
`${detail === "" ? "." : `: ${detail}`}`,
|
|
243
|
+ |
},
|
|
244
|
+ |
}),
|
|
245
|
+ |
);
|
|
246
|
+ |
return;
|
|
247
|
+ |
}
|
|
248
|
+ |
|
|
249
|
+ |
response.writeHead(upstream.status, {
|
|
250
|
+ |
"content-type": upstream.headers.get("content-type") ?? "text/event-stream",
|
|
251
|
+ |
});
|
|
252
|
+ |
|
|
253
|
+ |
if (upstream.body === null) {
|
|
254
|
+ |
response.end();
|
|
255
|
+ |
return;
|
|
256
|
+ |
}
|
|
257
|
+ |
|
|
258
|
+ |
const reader = upstream.body.getReader();
|
|
259
|
+ |
for (;;) {
|
|
260
|
+ |
// The body is a stream and each read depends on the one before it.
|
|
261
|
+ |
// eslint-disable-next-line no-await-in-loop
|
|
262
|
+ |
const { done, value } = await reader.read();
|
|
263
|
+ |
if (done) break;
|
|
264
|
+ |
response.write(Buffer.from(value));
|
|
265
|
+ |
}
|
|
266
|
+ |
response.end();
|
|
267
|
+ |
}
|