|
1
|
+ |
//! The end-to-end half of the live-execution audit for
|
|
2
|
+ |
//! OpenAgentsInc/openagents#89.
|
|
3
|
+ |
//!
|
|
4
|
+ |
//! Six defects were found by driving the compiled binary against the live
|
|
5
|
+ |
//! deployment rather than by reading the source. Five are fixed in
|
|
6
|
+ |
//! `c48fa5b138` and the sixth in the commit this file arrives on; each test
|
|
7
|
+ |
//! below now asserts the **fixed** behaviour, and each keeps the account of
|
|
8
|
+ |
//! what the defect was, because the reason a test exists outlives the
|
|
9
|
+ |
//! assertion.
|
|
10
|
+ |
//!
|
|
11
|
+ |
//! These run one layer out from the unit tests that accompany the fixes:
|
|
12
|
+ |
//! `tools::defect_tests` calls `run_real_shell` and `floor_char_boundary`
|
|
13
|
+ |
//! directly, while these go through `HarnessToolRegistry::execute_tool` with
|
|
14
|
+ |
//! real subprocesses, and through `CoderRuntimeSession` against a real socket.
|
|
15
|
+ |
//! Where a defect is covered at both layers the outer one is kept here and the
|
|
16
|
+ |
//! duplicate dropped — the hosted-lane transcript test lives in
|
|
17
|
+ |
//! `runtime_test.rs` as `the_second_turn_carries_what_the_first_turn_answered`,
|
|
18
|
+ |
//! so what remains here is the local lane, which that fix also changed and
|
|
19
|
+ |
//! nothing else covers.
|
|
20
|
+ |
//!
|
|
21
|
+ |
//! What the live run confirmed working, and what is therefore not re-asserted
|
|
22
|
+ |
//! here: a headless turn reaches a real model and prints its answer, a turn
|
|
23
|
+ |
//! calls `shell` and answers from the result, a two-child fan-out returns two
|
|
24
|
+ |
//! real outputs from separate git worktrees, every lane opens on the model it
|
|
25
|
+ |
//! names, and reported token counts match what `GET /api/v1/threads` records.
|
|
26
|
+ |
|
|
27
|
+ |
use openagents_cli::runtime::{CoderRuntimeSession, Lane};
|
|
28
|
+ |
use openagents_cli::tools::{
|
|
29
|
+ |
resolve_openagents_cli, HarnessToolRegistry, OpenAgentsCliSource, ToolCall,
|
|
30
|
+ |
};
|
|
31
|
+ |
use std::sync::{Arc, Mutex};
|
|
32
|
+ |
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
33
|
+ |
|
|
34
|
+ |
// ───────────────────────────────────────────────────────────────── the stub
|
|
35
|
+ |
|
|
36
|
+ |
/// What a request should be answered with.
|
|
37
|
+ |
enum Reply {
|
|
38
|
+ |
Json(String),
|
|
39
|
+ |
/// Server-sent events, the shape the inference proxy streams.
|
|
40
|
+ |
Sse(Vec<String>),
|
|
41
|
+ |
/// Newline-delimited JSON, the shape Ollama streams.
|
|
42
|
+ |
Ndjson(Vec<String>),
|
|
43
|
+ |
}
|
|
44
|
+ |
|
|
45
|
+ |
/// A stand-in for whichever server the session is talking to, recording what
|
|
46
|
+ |
/// it was asked.
|
|
47
|
+ |
///
|
|
48
|
+ |
/// It answers on a real socket, so everything between the session and the wire
|
|
49
|
+ |
/// is the production path and what these tests assert on is the bytes that
|
|
50
|
+ |
/// were actually sent.
|
|
51
|
+ |
struct Stub {
|
|
52
|
+ |
base: String,
|
|
53
|
+ |
origin: String,
|
|
54
|
+ |
requests: Arc<Mutex<Vec<String>>>,
|
|
55
|
+ |
}
|
|
56
|
+ |
|
|
57
|
+ |
impl Stub {
|
|
58
|
+ |
/// Every request this stub has taken, headers and body, oldest first.
|
|
59
|
+ |
fn requests(&self) -> Vec<String> {
|
|
60
|
+ |
self.requests.lock().unwrap().clone()
|
|
61
|
+ |
}
|
|
62
|
+ |
|
|
63
|
+ |
/// Just the bodies of the calls that asked a model to say something.
|
|
64
|
+ |
fn completions(&self) -> Vec<String> {
|
|
65
|
+ |
self.requests()
|
|
66
|
+ |
.into_iter()
|
|
67
|
+ |
.filter(|r| r.starts_with("POST /proxy") || r.starts_with("POST /api/chat"))
|
|
68
|
+ |
.collect()
|
|
69
|
+ |
}
|
|
70
|
+ |
}
|
|
71
|
+ |
|
|
72
|
+ |
/// Start a stub whose reply is chosen by the request and by how many
|
|
73
|
+ |
/// completions it has already served.
|
|
74
|
+ |
fn start<H>(handler: H) -> Stub
|
|
75
|
+ |
where
|
|
76
|
+ |
H: Fn(&str, usize, &str) -> Reply + Send + Sync + 'static,
|
|
77
|
+ |
{
|
|
78
|
+ |
let requests = Arc::new(Mutex::new(Vec::new()));
|
|
79
|
+ |
let recorder = Arc::clone(&requests);
|
|
80
|
+ |
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
81
|
+ |
listener.set_nonblocking(true).unwrap();
|
|
82
|
+ |
let port = listener.local_addr().unwrap().port();
|
|
83
|
+ |
let origin = format!("http://127.0.0.1:{port}");
|
|
84
|
+ |
let base = format!("{origin}/api/v1");
|
|
85
|
+ |
let handler_origin = origin.clone();
|
|
86
|
+ |
|
|
87
|
+ |
tokio::spawn(async move {
|
|
88
|
+ |
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
|
89
|
+ |
let mut served = 0usize;
|
|
90
|
+ |
loop {
|
|
91
|
+ |
let Ok((mut socket, _)) = listener.accept().await else {
|
|
92
|
+ |
return;
|
|
93
|
+ |
};
|
|
94
|
+ |
let Some(request) = read_request(&mut socket).await else {
|
|
95
|
+ |
continue;
|
|
96
|
+ |
};
|
|
97
|
+ |
recorder.lock().unwrap().push(request.clone());
|
|
98
|
+ |
|
|
99
|
+ |
let reply = handler(&request, served, &handler_origin);
|
|
100
|
+ |
match reply {
|
|
101
|
+ |
Reply::Json(body) => {
|
|
102
|
+ |
let response = format!(
|
|
103
|
+ |
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
|
104
|
+ |
body.len()
|
|
105
|
+ |
);
|
|
106
|
+ |
let _ = socket.write_all(response.as_bytes()).await;
|
|
107
|
+ |
}
|
|
108
|
+ |
Reply::Sse(frames) => {
|
|
109
|
+ |
served += 1;
|
|
110
|
+ |
let _ = socket
|
|
111
|
+ |
.write_all(b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n")
|
|
112
|
+ |
.await;
|
|
113
|
+ |
for frame in frames {
|
|
114
|
+ |
let _ = socket
|
|
115
|
+ |
.write_all(format!("data: {frame}\n\n").as_bytes())
|
|
116
|
+ |
.await;
|
|
117
|
+ |
}
|
|
118
|
+ |
let _ = socket.write_all(b"data: [DONE]\n\n").await;
|
|
119
|
+ |
}
|
|
120
|
+ |
Reply::Ndjson(lines) => {
|
|
121
|
+ |
served += 1;
|
|
122
|
+ |
let _ = socket
|
|
123
|
+ |
.write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/x-ndjson\r\nconnection: close\r\n\r\n")
|
|
124
|
+ |
.await;
|
|
125
|
+ |
for line in lines {
|
|
126
|
+ |
let _ = socket.write_all(format!("{line}\n").as_bytes()).await;
|
|
127
|
+ |
}
|
|
128
|
+ |
}
|
|
129
|
+ |
}
|
|
130
|
+ |
let _ = socket.flush().await;
|
|
131
|
+ |
}
|
|
132
|
+ |
});
|
|
133
|
+ |
|
|
134
|
+ |
Stub {
|
|
135
|
+ |
base,
|
|
136
|
+ |
origin,
|
|
137
|
+ |
requests,
|
|
138
|
+ |
}
|
|
139
|
+ |
}
|
|
140
|
+ |
|
|
141
|
+ |
/// Read one request, headers and declared body, and return it as text.
|
|
142
|
+ |
async fn read_request(socket: &mut tokio::net::TcpStream) -> Option<String> {
|
|
143
|
+ |
let mut request = Vec::new();
|
|
144
|
+ |
let mut buffer = [0u8; 4096];
|
|
145
|
+ |
loop {
|
|
146
|
+ |
let read = socket.read(&mut buffer).await.ok()?;
|
|
147
|
+ |
if read == 0 {
|
|
148
|
+ |
break;
|
|
149
|
+ |
}
|
|
150
|
+ |
request.extend_from_slice(&buffer[..read]);
|
|
151
|
+ |
let text = String::from_utf8_lossy(&request);
|
|
152
|
+ |
if let Some(headers_end) = text.find("\r\n\r\n") {
|
|
153
|
+ |
let length = text
|
|
154
|
+ |
.lines()
|
|
155
|
+ |
.find_map(|line| {
|
|
156
|
+ |
line.strip_prefix("content-length: ")
|
|
157
|
+ |
.or_else(|| line.strip_prefix("Content-Length: "))
|
|
158
|
+ |
})
|
|
159
|
+ |
.and_then(|value| value.trim().parse::<usize>().ok())
|
|
160
|
+ |
.unwrap_or(0);
|
|
161
|
+ |
if request.len() >= headers_end + 4 + length {
|
|
162
|
+ |
break;
|
|
163
|
+ |
}
|
|
164
|
+ |
}
|
|
165
|
+ |
}
|
|
166
|
+ |
Some(String::from_utf8_lossy(&request).to_string())
|
|
167
|
+ |
}
|
|
168
|
+ |
|
|
169
|
+ |
fn grant(origin: &str) -> String {
|
|
170
|
+ |
format!(
|
|
171
|
+ |
r#"{{"thread":{{"id":"th_test"}},"grant":{{"token":"tok_test","url":"{origin}/proxy","model":"ox-alpha"}}}}"#
|
|
172
|
+ |
)
|
|
173
|
+ |
}
|
|
174
|
+ |
|
|
175
|
+ |
/// One frame asking for a tool, whole rather than fragmented.
|
|
176
|
+ |
fn call_tool(id: &str, name: &str) -> String {
|
|
177
|
+ |
serde_json::json!({
|
|
178
|
+ |
"choices": [{ "delta": { "tool_calls": [{
|
|
179
|
+ |
"index": 0, "id": id,
|
|
180
|
+ |
"function": { "name": name, "arguments": "{}" }
|
|
181
|
+ |
}]}}]
|
|
182
|
+ |
})
|
|
183
|
+ |
.to_string()
|
|
184
|
+ |
}
|
|
185
|
+ |
|
|
186
|
+ |
fn registry() -> (tempfile::TempDir, HarnessToolRegistry) {
|
|
187
|
+ |
let dir = tempfile::tempdir().unwrap();
|
|
188
|
+ |
let registry = HarnessToolRegistry::new(Some(dir.path().to_path_buf()));
|
|
189
|
+ |
(dir, registry)
|
|
190
|
+ |
}
|
|
191
|
+ |
|
|
192
|
+ |
// ──────────────────────────────────────────────────────────────── defect 1
|
|
193
|
+ |
|
|
194
|
+ |
/// **Output cut through a multi-byte character no longer kills the agent.**
|
|
195
|
+ |
///
|
|
196
|
+ |
/// `run_real_shell` bounded output with `&combined[..OUTPUT_LIMIT]` — a *byte*
|
|
197
|
+ |
/// index into a `String`, which panics when the index is not a character
|
|
198
|
+ |
/// boundary. Any command whose combined output crossed 30,000 bytes
|
|
199
|
+ |
/// mid-character took the whole process down in the middle of a turn: a `git
|
|
200
|
+ |
/// log`, a test run with UTF-8 output, any file with an accent or an emoji
|
|
201
|
+ |
/// past 30 kB. It aborted before the thread could be revoked, so the grant's
|
|
202
|
+ |
/// remaining budget was stranded too.
|
|
203
|
+ |
///
|
|
204
|
+ |
/// Reproduced live against the built binary before the fix:
|
|
205
|
+ |
///
|
|
206
|
+ |
/// ```text
|
|
207
|
+ |
/// $ oa coder --headless "Use the shell tool to run exactly: sh repro.sh"
|
|
208
|
+ |
/// thread 'main' panicked at crates/openagents-cli/src/tools.rs:424:98:
|
|
209
|
+ |
/// byte index 30000 is not a char boundary; it is inside '€' (bytes 29999..30002)
|
|
210
|
+ |
/// EXIT=101
|
|
211
|
+ |
/// ```
|
|
212
|
+ |
///
|
|
213
|
+ |
/// `tools::defect_tests` covers `floor_char_boundary` on its own. This runs
|
|
214
|
+ |
/// the real `/bin/sh` through the tool dispatch and asserts the whole path
|
|
215
|
+ |
/// survives: the panic showed up through a command, so a command is what
|
|
216
|
+ |
/// proves it gone.
|
|
217
|
+ |
#[tokio::test]
|
|
218
|
+ |
async fn shell_output_cut_through_a_multibyte_character_is_truncated_not_fatal() {
|
|
219
|
+ |
let (_dir, registry) = registry();
|
|
220
|
+ |
|
|
221
|
+ |
// 29,999 single-byte characters, then three-byte ones. Byte 30,000 lands
|
|
222
|
+ |
// inside the first `€`, which occupies bytes 29,999-30,001.
|
|
223
|
+ |
let command = "head -c 29999 /dev/zero | tr '\\0' 'a' && printf '€€€€€€€€€€'";
|
|
224
|
+ |
let call = ToolCall {
|
|
225
|
+ |
id: "call_trunc".to_string(),
|
|
226
|
+ |
name: "shell".to_string(),
|
|
227
|
+ |
arguments: serde_json::json!({ "command": command }),
|
|
228
|
+ |
};
|
|
229
|
+ |
|
|
230
|
+ |
// On its own task, so a panic is reported rather than unwinding the test.
|
|
231
|
+ |
let output = tokio::spawn(async move { registry.execute_tool(&call).await })
|
|
232
|
+ |
.await
|
|
233
|
+ |
.expect("the shell tool panicked on a multi-byte truncation boundary");
|
|
234
|
+ |
|
|
235
|
+ |
assert!(
|
|
236
|
+ |
output
|
|
237
|
+ |
.output
|
|
238
|
+ |
.contains("[Output truncated: printed 30029 characters, limit is 30000]"),
|
|
239
|
+ |
"the output should say it was cut, and by how much: {}",
|
|
240
|
+ |
&output.output[output.output.len().saturating_sub(200)..]
|
|
241
|
+ |
);
|
|
242
|
+ |
assert!(
|
|
243
|
+ |
output.output.starts_with("aaa"),
|
|
244
|
+ |
"the kept head should be the start of the command's output"
|
|
245
|
+ |
);
|
|
246
|
+ |
assert!(
|
|
247
|
+ |
!output.output.contains('\u{FFFD}'),
|
|
248
|
+ |
"the cut left a broken character behind"
|
|
249
|
+ |
);
|
|
250
|
+ |
assert!(!output.is_error, "the command itself succeeded");
|
|
251
|
+ |
}
|
|
252
|
+ |
|
|
253
|
+ |
// ──────────────────────────────────────────────────────────────── defect 2
|
|
254
|
+ |
|
|
255
|
+ |
/// **The local lane records what it answered, so the next turn can see it.**
|
|
256
|
+ |
///
|
|
257
|
+ |
/// `run_tools` records an assistant turn only when that turn called a tool, so
|
|
258
|
+ |
/// a turn that simply answered never joined `self.messages`. The session is
|
|
259
|
+ |
/// reused across turns, so the model was never shown a word it had said
|
|
260
|
+ |
/// itself. Live, in the interactive session, it did not admit the gap — it
|
|
261
|
+ |
/// confabulated:
|
|
262
|
+ |
///
|
|
263
|
+ |
/// ```text
|
|
264
|
+ |
/// turn 1: "Invent a random six-letter nonsense codeword." -> QORVEN
|
|
265
|
+ |
/// turn 2: "What was the codeword you just invented?" -> ZORBEX
|
|
266
|
+ |
/// ```
|
|
267
|
+ |
///
|
|
268
|
+ |
/// A test that asks the model to recall something from the **user's** prompt
|
|
269
|
+ |
/// passes against the defect, because user messages were always recorded. The
|
|
270
|
+ |
/// word has to be one only the assistant ever said.
|
|
271
|
+ |
///
|
|
272
|
+ |
/// The hosted lane is covered by `the_second_turn_carries_what_the_first_turn_answered`
|
|
273
|
+ |
/// in `runtime_test.rs`. The fix changed `run_local_turn` the same way, and
|
|
274
|
+ |
/// this is that half: the local lane keeps its own message list, in Ollama's
|
|
275
|
+ |
/// shape, through a separate code path.
|
|
276
|
+ |
#[tokio::test]
|
|
277
|
+ |
async fn the_local_lane_records_what_it_answered() {
|
|
278
|
+ |
let stub = start(|request, served, _origin| {
|
|
279
|
+ |
if request.starts_with("GET /api/tags") {
|
|
280
|
+ |
return Reply::Json(
|
|
281
|
+ |
r#"{"models":[{"name":"qwen3:0.6b","modified_at":"2026-08-26T00:00:00Z"}]}"#
|
|
282
|
+ |
.to_string(),
|
|
283
|
+ |
);
|
|
284
|
+ |
}
|
|
285
|
+ |
let word = if served == 0 { "QORVEN" } else { "ZORBEX" };
|
|
286
|
+ |
Reply::Ndjson(vec![
|
|
287
|
+ |
serde_json::json!({
|
|
288
|
+ |
"model": "qwen3:0.6b",
|
|
289
|
+ |
"message": { "role": "assistant", "content": word },
|
|
290
|
+ |
"done": false
|
|
291
|
+ |
})
|
|
292
|
+ |
.to_string(),
|
|
293
|
+ |
serde_json::json!({
|
|
294
|
+ |
"model": "qwen3:0.6b",
|
|
295
|
+ |
"message": { "role": "assistant", "content": "" },
|
|
296
|
+ |
"done": true, "done_reason": "stop",
|
|
297
|
+ |
"prompt_eval_count": 10, "eval_count": 3
|
|
298
|
+ |
})
|
|
299
|
+ |
.to_string(),
|
|
300
|
+ |
])
|
|
301
|
+ |
});
|
|
302
|
+ |
|
|
303
|
+ |
let (_dir, tools) = registry();
|
|
304
|
+ |
let mut session = CoderRuntimeSession::new(
|
|
305
|
+ |
Lane::Local(String::new()),
|
|
306
|
+ |
Some(stub.base.clone()),
|
|
307
|
+ |
None,
|
|
308
|
+ |
tools,
|
|
309
|
+ |
);
|
|
310
|
+ |
session.ollama_host = stub.origin.clone();
|
|
311
|
+ |
|
|
312
|
+ |
let first = session
|
|
313
|
+ |
.execute_turn("invent a six-letter codeword", |_| {})
|
|
314
|
+ |
.await
|
|
315
|
+ |
.unwrap();
|
|
316
|
+ |
assert_eq!(first, "QORVEN");
|
|
317
|
+ |
|
|
318
|
+ |
session
|
|
319
|
+ |
.execute_turn("what was the codeword?", |_| {})
|
|
320
|
+ |
.await
|
|
321
|
+ |
.unwrap();
|
|
322
|
+ |
|
|
323
|
+ |
let chats = stub.completions();
|
|
324
|
+ |
assert_eq!(chats.len(), 2, "expected one chat call per turn");
|
|
325
|
+ |
assert!(
|
|
326
|
+ |
chats[1].contains("QORVEN"),
|
|
327
|
+ |
"the second turn did not carry the first turn's answer, so the local \
|
|
328
|
+ |
model cannot see what it said: {}",
|
|
329
|
+ |
chats[1]
|
|
330
|
+ |
);
|
|
331
|
+ |
assert!(
|
|
332
|
+ |
session
|
|
333
|
+ |
.messages
|
|
334
|
+ |
.iter()
|
|
335
|
+ |
.any(|m| m.role == "assistant" && m.content.as_deref() == Some("QORVEN")),
|
|
336
|
+ |
"the answer is missing from the session's own transcript"
|
|
337
|
+ |
);
|
|
338
|
+ |
}
|
|
339
|
+ |
|
|
340
|
+ |
// ──────────────────────────────────────────────────────────────── defect 3
|
|
341
|
+ |
|
|
342
|
+ |
/// **A turn that runs out of tool steps refuses instead of reporting success.**
|
|
343
|
+ |
///
|
|
344
|
+ |
/// `run_thread_turn` loops `for _ in 0..MAX_TOOL_STEPS` and assigned
|
|
345
|
+ |
/// `final_answer` only on the step that came back without tool calls. A model
|
|
346
|
+ |
/// that asked for a tool on all thirty steps fell out of the bottom and the
|
|
347
|
+ |
/// function returned `Ok(String::new())` — the same `Ok` a finished turn
|
|
348
|
+ |
/// returns, carrying nothing. `run_headless_coder` printed `Turn result:`
|
|
349
|
+ |
/// followed by a blank line and exited 0, and the interactive session settled
|
|
350
|
+ |
/// `TurnEvent::Done("")` as an answered turn.
|
|
351
|
+ |
///
|
|
352
|
+ |
/// That is the shape the issue was reopened over, one level down: the turn did
|
|
353
|
+ |
/// not finish and no caller could tell.
|
|
354
|
+ |
#[tokio::test]
|
|
355
|
+ |
async fn a_turn_that_exhausts_its_tool_steps_refuses() {
|
|
356
|
+ |
// Every step asks for a tool and never stops asking. The name is one no
|
|
357
|
+ |
// registry has, so the loop spends no time running anything.
|
|
358
|
+ |
let stub = start(|request, served, origin| {
|
|
359
|
+ |
if request.starts_with("POST /api/v1/threads") {
|
|
360
|
+ |
return Reply::Json(grant(origin));
|
|
361
|
+ |
}
|
|
362
|
+ |
Reply::Sse(vec![call_tool(
|
|
363
|
+ |
&format!("call_{served}"),
|
|
364
|
+ |
"a_tool_that_does_not_exist",
|
|
365
|
+ |
)])
|
|
366
|
+ |
});
|
|
367
|
+ |
|
|
368
|
+ |
let (_dir, tools) = registry();
|
|
369
|
+ |
let mut session = CoderRuntimeSession::new(
|
|
370
|
+ |
Lane::OxAlpha,
|
|
371
|
+ |
Some(stub.base.clone()),
|
|
372
|
+ |
Some("tok_user".to_string()),
|
|
373
|
+ |
tools,
|
|
374
|
+ |
);
|
|
375
|
+ |
|
|
376
|
+ |
let error = session
|
|
377
|
+ |
.execute_turn("loop forever", |_| {})
|
|
378
|
+ |
.await
|
|
379
|
+ |
.expect_err("a turn that never answered was reported as a finished turn");
|
|
380
|
+ |
let error = error.to_string();
|
|
381
|
+ |
|
|
382
|
+ |
assert!(
|
|
383
|
+ |
error.contains("30") && error.contains("tool steps"),
|
|
384
|
+ |
"the refusal should say the step budget ran out: {error}"
|
|
385
|
+ |
);
|
|
386
|
+ |
assert_eq!(
|
|
387
|
+ |
stub.completions().len(),
|
|
388
|
+ |
30,
|
|
389
|
+ |
"MAX_TOOL_STEPS is 30, so the turn should spend the whole budget before refusing"
|
|
390
|
+ |
);
|
|
391
|
+ |
}
|
|
392
|
+ |
|
|
393
|
+ |
// ──────────────────────────────────────────────────────────────── defect 4
|
|
394
|
+ |
|
|
395
|
+ |
/// **The `openagents` tool prefers `PATH` and falls back to this binary.**
|
|
396
|
+ |
///
|
|
397
|
+ |
/// The tool is declared as "Run the OpenAgents CLI commands (issue, project,
|
|
398
|
+ |
/// repo, auth, etc.)" and was implemented as `Command::new("openagents")` — a
|
|
399
|
+ |
/// bare name resolved through `PATH`, with nothing behind it. On the machine
|
|
400
|
+ |
/// this was verified on that reached `openagents v0.4.0`, the TypeScript CLI,
|
|
401
|
+ |
/// while the agent running the tool was `oa 0.1.0`; on a machine carrying only
|
|
402
|
+ |
/// the Rust CLI there is no `openagents` on `PATH` at all and every call
|
|
403
|
+ |
/// failed with `No such file or directory`.
|
|
404
|
+ |
///
|
|
405
|
+ |
/// The contract now: prefer `PATH`, because the CLI installed under that name
|
|
406
|
+ |
/// covers more subcommands than this binary does; fall back to
|
|
407
|
+ |
/// `current_exe()`, so the tool still works where only the Rust binary exists;
|
|
408
|
+ |
/// name which one ran in the result, so a model reading `unknown command`
|
|
409
|
+ |
/// knows which CLI said it; and if neither resolves, return an error rather
|
|
410
|
+ |
/// than a success carrying nothing.
|
|
411
|
+ |
#[tokio::test]
|
|
412
|
+ |
async fn the_openagents_tool_prefers_path_and_falls_back_to_this_binary() {
|
|
413
|
+ |
let dir = tempfile::tempdir().unwrap();
|
|
414
|
+ |
let shim = dir.path().join("openagents");
|
|
415
|
+ |
std::fs::write(&shim, "#!/bin/sh\necho SHIM-ON-PATH\n").unwrap();
|
|
416
|
+ |
#[cfg(unix)]
|
|
417
|
+ |
{
|
|
418
|
+ |
use std::os::unix::fs::PermissionsExt;
|
|
419
|
+ |
std::fs::set_permissions(&shim, std::fs::Permissions::from_mode(0o755)).unwrap();
|
|
420
|
+ |
}
|
|
421
|
+ |
|
|
422
|
+ |
// The only test in this file that touches the environment.
|
|
423
|
+ |
let original = std::env::var("PATH").unwrap_or_default();
|
|
424
|
+ |
|
|
425
|
+ |
// With something named `openagents` on PATH, that is what runs.
|
|
426
|
+ |
std::env::set_var("PATH", format!("{}:{original}", dir.path().display()));
|
|
427
|
+ |
let registry = HarnessToolRegistry::new(Some(dir.path().to_path_buf()));
|
|
428
|
+ |
let output = registry
|
|
429
|
+ |
.execute_tool(&ToolCall {
|
|
430
|
+ |
id: "call_oa".to_string(),
|
|
431
|
+ |
name: "openagents".to_string(),
|
|
432
|
+ |
arguments: serde_json::json!({ "args": ["--version"] }),
|
|
433
|
+ |
})
|
|
434
|
+ |
.await;
|
|
435
|
+ |
let on_path = resolve_openagents_cli();
|
|
436
|
+ |
|
|
437
|
+ |
// With nothing named `openagents` anywhere, this binary answers for it.
|
|
438
|
+ |
std::env::set_var("PATH", "");
|
|
439
|
+ |
let fallback = resolve_openagents_cli();
|
|
440
|
+ |
|
|
441
|
+ |
std::env::set_var("PATH", original);
|
|
442
|
+ |
|
|
443
|
+ |
assert!(
|
|
444
|
+ |
output.output.contains("SHIM-ON-PATH"),
|
|
445
|
+ |
"the CLI on PATH should have run: {}",
|
|
446
|
+ |
output.output
|
|
447
|
+ |
);
|
|
448
|
+ |
assert!(
|
|
449
|
+ |
output.output.contains("[ran the `openagents` CLI on PATH:")
|
|
450
|
+ |
&& output.output.contains(&shim.display().to_string()),
|
|
451
|
+ |
"the result should name the program that answered: {}",
|
|
452
|
+ |
output.output
|
|
453
|
+ |
);
|
|
454
|
+ |
assert!(!output.is_error, "the shim exited zero");
|
|
455
|
+ |
|
|
456
|
+ |
let (found, source) = on_path.expect("a shim on PATH must resolve");
|
|
457
|
+ |
assert_eq!(source, OpenAgentsCliSource::Path);
|
|
458
|
+ |
assert_eq!(found, shim);
|
|
459
|
+ |
|
|
460
|
+ |
let (found, source) = fallback.expect("an empty PATH must still resolve to this binary");
|
|
461
|
+ |
assert_eq!(
|
|
462
|
+ |
source,
|
|
463
|
+ |
OpenAgentsCliSource::ThisBinary,
|
|
464
|
+ |
"with nothing on PATH the tool should fall back rather than fail"
|
|
465
|
+ |
);
|
|
466
|
+ |
assert_eq!(found, std::env::current_exe().unwrap());
|
|
467
|
+ |
}
|
|
468
|
+ |
|
|
469
|
+ |
// ──────────────────────────────────────────────────────────────── defect 5
|
|
470
|
+ |
|
|
471
|
+ |
/// **Cancelling a turn stops the command the `shell` tool started.**
|
|
472
|
+ |
///
|
|
473
|
+ |
/// `run_real_shell` spawned with neither `process_group(0)` nor
|
|
474
|
+ |
/// `kill_on_drop(true)`, while `delegate.rs`, `computer.rs` and `acp.rs` all
|
|
475
|
+ |
/// already put their children in a group. Dropping the future — which is what
|
|
476
|
+ |
/// `run_proxy_child`'s `tokio::select!` does when a fan-out is cancelled —
|
|
477
|
+ |
/// left the operating-system process running, reparented to init. On the
|
|
478
|
+ |
/// default `ox-alpha` lane a child is an in-process task with no pid at all,
|
|
479
|
+ |
/// so `signals::stop_tree` is never called for it and nothing else stopped
|
|
480
|
+ |
/// what it had started.
|
|
481
|
+ |
///
|
|
482
|
+ |
/// Observed live before the fix: `oa coder --delegate --count 2` told to run
|
|
483
|
+ |
/// `sleep 300`, interrupted with `SIGINT`, printed "Stopping the fan-out;
|
|
484
|
+ |
/// children are being signalled", reported both children "stopped before
|
|
485
|
+ |
/// finishing", exited — and left two `sleep 300` processes at `PPID 1`.
|
|
486
|
+ |
///
|
|
487
|
+ |
/// Asserted without pgrep: the abandoned command must not go on to finish its
|
|
488
|
+ |
/// work, so the file it was told to create must never appear.
|
|
489
|
+ |
#[tokio::test]
|
|
490
|
+ |
async fn cancelling_a_shell_tool_call_stops_the_command_it_started() {
|
|
491
|
+ |
let dir = tempfile::tempdir().unwrap();
|
|
492
|
+ |
let witness = dir.path().join("the-orphan-kept-going.txt");
|
|
493
|
+ |
let registry = HarnessToolRegistry::new(Some(dir.path().to_path_buf()));
|
|
494
|
+ |
|
|
495
|
+ |
let call = ToolCall {
|
|
496
|
+ |
id: "call_orphan".to_string(),
|
|
497
|
+ |
name: "shell".to_string(),
|
|
498
|
+ |
arguments: serde_json::json!({
|
|
499
|
+ |
"command": format!("sleep 2 && touch '{}'", witness.display())
|
|
500
|
+ |
}),
|
|
501
|
+ |
};
|
|
502
|
+ |
|
|
503
|
+ |
let handle = tokio::spawn(async move { registry.execute_tool(&call).await });
|
|
504
|
+ |
// Long enough for the shell to be spawned, far short of its `sleep`.
|
|
505
|
+ |
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
|
|
506
|
+ |
handle.abort();
|
|
507
|
+ |
assert!(handle.await.is_err(), "the call should have been cancelled");
|
|
508
|
+ |
assert!(
|
|
509
|
+ |
!witness.exists(),
|
|
510
|
+ |
"the command finished before it could be cancelled; the test is racing"
|
|
511
|
+ |
);
|
|
512
|
+ |
|
|
513
|
+ |
// Well past the command's own sleep.
|
|
514
|
+ |
tokio::time::sleep(std::time::Duration::from_millis(3500)).await;
|
|
515
|
+ |
|
|
516
|
+ |
assert!(
|
|
517
|
+ |
!witness.exists(),
|
|
518
|
+ |
"the shell subprocess outlived the cancelled call and finished its work"
|
|
519
|
+ |
);
|
|
520
|
+ |
}
|
|
521
|
+ |
|
|
522
|
+ |
// ──────────────────────────────────────────────────────────────── defect 6
|
|
523
|
+ |
|
|
524
|
+ |
/// **A failed tool is reported to the model as a failure.**
|
|
525
|
+ |
///
|
|
526
|
+ |
/// `execute_tool`'s `shell` arm returned `is_error: false` unconditionally,
|
|
527
|
+ |
/// whatever the command exited with, and the `openagents` arm did the same
|
|
528
|
+ |
/// even when the program could not be spawned. Only the refusal check and an
|
|
529
|
+ |
/// unknown tool name ever set the flag, so a failing build read to the model
|
|
530
|
+ |
/// exactly like a passing one.
|
|
531
|
+ |
///
|
|
532
|
+ |
/// `tools::defect_tests` asserts this on `run_real_shell`'s own return value.
|
|
533
|
+ |
/// This asserts the layer above — that the outcome survives the dispatch into
|
|
534
|
+ |
/// `ToolOutput`, which is the field a caller actually reads.
|
|
535
|
+ |
#[tokio::test]
|
|
536
|
+ |
async fn a_failing_shell_command_is_reported_as_a_failure_through_the_tool_result() {
|
|
537
|
+ |
let (_dir, registry) = registry();
|
|
538
|
+ |
|
|
539
|
+ |
let failed = registry
|
|
540
|
+ |
.execute_tool(&ToolCall {
|
|
541
|
+ |
id: "call_fail".to_string(),
|
|
542
|
+ |
name: "shell".to_string(),
|
|
543
|
+ |
arguments: serde_json::json!({ "command": "exit 42" }),
|
|
544
|
+ |
})
|
|
545
|
+ |
.await;
|
|
546
|
+ |
assert!(
|
|
547
|
+ |
failed.output.contains("exited with code 42"),
|
|
548
|
+ |
"the exit code belongs in the text the model reads: {}",
|
|
549
|
+ |
failed.output
|
|
550
|
+ |
);
|
|
551
|
+ |
assert!(failed.is_error, "a non-zero exit must reach the caller");
|
|
552
|
+ |
|
|
553
|
+ |
let worked = registry
|
|
554
|
+ |
.execute_tool(&ToolCall {
|
|
555
|
+ |
id: "call_ok".to_string(),
|
|
556
|
+ |
name: "shell".to_string(),
|
|
557
|
+ |
arguments: serde_json::json!({ "command": "echo fine" }),
|
|
558
|
+ |
})
|
|
559
|
+ |
.await;
|
|
560
|
+ |
assert_eq!(worked.output, "fine");
|
|
561
|
+ |
assert!(
|
|
562
|
+ |
!worked.is_error,
|
|
563
|
+ |
"a successful command must not be an error"
|
|
564
|
+ |
);
|
|
565
|
+ |
}
|