|
1
|
+ |
defmodule OpenAgents.SCV.DeploymentsTest do
|
|
2
|
+ |
@moduledoc """
|
|
3
|
+ |
SCV-001: the lane that spends OpenAgents capacity.
|
|
4
|
+ |
|
|
5
|
+ |
These tests exercise the refusals first — a non-operator, a disabled feature,
|
|
6
|
+ |
an oversized objective, a full concurrency ceiling — and then run one
|
|
7
|
+ |
deployment end to end against a fake OpenCode binary, so the admitted model
|
|
8
|
+ |
slug is proved to reach the process invocation rather than only the
|
|
9
|
+ |
configuration.
|
|
10
|
+ |
"""
|
|
11
|
+ |
|
|
12
|
+ |
use OpenAgents.DataCase, async: false
|
|
13
|
+ |
|
|
14
|
+ |
alias OpenAgents.AccountsFixtures
|
|
15
|
+ |
alias OpenAgents.Conversations
|
|
16
|
+ |
alias OpenAgents.Forge.Repos
|
|
17
|
+ |
alias OpenAgents.Repositories
|
|
18
|
+ |
alias OpenAgents.RuntimeConfig
|
|
19
|
+ |
alias OpenAgents.SCV.Deployments
|
|
20
|
+ |
alias OpenAgents.Tools.{ExecutionContext, ScvDeploy}
|
|
21
|
+ |
alias OpenAgents.Work.{Job, Scv}
|
|
22
|
+ |
|
|
23
|
+ |
@model "opencode/x-preview-f-free"
|
|
24
|
+ |
|
|
25
|
+ |
setup do
|
|
26
|
+ |
Ecto.Adapters.SQL.Sandbox.mode(OpenAgents.Repo, {:shared, self()})
|
|
27
|
+ |
|
|
28
|
+ |
root = Path.join(System.tmp_dir!(), "scv-deploy-#{System.unique_integer([:positive])}")
|
|
29
|
+ |
executable = Path.join(root, "fake-opencode")
|
|
30
|
+ |
File.mkdir_p!(root)
|
|
31
|
+ |
File.write!(executable, fake_opencode())
|
|
32
|
+ |
File.chmod!(executable, 0o700)
|
|
33
|
+ |
|
|
34
|
+ |
previous =
|
|
35
|
+ |
for key <- [:forge_data_dir, :scv_deploy, :admin_github_ids] do
|
|
36
|
+ |
{key, Application.get_env(:openagents, key)}
|
|
37
|
+ |
end
|
|
38
|
+ |
|
|
39
|
+ |
Application.put_env(:openagents, :forge_data_dir, Path.join(root, "forge"))
|
|
40
|
+ |
|
|
41
|
+ |
Application.put_env(:openagents, :scv_deploy,
|
|
42
|
+ |
enabled: true,
|
|
43
|
+ |
model: @model,
|
|
44
|
+ |
reasoning_effort: "low",
|
|
45
|
+ |
opencode_api_key: nil,
|
|
46
|
+ |
executable: executable,
|
|
47
|
+ |
concurrency_limit: 2,
|
|
48
|
+ |
wall_clock_ms: 60_000,
|
|
49
|
+ |
maximum_output_bytes: 65_536,
|
|
50
|
+ |
output_root: Path.join(root, "runs")
|
|
51
|
+ |
)
|
|
52
|
+ |
|
|
53
|
+ |
on_exit(fn ->
|
|
54
|
+ |
for {key, value} <- previous do
|
|
55
|
+ |
if is_nil(value),
|
|
56
|
+ |
do: Application.delete_env(:openagents, key),
|
|
57
|
+ |
else: Application.put_env(:openagents, key, value)
|
|
58
|
+ |
end
|
|
59
|
+ |
|
|
60
|
+ |
File.rm_rf(root)
|
|
61
|
+ |
end)
|
|
62
|
+ |
|
|
63
|
+ |
%{root: root}
|
|
64
|
+ |
end
|
|
65
|
+ |
|
|
66
|
+ |
describe "operator authority" do
|
|
67
|
+ |
test "a signed-in non-operator is refused by the code that starts the run" do
|
|
68
|
+ |
%{user: user, conversation: conversation} = account("scv-non-operator")
|
|
69
|
+ |
|
|
70
|
+ |
assert {:error, :operator_required} =
|
|
71
|
+ |
Deployments.start(user, %{
|
|
72
|
+ |
conversation_id: conversation.id,
|
|
73
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
74
|
+ |
surface: "text",
|
|
75
|
+ |
repository: "OpenAgentsInc/openagents.com",
|
|
76
|
+ |
objective: "Describe the README."
|
|
77
|
+ |
})
|
|
78
|
+ |
|
|
79
|
+ |
# Nothing was written and nothing was spawned.
|
|
80
|
+ |
assert Deployments.active_count() == 0
|
|
81
|
+ |
end
|
|
82
|
+ |
|
|
83
|
+ |
test "the tool refuses a non-operator even though the catalog advertises it" do
|
|
84
|
+ |
%{user: user, conversation: conversation} = account("scv-tool-non-operator")
|
|
85
|
+ |
_repository = seed_repository!(user, "scvtool", "sample")
|
|
86
|
+ |
|
|
87
|
+ |
assert {:error, :operator_required} =
|
|
88
|
+ |
ScvDeploy.execute(
|
|
89
|
+ |
%{"repository" => "scvtool/sample", "objective" => "Describe the README."},
|
|
90
|
+ |
context(conversation)
|
|
91
|
+ |
)
|
|
92
|
+ |
|
|
93
|
+ |
assert Deployments.active_count() == 0
|
|
94
|
+ |
end
|
|
95
|
+ |
|
|
96
|
+ |
test "only an operator receives the approval receipt the surface policy demands" do
|
|
97
|
+ |
%{user: user} = account("scv-receipts")
|
|
98
|
+ |
%{user: operator} = operator_account("scv-receipts-operator")
|
|
99
|
+ |
|
|
100
|
+ |
assert Deployments.approval_receipts(user, "conversation:abc") == []
|
|
101
|
+ |
assert Deployments.approval_receipts(nil, "conversation:abc") == []
|
|
102
|
+ |
|
|
103
|
+ |
assert [receipt] = Deployments.approval_receipts(operator, "conversation:abc")
|
|
104
|
+ |
assert receipt["schema"] == "sarah.module_approval.v1"
|
|
105
|
+ |
assert receipt["approval_class"] == "explicit_operator_approval"
|
|
106
|
+ |
assert receipt["module_id"] == "sarah.tool.scv_deploy.v1"
|
|
107
|
+ |
assert receipt["actor_type"] == "operator"
|
|
108
|
+ |
assert receipt["explicit"] == true
|
|
109
|
+ |
assert receipt["receipt_ref"] == "operator:#{operator.id}"
|
|
110
|
+ |
end
|
|
111
|
+ |
end
|
|
112
|
+ |
|
|
113
|
+ |
describe "bounds" do
|
|
114
|
+ |
test "a disabled lane refuses before authority is even considered" do
|
|
115
|
+ |
settings = Application.fetch_env!(:openagents, :scv_deploy)
|
|
116
|
+ |
Application.put_env(:openagents, :scv_deploy, Keyword.put(settings, :enabled, false))
|
|
117
|
+ |
|
|
118
|
+ |
%{user: operator, conversation: conversation} = operator_account("scv-disabled")
|
|
119
|
+ |
|
|
120
|
+ |
assert {:error, :scv_deploy_disabled} =
|
|
121
|
+ |
Deployments.start(operator, %{
|
|
122
|
+ |
conversation_id: conversation.id,
|
|
123
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
124
|
+ |
surface: "text",
|
|
125
|
+ |
repository: "scvbounds/sample",
|
|
126
|
+ |
objective: "Describe the README."
|
|
127
|
+ |
})
|
|
128
|
+ |
end
|
|
129
|
+ |
|
|
130
|
+ |
test "an objective past its bound is refused" do
|
|
131
|
+ |
%{user: operator, conversation: conversation} = operator_account("scv-objective")
|
|
132
|
+ |
|
|
133
|
+ |
for objective <- ["", " ", String.duplicate("a", Scv.maximum_objective_bytes() + 1)] do
|
|
134
|
+ |
assert {:error, :scv_objective_invalid} =
|
|
135
|
+ |
Deployments.start(operator, %{
|
|
136
|
+ |
conversation_id: conversation.id,
|
|
137
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
138
|
+ |
surface: "text",
|
|
139
|
+ |
repository: "scvbounds/sample",
|
|
140
|
+ |
objective: objective
|
|
141
|
+ |
})
|
|
142
|
+ |
end
|
|
143
|
+ |
end
|
|
144
|
+ |
|
|
145
|
+ |
test "an unknown repository is refused before any process starts" do
|
|
146
|
+ |
%{user: operator, conversation: conversation} = operator_account("scv-repository")
|
|
147
|
+ |
|
|
148
|
+ |
assert {:error, :scv_repository_not_found} =
|
|
149
|
+ |
Deployments.start(operator, %{
|
|
150
|
+ |
conversation_id: conversation.id,
|
|
151
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
152
|
+ |
surface: "text",
|
|
153
|
+ |
repository: "nobody/nothing",
|
|
154
|
+ |
objective: "Describe the README."
|
|
155
|
+ |
})
|
|
156
|
+ |
|
|
157
|
+ |
# A filesystem path is not a repository name, and never becomes one.
|
|
158
|
+ |
assert {:error, :scv_repository_not_found} =
|
|
159
|
+ |
Deployments.start(operator, %{
|
|
160
|
+ |
conversation_id: conversation.id,
|
|
161
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
162
|
+ |
surface: "text",
|
|
163
|
+ |
repository: "/etc",
|
|
164
|
+ |
objective: "Describe the README."
|
|
165
|
+ |
})
|
|
166
|
+ |
end
|
|
167
|
+ |
|
|
168
|
+ |
test "the concurrency ceiling refuses the run rather than queueing it" do
|
|
169
|
+ |
%{user: operator, conversation: conversation} = operator_account("scv-capacity")
|
|
170
|
+ |
repository = seed_repository!(operator, "scvcap", "sample")
|
|
171
|
+ |
|
|
172
|
+ |
settings = Application.fetch_env!(:openagents, :scv_deploy)
|
|
173
|
+ |
Application.put_env(:openagents, :scv_deploy, Keyword.put(settings, :concurrency_limit, 1))
|
|
174
|
+ |
|
|
175
|
+ |
# One job already occupies the single admitted slot.
|
|
176
|
+ |
{:ok, _running} =
|
|
177
|
+ |
OpenAgents.Work.create_job(%{
|
|
178
|
+ |
conversation_id: conversation.id,
|
|
179
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
180
|
+ |
surface: "text",
|
|
181
|
+ |
goal: "an SCV already holding the slot",
|
|
182
|
+ |
kind: "scv"
|
|
183
|
+ |
})
|
|
184
|
+ |
|
|
185
|
+ |
assert Deployments.active_count() == 1
|
|
186
|
+ |
|
|
187
|
+ |
assert {:error, :scv_capacity_reached} =
|
|
188
|
+ |
Deployments.start(operator, %{
|
|
189
|
+ |
conversation_id: conversation.id,
|
|
190
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
191
|
+ |
surface: "text",
|
|
192
|
+ |
repository: "#{repository.owner}/#{repository.name}",
|
|
193
|
+ |
objective: "Describe the README."
|
|
194
|
+ |
})
|
|
195
|
+ |
end
|
|
196
|
+ |
|
|
197
|
+ |
test "the runtime configuration refuses the lane without the work lane or bounds" do
|
|
198
|
+ |
settings = Application.get_all_env(:openagents) |> Map.new()
|
|
199
|
+ |
|
|
200
|
+ |
enabled =
|
|
201
|
+ |
Map.put(settings, :scv_deploy, Keyword.put(base_deploy_settings(), :enabled, true))
|
|
202
|
+ |
|
|
203
|
+ |
assert {:error, %{setting: :scv_deploy, reason: "requires the work lane"}} =
|
|
204
|
+ |
RuntimeConfig.validate(
|
|
205
|
+ |
enabled
|
|
206
|
+ |
|> Map.put(:work, enabled: false)
|
|
207
|
+ |
|> Map.put(:work_workers_enabled, false)
|
|
208
|
+ |
)
|
|
209
|
+ |
|
|
210
|
+ |
unbounded =
|
|
211
|
+ |
Map.put(
|
|
212
|
+ |
enabled,
|
|
213
|
+ |
:scv_deploy,
|
|
214
|
+ |
base_deploy_settings()
|
|
215
|
+ |
|> Keyword.put(:enabled, true)
|
|
216
|
+ |
|> Keyword.put(:concurrency_limit, 100)
|
|
217
|
+ |
)
|
|
218
|
+ |
|
|
219
|
+ |
work_enabled =
|
|
220
|
+ |
unbounded
|
|
221
|
+ |
|> Map.put(:work, enabled: true)
|
|
222
|
+ |
|> Map.put(:work_workers_enabled, true)
|
|
223
|
+ |
|
|
224
|
+ |
assert {:error, %{setting: :scv_deploy, reason: reason}} =
|
|
225
|
+ |
RuntimeConfig.validate(work_enabled)
|
|
226
|
+ |
|
|
227
|
+ |
assert reason == "requires an admitted model, bounds, and output root"
|
|
228
|
+ |
end
|
|
229
|
+ |
end
|
|
230
|
+ |
|
|
231
|
+ |
describe "an admitted deployment" do
|
|
232
|
+ |
test "runs the admitted model on our capacity and reports back into the conversation" do
|
|
233
|
+ |
%{user: operator, conversation: conversation} = operator_account("scv-run")
|
|
234
|
+ |
repository = seed_repository!(operator, "scvrun", "sample")
|
|
235
|
+ |
|
|
236
|
+ |
assert {:ok, job} =
|
|
237
|
+ |
Deployments.start(operator, %{
|
|
238
|
+ |
conversation_id: conversation.id,
|
|
239
|
+ |
owner_visitor_id: conversation.visitor_id,
|
|
240
|
+ |
surface: "text",
|
|
241
|
+ |
repository: "#{repository.owner}/#{repository.name}",
|
|
242
|
+ |
objective: "Describe the README."
|
|
243
|
+ |
})
|
|
244
|
+ |
|
|
245
|
+ |
assert job.kind == "scv"
|
|
246
|
+ |
assert job.machine_id == nil
|
|
247
|
+ |
|
|
248
|
+ |
# The authority is snapshotted at admission, not read at run time.
|
|
249
|
+ |
assert job.authority_snapshot["model"] == @model
|
|
250
|
+ |
assert job.authority_snapshot["permission_profile"] == "read_only"
|
|
251
|
+ |
assert job.authority_snapshot["driver"] == "opencode"
|
|
252
|
+ |
assert job.authority_snapshot["repository_id"] == repository.id
|
|
253
|
+ |
assert job.authority_snapshot["operator_user_id"] == operator.id
|
|
254
|
+ |
assert Regex.match?(~r/\A[0-9a-f]{40}\z/, job.authority_snapshot["repository_revision"])
|
|
255
|
+ |
assert job.budget_snapshot["wall_clock_ms"] == 60_000
|
|
256
|
+ |
assert job.budget_snapshot["maximum_output_bytes"] == 65_536
|
|
257
|
+ |
|
|
258
|
+ |
terminal = await_terminal(job.id)
|
|
259
|
+ |
assert terminal.status == "completed"
|
|
260
|
+ |
|
|
261
|
+ |
# The fake binary echoes what it was invoked with, so this asserts the
|
|
262
|
+ |
# slug reached the process, not merely the configuration.
|
|
263
|
+ |
assert terminal.report =~ "model=#{@model}"
|
|
264
|
+ |
assert terminal.report =~ "fetch=0"
|
|
265
|
+ |
assert terminal.report =~ "openai_key=absent"
|
|
266
|
+ |
assert terminal.report =~ "SCV deployment on scvrun/sample"
|
|
267
|
+ |
|
|
268
|
+ |
# The report is a durable assistant message in the conversation.
|
|
269
|
+ |
assert terminal.report_message_id != nil
|
|
270
|
+ |
|
|
271
|
+ |
# The disposable workspace does not outlive the run. Cleanup happens after
|
|
272
|
+ |
# the terminal row commits, so this waits for it rather than racing it.
|
|
273
|
+ |
assert await_removed(terminal.delegation["workspace_path"])
|
|
274
|
+ |
end
|
|
275
|
+ |
end
|
|
276
|
+ |
|
|
277
|
+ |
# ── helpers ────────────────────────────────────────────────────────────────
|
|
278
|
+ |
|
|
279
|
+ |
defp base_deploy_settings do
|
|
280
|
+ |
[
|
|
281
|
+ |
model: @model,
|
|
282
|
+ |
reasoning_effort: "low",
|
|
283
|
+ |
concurrency_limit: 2,
|
|
284
|
+ |
wall_clock_ms: 900_000,
|
|
285
|
+ |
maximum_output_bytes: 16_777_216,
|
|
286
|
+ |
output_root: "/var/lib/openagents/scv/opencode-runs"
|
|
287
|
+ |
]
|
|
288
|
+ |
end
|
|
289
|
+ |
|
|
290
|
+ |
defp account(login) do
|
|
291
|
+ |
user = AccountsFixtures.repository_user_fixture(login)
|
|
292
|
+ |
{:ok, conversation} = Conversations.ensure_conversation(user)
|
|
293
|
+ |
%{user: user, conversation: conversation}
|
|
294
|
+ |
end
|
|
295
|
+ |
|
|
296
|
+ |
defp operator_account(login) do
|
|
297
|
+ |
%{user: user} = built = account(login)
|
|
298
|
+ |
configured = Application.get_env(:openagents, :admin_github_ids, [])
|
|
299
|
+ |
Application.put_env(:openagents, :admin_github_ids, [user.github_id | configured])
|
|
300
|
+ |
built
|
|
301
|
+ |
end
|
|
302
|
+ |
|
|
303
|
+ |
defp context(conversation) do
|
|
304
|
+ |
%ExecutionContext{
|
|
305
|
+ |
scope: "browser_conversation",
|
|
306
|
+ |
scope_ref: "conversation:#{conversation.id}",
|
|
307
|
+ |
authorities: MapSet.new(["scv.deploy"]),
|
|
308
|
+ |
surface: "text",
|
|
309
|
+ |
conversation_id: conversation.id,
|
|
310
|
+ |
owner_visitor_id: conversation.visitor_id
|
|
311
|
+ |
}
|
|
312
|
+ |
end
|
|
313
|
+ |
|
|
314
|
+ |
defp seed_repository!(user, owner, name) do
|
|
315
|
+ |
{:ok, repository} =
|
|
316
|
+ |
Repositories.create_repository(%{
|
|
317
|
+ |
owner: owner,
|
|
318
|
+ |
name: name,
|
|
319
|
+ |
visibility: "public",
|
|
320
|
+ |
default_branch: "main",
|
|
321
|
+ |
created_by_user_id: user.id
|
|
322
|
+ |
})
|
|
323
|
+ |
|
|
324
|
+ |
path = Repos.ensure_repo!(repository.storage_key, "main")
|
|
325
|
+ |
{blob, 0} = plumb(path, ["hash-object", "-w", "--stdin"], "an SCV fixture repository\n")
|
|
326
|
+ |
{tree, 0} = plumb(path, ["mktree"], "100644 blob #{String.trim(blob)}\tREADME.md\n")
|
|
327
|
+ |
|
|
328
|
+ |
{commit, 0} =
|
|
329
|
+ |
plumb(path, ["commit-tree", String.trim(tree), "-m", "seed"], "",
|
|
330
|
+ |
env: [
|
|
331
|
+ |
{"GIT_AUTHOR_NAME", "t"},
|
|
332
|
+ |
{"GIT_AUTHOR_EMAIL", "t@t"},
|
|
333
|
+ |
{"GIT_COMMITTER_NAME", "t"},
|
|
334
|
+ |
{"GIT_COMMITTER_EMAIL", "t@t"}
|
|
335
|
+ |
]
|
|
336
|
+ |
)
|
|
337
|
+ |
|
|
338
|
+ |
{_output, 0} = Repos.git(path, ["update-ref", "refs/heads/main", String.trim(commit)])
|
|
339
|
+ |
repository
|
|
340
|
+ |
end
|
|
341
|
+ |
|
|
342
|
+ |
defp plumb(path, args, stdin, options \\ []) do
|
|
343
|
+ |
input = Path.join(System.tmp_dir!(), "plumb-#{System.unique_integer([:positive])}")
|
|
344
|
+ |
File.write!(input, stdin)
|
|
345
|
+ |
|
|
346
|
+ |
try do
|
|
347
|
+ |
System.cmd(
|
|
348
|
+ |
"sh",
|
|
349
|
+ |
["-c", ~s(exec git --git-dir "$GD" "$@" < "$IN"), "sh"] ++ args,
|
|
350
|
+ |
env: [{"GD", path}, {"IN", input}] ++ Keyword.get(options, :env, [])
|
|
351
|
+ |
)
|
|
352
|
+ |
after
|
|
353
|
+ |
File.rm(input)
|
|
354
|
+ |
end
|
|
355
|
+ |
end
|
|
356
|
+ |
|
|
357
|
+ |
defp await_terminal(job_id) do
|
|
358
|
+ |
Enum.reduce_while(1..200, nil, fn _attempt, _accumulator ->
|
|
359
|
+ |
job = Repo.get!(Job, job_id)
|
|
360
|
+ |
|
|
361
|
+ |
if job.status in Job.terminal_statuses() do
|
|
362
|
+ |
{:halt, job}
|
|
363
|
+ |
else
|
|
364
|
+ |
Process.sleep(50)
|
|
365
|
+ |
{:cont, job}
|
|
366
|
+ |
end
|
|
367
|
+ |
end)
|
|
368
|
+ |
end
|
|
369
|
+ |
|
|
370
|
+ |
defp await_removed(path) when is_binary(path) do
|
|
371
|
+ |
Enum.reduce_while(1..100, false, fn _attempt, _accumulator ->
|
|
372
|
+ |
if File.exists?(path) do
|
|
373
|
+ |
Process.sleep(20)
|
|
374
|
+ |
{:cont, false}
|
|
375
|
+ |
else
|
|
376
|
+ |
{:halt, true}
|
|
377
|
+ |
end
|
|
378
|
+ |
end)
|
|
379
|
+ |
end
|
|
380
|
+ |
|
|
381
|
+ |
# Echoes the invocation back as one OpenCode text event, so the test can
|
|
382
|
+ |
# assert on what the process actually received.
|
|
383
|
+ |
defp fake_opencode do
|
|
384
|
+ |
"""
|
|
385
|
+ |
#!/bin/sh
|
|
386
|
+ |
model=""
|
|
387
|
+ |
while [ $# -gt 0 ]; do
|
|
388
|
+ |
case "$1" in
|
|
389
|
+ |
--model) model="$2"; shift 2 ;;
|
|
390
|
+ |
*) shift ;;
|
|
391
|
+ |
esac
|
|
392
|
+ |
done
|
|
393
|
+ |
prompt=$(cat)
|
|
394
|
+ |
if [ -z "$prompt" ]; then exit 30; fi
|
|
395
|
+ |
if [ "${OPENAI_API_KEY+x}" = "x" ]; then openai_key=present; else openai_key=absent; fi
|
|
396
|
+ |
printf '{"type":"text","timestamp":1,"sessionID":"ses_scv","part":{"type":"text","text":"model=%s fetch=%s openai_key=%s"}}\\n' \\
|
|
397
|
+ |
"$model" "${OPENCODE_DISABLE_MODELS_FETCH}" "$openai_key"
|
|
398
|
+ |
"""
|
|
399
|
+ |
end
|
|
400
|
+ |
end
|