OpenAgents

Commit

Stabilize deployment gate coverage

6999983b4487 · Christopher David · · parent 3a96e867a7e2 

Stabilize deployment gate coverage

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified test/openagents/forge/boot_converge_test.exs
  • modified test/openagents/forge/deployment_node_test.exs
  • modified test/openagents/forge/hot_loader_test.exs

Diff

3 files changed, +130 -9

test/openagents/forge/boot_converge_test.exs modified +26 -9

@@ -188,11 +188,14 @@ defmodule OpenAgents.Forge.BootConvergeTest do

188 188 
    orphan_digest = String.duplicate("a", 64)
189 189 
    orphan_path = Path.join([Repos.data_dir(), "beams", orphan_digest <> ".tar"])
190 190 
    File.write!(orphan_path, "obsolete-cache-entry")
191 
    unrelated_path = Path.join([Repos.data_dir(), "beams", "README"])
192 
    File.write!(unrelated_path, "operator note")
191 193
192 194 
    assert %{"state" => "converged"} = BootConverge.converge(@repo)
193 195 
    assert File.exists?(Path.join(Repos.data_dir(), current.details["artifact"]))
194 196 
    assert File.exists?(Path.join(Repos.data_dir(), predecessor.details["artifact"]))
195 197 
    refute File.exists?(orphan_path)
198 
    assert File.exists?(unrelated_path)
196 199
197 200 
    :code.purge(current_module)
198 201 
    :code.delete(current_module)

@@ -255,8 +258,8 @@ defmodule OpenAgents.Forge.BootConvergeTest do

255 258 
    previous_max = Application.get_env(:openagents, :forge_boot_retry_max_ms)
256 259
257 260 
    Application.put_env(:openagents, :forge_boot_converge_enabled, true)
258 
    Application.put_env(:openagents, :forge_boot_retry_min_ms, 10)
259 
    Application.put_env(:openagents, :forge_boot_retry_max_ms, 20)
261 
    Application.put_env(:openagents, :forge_boot_retry_min_ms, 10_000)
262 
    Application.put_env(:openagents, :forge_boot_retry_max_ms, 20_000)
260 263
261 264 
    name = Module.concat(__MODULE__, "Retry#{System.unique_integer([:positive])}")
262 265

@@ -272,8 +275,16 @@ defmodule OpenAgents.Forge.BootConvergeTest do

272 275
273 276 
    refute convergence["ready"]
274 277 
    assert convergence["state"] == "degraded"
275 
    assert convergence["retry_in_ms"] == 10
276 
    assert server_state.retry_ms == 20
278 
    assert convergence["retry_in_ms"] == 10_000
279 
    assert server_state.retry_ms == 20_000
280
281 
    send(name, :retry_convergence)
282 
    server_state = :sys.get_state(name)
283 
    convergence = BootConverge.state()
284
285 
    assert convergence["attempts"] == 2
286 
    assert convergence["retry_in_ms"] == 20_000
287 
    assert server_state.retry_ms == 20_000
277 288
278 289 
    on_exit(fn ->
279 290 
      restore_env(:forge_boot_converge_enabled, previous_enabled)

@@ -288,8 +299,8 @@ defmodule OpenAgents.Forge.BootConvergeTest do

288 299 
    previous_max = Application.get_env(:openagents, :forge_boot_retry_max_ms)
289 300
290 301 
    Application.put_env(:openagents, :forge_boot_converge_enabled, true)
291 
    Application.put_env(:openagents, :forge_boot_retry_min_ms, 10)
292 
    Application.put_env(:openagents, :forge_boot_retry_max_ms, 20)
302 
    Application.put_env(:openagents, :forge_boot_retry_min_ms, 10_000)
303 
    Application.put_env(:openagents, :forge_boot_retry_max_ms, 20_000)
293 304
294 305 
    name = Module.concat(__MODULE__, "Ready#{System.unique_integer([:positive])}")
295 306

@@ -301,13 +312,13 @@ defmodule OpenAgents.Forge.BootConvergeTest do

301 312 
    )
302 313
303 314 
    assert %{"state" => "image", "ready" => true} = BootConverge.state()
304 
    assert :sys.get_state(name).retry_ms == 10
315 
    assert :sys.get_state(name).retry_ms == 10_000
305 316
306 317 
    send(name, :retry_convergence)
307 
    assert :sys.get_state(name).retry_ms == 10
318 
    assert :sys.get_state(name).retry_ms == 10_000
308 319
309 320 
    send(name, :irrelevant_message)
310 
    assert :sys.get_state(name).retry_ms == 10
321 
    assert :sys.get_state(name).retry_ms == 10_000
311 322
312 323 
    on_exit(fn ->
313 324 
      restore_env(:forge_boot_converge_enabled, previous_enabled)

@@ -318,6 +329,10 @@ defmodule OpenAgents.Forge.BootConvergeTest do

318 329
319 330 
  test "image-matching legacy target remains ready without artifact metadata" do
320 331 
    target = insert_target!("live", %{})
332 
    previous_enabled = Application.get_env(:openagents, :forge_boot_converge_enabled)
333 
    Application.put_env(:openagents, :forge_boot_converge_enabled, true)
334
335 
    on_exit(fn -> restore_env(:forge_boot_converge_enabled, previous_enabled) end)
321 336
322 337 
    target
323 338 
    |> Ecto.Changeset.change(%{sha: OpenAgents.BuildInfo.revision()})

@@ -329,6 +344,8 @@ defmodule OpenAgents.Forge.BootConvergeTest do

329 344 
             "reason" => "image_matches_live",
330 345 
             "sha" => "image"
331 346 
           } = BootConverge.converge(@repo)
347
348 
    assert BootConverge.ready?(@repo)
332 349 
  end
333 350
334 351 
  test "an unreadable cache entry degrades with a bounded reason" do
test/openagents/forge/deployment_node_test.exs modified +35

@@ -6,6 +6,7 @@ defmodule OpenAgents.Forge.DeploymentNodeTest do

6 6 
  alias OpenAgents.Forge.BuildProtocol
7 7 
  alias OpenAgents.Forge.Deployment
8 8 
  alias OpenAgents.Forge.DeploymentNode
9 
  alias OpenAgents.Forge.Repos
9 10 
  alias OpenAgents.Forge.Target
10 11
11 12 
  setup do

@@ -67,6 +68,12 @@ defmodule OpenAgents.Forge.DeploymentNodeTest do

67 68 
    assert DeploymentNode.health()["revision"] == request.sha
68 69 
  end
69 70
71 
  test "participant start APIs refuse a duplicate registered participant" do
72 
    pid = Process.whereis(DeploymentNode)
73 
    assert {:error, {:already_started, ^pid}} = DeploymentNode.start()
74 
    assert {:error, {:already_started, ^pid}} = DeploymentNode.start_link()
75 
  end
76
70 77 
  test "rollback restores and verifies exact prior object code" do
71 78 
    fixture = versions("ExactRollback")
72 79 
    request = request(fixture)

@@ -289,6 +296,20 @@ defmodule OpenAgents.Forge.DeploymentNodeTest do

289 296 
             DeploymentNode.prepare(request(wrong_runtime))
290 297 
  end
291 298
299 
  test "artifact cache refuses a digest-address collision" do
300 
    fixture = absent_version("CacheCollision")
301 
    request = request(fixture)
302
303 
    assert {:ok, %{"token" => token}} = DeploymentNode.prepare(request)
304 
    assert {:ok, %{"restored" => true}} = DeploymentNode.rollback(request.deployment_id, token)
305
306 
    cache_path = Path.join([Repos.data_dir(), "beams", request.artifact_digest <> ".tar"])
307 
    File.write!(cache_path, "tampered artifact")
308
309 
    assert {:error, :digest_collision} = DeploymentNode.prepare(request)
310 
    assert DeploymentNode.health()["ready"]
311 
  end
312
292 313 
  test "fault injection is bounded and participant phase notifications are content-free" do
293 314 
    fixture = absent_version("FaultBoundary")
294 315 
    request = request(fixture)

@@ -486,6 +507,20 @@ defmodule OpenAgents.Forge.DeploymentNodeTest do

486 507
487 508 
    Application.put_env(:openagents, :forge_expected_fleet_size, 1)
488 509
510 
    calls = :counters.new(1, [])
511
512 
    membership = fn ->
513 
      :counters.add(calls, 1, 1)
514
515 
      if :counters.get(calls, 1) == 1,
516 
        do: [Node.self()],
517 
        else: [Node.self(), :unexpected@node]
518 
    end
519
520 
    assert {:error, changed} = run_deployment(fixture, members: membership)
521 
    assert changed.error_code == "membership_changed"
522 
    assert changed.node_results[to_string(Node.self())] == "restored"
523
489 524 
    :sys.replace_state(DeploymentNode, fn state ->
490 525 
      %{state | divergence: "test_divergence"}
491 526 
    end)
test/openagents/forge/hot_loader_test.exs modified +69

@@ -169,6 +169,54 @@ defmodule OpenAgents.Forge.HotLoaderTest do

169 169 
    assert_receive {:forge_deploy, %{repo: "openagents.com", sha: ^sha, result: "live"}}
170 170 
  end
171 171
172 
  test "a participant prepare refusal becomes a durable failed deployment", %{loader: loader} do
173 
    %{mod: mod, name: name, binary: binary} = compiled_scratch_module()
174 
    sha = unique_sha()
175 
    target = insert_target(sha, "built")
176 
    artifact = artifact([{name, binary}], sha)
177
178 
    :sys.replace_state(DeploymentNode, fn state ->
179 
      %{state | faults: %{prepare: :error}}
180 
    end)
181
182 
    Phoenix.PubSub.subscribe(OpenAgents.PubSub, @deploys_topic)
183 
    broadcast_build_ready(loader, build_payload(target, sha, artifact))
184
185 
    refute Code.ensure_loaded?(mod)
186 
    assert Repo.get!(Target, target.id).status == "failed"
187
188 
    receipt = deploy_receipt(sha)
189 
    assert receipt.result == "failed"
190 
    assert receipt.error_code == "prepare_failed"
191 
    assert receipt.rollback_verified == false
192
193 
    assert_receive {:forge_deploy, %{repo: "openagents.com", sha: ^sha, result: "failed"}}
194 
  end
195
196 
  test "a finalize refusal keeps durable live authority while fencing readiness", %{
197 
    loader: loader
198 
  } do
199 
    %{mod: mod, name: name, revision: revision, binary: binary} = compiled_scratch_module()
200 
    sha = unique_sha()
201 
    target = insert_target(sha, "built")
202 
    artifact = artifact([{name, binary}], sha)
203
204 
    :sys.replace_state(DeploymentNode, fn state ->
205 
      %{state | faults: %{finalize: :error}}
206 
    end)
207
208 
    Phoenix.PubSub.subscribe(OpenAgents.PubSub, @deploys_topic)
209 
    broadcast_build_ready(loader, build_payload(target, sha, artifact))
210
211 
    assert Code.ensure_loaded?(mod)
212 
    assert mod.revision() == revision
213 
    assert Repo.get!(Target, target.id).status == "live"
214 
    assert deploy_receipt(sha).result == "live"
215 
    refute DeploymentNode.health()["ready"]
216
217 
    assert_receive {:forge_deploy, %{repo: "openagents.com", sha: ^sha, result: "live"}}
218 
  end
219
172 220 
  test "allowlist refusal: off-allowlist module means needs_rolling_replace and no load", %{
173 221 
    loader: loader
174 222 
  } do

@@ -281,6 +329,27 @@ defmodule OpenAgents.Forge.HotLoaderTest do

281 329 
    refute HotLoader.allowlisted?("Anything", [])
282 330 
  end
283 331
332 
  test "extract!/1 verifies both successful and malformed artifacts" do
333 
    %{mod: mod, name: name, binary: binary} = compiled_scratch_module()
334 
    built = artifact([{name, binary}], unique_sha())
335
336 
    assert [{^mod, extracted_binary}] = HotLoader.extract!(built.artifact)
337 
    assert is_binary(extracted_binary)
338 
    assert {:ok, {^mod, _md5}} = :beam_lib.md5(extracted_binary)
339
340 
    malformed = malformed_artifact([{name, binary}])
341
342 
    assert_raise RuntimeError, ~r/artifact verification failed/, fn ->
343 
      HotLoader.extract!(malformed)
344 
    end
345 
  end
346
347 
  test "duplicate startup and unrelated messages leave the loader intact", %{loader: loader} do
348 
    assert {:error, {:already_started, ^loader}} = HotLoader.start_link()
349 
    send(loader, :unrelated_message)
350 
    assert :sys.get_state(loader) == %{}
351 
  end
352
284 353 
  test "OpenAgents.BuildInfo compiled-in revision is the boot image" do
285 354 
    assert OpenAgents.BuildInfo.revision() == "image"
286 355 
    assert OpenAgents.BuildInfo.loaded_at() == nil

This page updates live while a promote is in flight · changelog