OpenAgents

Commit

Keep a rolling replacement node in load balancer rotation

a79bfce11b00 · Devin AI · · parent 82136c3934a3 

Keep a rolling replacement node in load balancer rotation

A node booted with the newest pending rolling target's image degraded when
the older live target's artifact was structural, so its /healthz reported
503 for the whole roll. Readiness now accepts an image that matches the
newest needs_rolling_replace target.

Co-Authored-By: Christopher David <chris@openagents.com>
Co-Authored-By
Christopher David <chris@openagents.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified lib/openagents/forge/boot_converge.ex
  • modified test/openagents/forge/boot_converge_test.exs

Diff

2 files changed, +85 -3

lib/openagents/forge/boot_converge.ex modified +38 -3

@@ -146,8 +146,12 @@ defmodule OpenAgents.Forge.BootConverge do

146 146
147 147 
  defp durable_target_ready?(repo, convergence) do
148 148 
    case Targets.current(repo) do
149 
      %{status: "deploying"} -> false
150 
      _not_deploying -> current_target_matches?(repo, convergence)
149 
      %{status: "deploying"} ->
150 
        false
151
152 
      _not_deploying ->
153 
        current_target_matches?(repo, convergence) or
154 
          rolling_convergence_matches?(repo, convergence)
151 155 
    end
152 156 
  end
153 157

@@ -188,10 +192,41 @@ defmodule OpenAgents.Forge.BootConverge do

188 192 
        image_ready("no_live_target", attempts)
189 193
190 194 
      %{sha: sha, details: details} = target ->
191 
        converge_target(repo, target, sha, details || %{}, attempts)
195 
        if rolling_target_matches?(repo) do
196 
          image_ready("image_matches_rolling_target", attempts)
197 
        else
198 
          converge_target(repo, target, sha, details || %{}, attempts)
199 
        end
200 
    end
201 
  end
202
203 
  # A node whose booted image carries the newest target's revision is a
204 
  # participant in an operator rolling replacement. It must enter readiness
205 
  # so the load balancer keeps it in rotation while the remaining nodes are
206 
  # replaced; settlement later flips the target live and the periodic
207 
  # convergence attempt reports `image_matches_live`.
208 
  defp rolling_target_matches?(repo) do
209 
    case Targets.current(repo) do
210 
      %{status: "needs_rolling_replace", sha: sha} ->
211 
        OpenAgents.BuildInfo.revision() == sha
212
213 
      _other ->
214 
        false
192 215 
    end
193 216 
  end
194 217
218 
  defp rolling_convergence_matches?(repo, %{
219 
         "reason" => "image_matches_rolling_target",
220 
         "sha" => sha
221 
       }) do
222 
    case Targets.current(repo) do
223 
      %{status: "needs_rolling_replace", sha: ^sha} -> OpenAgents.BuildInfo.revision() == sha
224 
      _other -> false
225 
    end
226 
  end
227
228 
  defp rolling_convergence_matches?(_repo, _convergence), do: false
229
195 230 
  defp converge_target(repo, target, sha, details, attempts) do
196 231 
    if image_target_matches?(sha, details) do
197 232 
      image_ready("image_matches_live", attempts, sha)
test/openagents/forge/boot_converge_test.exs modified +47

@@ -385,6 +385,53 @@ defmodule OpenAgents.Forge.BootConvergeTest do

385 385 
    refute Code.ensure_loaded?(module)
386 386 
  end
387 387
388 
  test "a node running a pending rolling target's image stays in readiness" do
389 
    # An older live target whose artifact this node cannot install: without
390 
    # the rolling-target branch this node would degrade and leave rotation.
391 
    insert_target!("live", %{})
392
393 
    rolling =
394 
      %Target{}
395 
      |> Target.changeset(%{
396 
        repo: @repo,
397 
        sha: String.duplicate("e", 40),
398 
        promoted_by: "operator:t",
399 
        status: "promoted"
400 
      })
401 
      |> Repo.insert!()
402 
      |> Ecto.Changeset.change(%{
403 
        status: "needs_rolling_replace",
404 
        sha: OpenAgents.BuildInfo.revision()
405 
      })
406 
      |> Repo.update!()
407
408 
    previous_enabled = Application.get_env(:openagents, :forge_boot_converge_enabled)
409 
    Application.put_env(:openagents, :forge_boot_converge_enabled, true)
410
411 
    on_exit(fn -> restore_env(:forge_boot_converge_enabled, previous_enabled) end)
412
413 
    assert %{
414 
             "state" => "image",
415 
             "ready" => true,
416 
             "reason" => "image_matches_rolling_target"
417 
           } = BootConverge.converge(@repo)
418
419 
    assert BootConverge.ready?(@repo)
420
421 
    # Once the rolling target settles, readiness follows the live-target path.
422 
    rolling
423 
    |> Ecto.Changeset.change(%{
424 
      status: "live",
425 
      details: %{"image_digest" => OpenAgents.BuildInfo.image_digest()}
426 
    })
427 
    |> Repo.update!()
428
429 
    assert %{"state" => "image", "reason" => "image_matches_live", "ready" => true} =
430 
             BootConverge.converge(@repo)
431
432 
    assert BootConverge.ready?(@repo)
433 
  end
434
388 435 
  test "rolling target stays degraded when its image digest does not match the runtime" do
389 436 
    runtime_sha = OpenAgents.BuildInfo.revision()
390 437 
    previous_digest = Application.get_env(:openagents, :image_digest)

This page updates live while a promote is in flight · changelog