OpenAgents

Commit

Let the forge build a promoted target instead of asserting it did

2c123e1f625f · AtlantisPleb · · parent 496dcdfea5d7 

Let the forge build a promoted target instead of asserting it did

The release advanced a target through promoted, building, and built by hand
and then rolled the fleet. `OpenAgents.Forge.Builder` is what actually builds
a promoted target — it subscribes to the promotion broadcast, builds the
artifact, and writes the complete build receipt that
`finish_rolling_replacement/2` requires — and the hand-advance raced straight
past its window. The statuses then said a build had happened while no receipt
existed, so every node rolled onto the new image and settlement refused with
`complete_build_receipt_not_found`, leaving production serving a release the
forge would not record as live.

It now promotes and waits for the builder, and it treats a target as built
only when the receipt is really there — a target parked at
`needs_rolling_replace` with no receipt is exactly what the old path left
behind, and reusing one walks into the same refusal.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SoZMfWRSGnf6FZX2Ar9rQ2
Co-Authored-By
Claude Fable 5 <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified ops/deploy/release-to-production.sh

Diff

1 file changed, +61 -11

ops/deploy/release-to-production.sh modified +61 -11

@@ -78,21 +78,71 @@ previous_sha = (live && live.sha) || "$sha"

78 78 
previous_image_digest = (live && live.details["image_digest"]) || "$digest"
79 79 
expected_nodes = Enum.sort(["openagents@10.128.0.4", "openagents@10.128.0.110", "openagents@10.128.0.111"])
80 80
81 
# The forge builds a promoted target itself: OpenAgents.Forge.Builder
82 
# subscribes to the promotion broadcast, builds the artifact, and writes the
83 
# complete build receipt that finish_rolling_replacement/2 requires. This used
84 
# to advance promoted -> building -> built -> needs_rolling_replace by hand,
85 
# which raced the builder past its own window: the statuses said a build had
86 
# happened while no receipt existed, every node rolled, and settle then refused
87 
# with complete_build_receipt_not_found on a fleet already serving the new
88 
# image. Promote, then wait for the builder to say what it found.
89 
#
90 
# No backticks anywhere in this heredoc: it is unquoted so the shell expands it,
91 
# and a backticked word in a comment runs as a command.
92 
import Ecto.Query
93
94 
built? = fn target ->
95 
  OpenAgents.Repo.exists?(
96 
    from b in OpenAgents.Forge.BuildReceipt,
97 
      where: b.target_id == ^target.id and b.status == "complete"
98 
  )
99 
end
100
101 
await_built = fn await_built, remaining ->
102 
  t = OpenAgents.Forge.Targets.current("openagents.com")
103
104 
  cond do
105 
    t == nil or t.sha != "$sha" ->
106 
      raise "current target is #{inspect(t && t.sha)}, not $sha"
107
108 
    t.status == "failed" ->
109 
      raise "target #{t.id} failed to build"
110
111 
    t.status in ["needs_rolling_replace", "built"] and built?.(t) ->
112 
      t
113
114 
    remaining > 0 ->
115 
      Process.sleep(5_000)
116 
      await_built.(await_built, remaining - 1)
117
118 
    true ->
119 
      raise "target #{t.id} is #{t.status} with no complete build receipt"
120 
  end
121 
end
122
123 
promote_fresh = fn ->
124 
  {:ok, _} =
125 
    OpenAgents.Forge.Targets.promote("openagents.com", "$sha", "operator:14167547",
126 
      details: %{"source" => "operator_console"}
127 
    )
128
129 
  await_built.(await_built, 120)
130 
end
131
132 
# A target only counts as reusable when the receipt settlement needs is really
133 
# there. One parked at needs_rolling_replace with no receipt is what the old
134 
# hand-advance left behind, and reusing it walks into the same refusal, so it
135 
# is promoted again rather than waited on.
81 136 
target =
82 137 
  case OpenAgents.Forge.Targets.current("openagents.com") do
83 138 
    %{sha: "$sha", status: "needs_rolling_replace"} = t ->
84 
      t
85 
    %{sha: "$sha", status: "promoted"} = t ->
86 
      {:ok, t2} = OpenAgents.Forge.Targets.advance(t.id, "building")
87 
      {:ok, t3} = OpenAgents.Forge.Targets.advance(t2.id, "built")
88 
      {:ok, t4} = OpenAgents.Forge.Targets.advance(t3.id, "needs_rolling_replace")
89 
      t4
139 
      if built?.(t), do: t, else: promote_fresh.()
140
141 
    %{sha: "$sha", status: status} when status in ["promoted", "building", "built"] ->
142 
      await_built.(await_built, 120)
143
90 144 
    _ ->
91 
      {:ok, t} = OpenAgents.Forge.Targets.promote("openagents.com", "$sha", "operator:14167547", details: %{"source" => "operator_console"})
92 
      {:ok, t2} = OpenAgents.Forge.Targets.advance(t.id, "building")
93 
      {:ok, t3} = OpenAgents.Forge.Targets.advance(t2.id, "built")
94 
      {:ok, t4} = OpenAgents.Forge.Targets.advance(t3.id, "needs_rolling_replace")
95 
      t4
145 
      promote_fresh.()
96 146 
  end
97 147
98 148 
{:ok, authorized} =

This page updates live while a promote is in flight · changelog