OpenAgents

Commit

Harden staging rolling replacement

2e9b3f23fceb · AtlantisPleb · · parent 2a2aa0daf679 

Harden staging rolling replacement

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified docs/operations/release-deployment-fallbacks.md
  • modified infra/staging/main.tf
  • modified infra/staging/templates/deployer-startup.sh.tftpl
  • modified infra/staging/templates/fleet-startup.sh.tftpl
  • modified infra/staging/tests/safety.tftest.hcl
  • modified lib/openagents/forge/rolling_provider/gcp.ex
  • modified lib/openagents/forge/rolling_provider/gcp/deployer.ex
  • added test/openagents/forge/rolling_provider/gcp/deployer_test.exs
  • modified test/openagents/forge/rolling_provider/gcp_test.exs

Diff

9 files changed, +133 -8

docs/operations/release-deployment-fallbacks.md modified +5 -4

@@ -155,10 +155,11 @@ For each node, the coordinator performs this sequence:

155 155 
   access, source SHA, and image digest.
156 156 
7. Recheck exact fleet membership before selecting another node.
157 157
158 
An Erlang distribution disconnect is an expected transient state while a VM
159 
reboots. The GCP provider reports that node as unavailable so the coordinator
160 
continues its bounded readiness polling. Other RPC errors fail the rollout
161 
closed.
158 
An Erlang distribution transport failure is an expected transient state while
159 
a VM reboots. The GCP provider reports that node as unavailable so the
160 
coordinator continues its bounded readiness polling. An invalid probe response
161 
fails immediately, and a node that remains unavailable fails at the bounded
162 
timeout.
162 163
163 164 
If a node does not rejoin, the coordinator asks the provider to restore the
164 165 
last-known-good SHA and digest, waits for that node's full health, records the
infra/staging/main.tf modified +6 -2

@@ -750,8 +750,12 @@ resource "google_compute_instance" "deployer" {

750 750 
    openagents-controller-sha   = ""
751 751 
    openagents-cookie-secret    = google_secret_manager_secret.runtime["openagents-staging-release-cookie"].secret_id
752 752 
    startup-script = templatefile("${path.module}/templates/deployer-startup.sh.tftpl", {
753 
      project_id = var.staging_project_id
754 
      region     = var.region
753 
      image_repository       = "${var.region}-docker.pkg.dev/${var.staging_project_id}/openagents-staging/openagents"
754 
      production_project_id  = var.production_project_id
755 
      project_id             = var.staging_project_id
756 
      region                 = var.region
757 
      rolling_instances_json = jsonencode({ for instance, ip in local.nodes : "openagents@${ip}" => instance })
758 
      zone                    = var.zone
755 759 
    })
756 760 
  }
757 761
infra/staging/templates/deployer-startup.sh.tftpl modified +5

@@ -71,6 +71,11 @@ docker run --detach \

71 71 
  --restart always \
72 72 
  --env-file /run/openagents/deployer.env \
73 73 
  --env "OPENAGENTS_CONTROLLER_SHA=$source_sha" \
74 
  --env "OPENAGENTS_GCP_IMAGE_REPOSITORY=${image_repository}" \
75 
  --env 'OPENAGENTS_GCP_ROLLING_INSTANCES_JSON=${rolling_instances_json}' \
76 
  --env "OPENAGENTS_GCP_ROLLING_PROJECT_ID=${project_id}" \
77 
  --env "OPENAGENTS_GCP_ROLLING_ZONE=${zone}" \
78 
  --env "OPENAGENTS_PRODUCTION_PROJECT_ID=${production_project_id}" \
74 79 
  --entrypoint /bin/sh \
75 80 
  "$image" -c '
76 81 
    set -eu
infra/staging/templates/fleet-startup.sh.tftpl modified +4

@@ -154,6 +154,10 @@ mkdir -p "$DOCKER_CONFIG"

154 154 
chmod 0700 "$DOCKER_CONFIG"
155 155 
docker-credential-gcr configure-docker --registries=${region}-docker.pkg.dev
156 156
157 
# Fleet images are immutable and recoverable from Artifact Registry. Reclaim
158 
# only images that no running container uses before pulling the next candidate.
159 
docker image prune --all --force
160
157 161 
cloud_sql_proxy_image=gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:825d5e4ce70d38bd0006c9eea15a6a2e2983e87b31ac6924d33e2dba56eafc9f
158 162 
docker pull "$cloud_sql_proxy_image"
159 163 
docker rm --force openagents-cloud-sql-proxy 2>/dev/null || true
infra/staging/tests/safety.tftest.hcl modified +17

@@ -46,6 +46,23 @@ run "isolated_topology" {

46 46 
    error_message = "The staging deployer must use a hidden Erlang node so it can probe every fleet member without joining the global cluster."
47 47 
  }
48 48
49 
  assert {
50 
    condition = alltrue([
51 
      strcontains(google_compute_instance.deployer.metadata["startup-script"], "OPENAGENTS_GCP_ROLLING_PROJECT_ID"),
52 
      strcontains(google_compute_instance.deployer.metadata["startup-script"], "OPENAGENTS_GCP_ROLLING_INSTANCES_JSON"),
53 
      strcontains(google_compute_instance.deployer.metadata["startup-script"], "OPENAGENTS_PRODUCTION_PROJECT_ID")
54 
    ])
55 
    error_message = "The staging deployer must receive its bounded, non-secret rolling-provider inventory."
56 
  }
57
58 
  assert {
59 
    condition = strcontains(
60 
      google_compute_instance.fleet["openagents-fleet-1"].metadata["startup-script"],
61 
      "docker image prune --all --force"
62 
    )
63 
    error_message = "Fleet startup must reclaim only unused registry-backed images before pulling a candidate."
64 
  }
65
49 66 
  assert {
50 67 
    condition     = google_sql_database_instance.staging.deletion_protection
51 68 
    error_message = "The staging database must keep Terraform deletion protection enabled."
lib/openagents/forge/rolling_provider/gcp.ex modified +1 -2

@@ -136,8 +136,7 @@ defmodule OpenAgents.Forge.RollingProvider.Gcp do

136 136 
  defp probe(config, node, expected_fleet_size) do
137 137 
    case rpc(config, node, RollingNodeProbe, :status, [expected_fleet_size]) do
138 138 
      %{member: true} = result -> {:ok, result}
139 
      {:error, :noconnection} -> {:ok, unavailable_probe()}
140 
      {:error, reason} -> {:error, reason}
139 
      {:error, _transport_reason} -> {:ok, unavailable_probe()}
141 140 
      other -> {:error, {:invalid_node_probe, other}}
142 141 
    end
143 142 
  end
lib/openagents/forge/rolling_provider/gcp/deployer.ex modified +41

@@ -9,6 +9,7 @@ defmodule OpenAgents.Forge.RollingProvider.Gcp.Deployer do

9 9 
  """
10 10
11 11 
  @sha_pattern ~r/\A[0-9a-f]{40}\z/
12 
  @deployer_node :"openagents-deployer@openagents-deployer.staging.internal"
12 13
13 14 
  @doc false
14 15 
  def start do

@@ -19,7 +20,47 @@ defmodule OpenAgents.Forge.RollingProvider.Gcp.Deployer do

19 20 
      raise "deployer image revision does not match its assigned Git SHA"
20 21 
    end
21 22
23 
    :ok = configure_provider!()
22 24 
    {:ok, _applications} = Application.ensure_all_started(:req)
23 25 
    Process.sleep(:infinity)
24 26 
  end
27
28 
  @doc false
29 
  def configure_provider!(environment \\ System.get_env()) when is_map(environment) do
30 
    config = [
31 
      project_id: fetch!(environment, "OPENAGENTS_GCP_ROLLING_PROJECT_ID"),
32 
      production_project_id: fetch!(environment, "OPENAGENTS_PRODUCTION_PROJECT_ID"),
33 
      zone: fetch!(environment, "OPENAGENTS_GCP_ROLLING_ZONE"),
34 
      instances: decode_instances!(environment),
35 
      image_repository: fetch!(environment, "OPENAGENTS_GCP_IMAGE_REPOSITORY"),
36 
      deployer_node: @deployer_node,
37 
      rpc_timeout_ms: 5_000,
38 
      compute_timeout_ms: 300_000
39 
    ]
40
41 
    case OpenAgents.Forge.RollingProvider.Gcp.validate_config(config) do
42 
      :ok ->
43 
        Application.put_env(:openagents, OpenAgents.Forge.RollingProvider.Gcp, config)
44
45 
      {:error, _reason} ->
46 
        raise ArgumentError, "staging rolling-provider configuration is invalid"
47 
    end
48 
  end
49
50 
  defp decode_instances!(environment) do
51 
    environment
52 
    |> fetch!("OPENAGENTS_GCP_ROLLING_INSTANCES_JSON")
53 
    |> Jason.decode()
54 
    |> case do
55 
      {:ok, instances} when is_map(instances) -> instances
56 
      _invalid -> raise ArgumentError, "staging rolling-provider configuration is invalid"
57 
    end
58 
  end
59
60 
  defp fetch!(environment, name) do
61 
    case Map.fetch(environment, name) do
62 
      {:ok, value} when is_binary(value) and value != "" -> value
63 
      _missing -> raise ArgumentError, "staging rolling-provider configuration is invalid"
64 
    end
65 
  end
25 66 
end
test/openagents/forge/rolling_provider/gcp/deployer_test.exs added +44

@@ -0,0 +1,44 @@

1 
defmodule OpenAgents.Forge.RollingProvider.Gcp.DeployerTest do
2 
  use ExUnit.Case, async: false
3
4 
  alias OpenAgents.Forge.RollingProvider.Gcp
5 
  alias OpenAgents.Forge.RollingProvider.Gcp.Deployer
6
7 
  @environment %{
8 
    "OPENAGENTS_GCP_IMAGE_REPOSITORY" =>
9 
      "us-central1-docker.pkg.dev/openagents-staging-project/openagents/openagents",
10 
    "OPENAGENTS_GCP_ROLLING_INSTANCES_JSON" =>
11 
      ~s({"openagents@10.42.0.11":"openagents-fleet-1","openagents@10.42.0.12":"openagents-fleet-2","openagents@10.42.0.13":"openagents-fleet-3"}),
12 
    "OPENAGENTS_GCP_ROLLING_PROJECT_ID" => "openagents-staging-project",
13 
    "OPENAGENTS_GCP_ROLLING_ZONE" => "us-central1-a",
14 
    "OPENAGENTS_PRODUCTION_PROJECT_ID" => "production-project"
15 
  }
16
17 
  setup do
18 
    previous = Application.get_env(:openagents, Gcp)
19 
    on_exit(fn -> restore_config(previous) end)
20 
    :ok
21 
  end
22
23 
  test "loads the bounded provider inventory for the minimal controller" do
24 
    assert :ok = Deployer.configure_provider!(@environment)
25
26 
    config = Application.fetch_env!(:openagents, Gcp)
27 
    assert config[:project_id] == "openagents-staging-project"
28 
    assert config[:production_project_id] == "production-project"
29 
    assert config[:zone] == "us-central1-a"
30 
    assert map_size(config[:instances]) == 3
31 
    assert config[:deployer_node] == :"openagents-deployer@openagents-deployer.staging.internal"
32 
  end
33
34 
  test "refuses malformed controller inventory" do
35 
    environment = Map.put(@environment, "OPENAGENTS_GCP_ROLLING_INSTANCES_JSON", "{}")
36
37 
    assert_raise ArgumentError, "staging rolling-provider configuration is invalid", fn ->
38 
      Deployer.configure_provider!(environment)
39 
    end
40 
  end
41
42 
  defp restore_config(nil), do: Application.delete_env(:openagents, Gcp)
43 
  defp restore_config(config), do: Application.put_env(:openagents, Gcp, config)
44 
end
test/openagents/forge/rolling_provider/gcp_test.exs modified +10

@@ -123,6 +123,16 @@ defmodule OpenAgents.Forge.RollingProvider.GcpTest do

123 123 
            }} = Gcp.status(hd(@nodes), context())
124 124 
  end
125 125
126 
  test "reports other reboot transport exits as unavailable" do
127 
    rpc = fn _node, RollingNodeProbe, :status, [_expected], _timeout ->
128 
      exit(:nodedown)
129 
    end
130
131 
    put_config(rpc)
132
133 
    assert {:ok, %{ready: 0, quorum: false}} = Gcp.capacity([hd(@nodes)], context())
134 
  end
135
126 136 
  test "reports only connected nodes in the configured fleet inventory" do
127 137 
    rpc = fn _node, _module, _function, _arguments, _timeout -> :ok end
128 138

This page updates live while a promote is in flight · changelog