infrastructure. Keep machine inventory, credentials, addresses, and provider

resource names outside the repository.

The provider reports its exact connected infrastructure inventory through

`members/0`. A hidden controller must not include itself or a temporary RPC

client in that inventory. The GCP provider intersects connected Erlang nodes

with its configured three-node instance map, so every membership check uses

the same bounded fleet identity.

For each node, the coordinator performs this sequence:

1. Verify the exact-SHA release receipt and exact initial member set.

   access, source SHA, and image digest.

7. Recheck exact fleet membership before selecting another node.

An Erlang distribution disconnect is an expected transient state while a VM

reboots. The GCP provider reports that node as unavailable so the coordinator

continues its bounded readiness polling. Other RPC errors fail the rollout

closed.

If a node does not rejoin, the coordinator asks the provider to restore the

last-known-good SHA and digest, waits for that node's full health, records the

recovery result, and aborts. It never replaces a second node while the first is

        }

  @callback remove_readiness(node(), context()) :: :ok | {:error, term()}

  @callback members() :: [node()]

  @callback restore_readiness(node(), context()) :: :ok | {:error, term()}

  @callback drain(node(), context()) :: {:ok, non_neg_integer()} | {:error, term()}

  @callback capacity([node()], context()) ::

  alias OpenAgents.Forge.RollingNodeProbe

  alias OpenAgents.Forge.RollingProvider.Gcp.Compute

  @impl true

  def members do

    with {:ok, config} <- config() do

      configured = config |> Keyword.fetch!(:instances) |> Map.keys() |> MapSet.new()

      config

      |> Keyword.get(:node_list, &connected_nodes/0)

      |> then(& &1.())

      |> Enum.filter(&MapSet.member?(configured, to_string(&1)))

      |> Enum.sort()

    else

      {:error, _reason} -> []

    end

  end

  @impl true

  def remove_readiness(node, _context) do

    with {:ok, config} <- config(),

  defp probe(config, node, expected_fleet_size) do

    case rpc(config, node, RollingNodeProbe, :status, [expected_fleet_size]) do

      %{member: true} = result -> {:ok, result}

      {:error, :noconnection} -> {:ok, unavailable_probe()}

      {:error, reason} -> {:error, reason}

      other -> {:error, {:invalid_node_probe, other}}

    end

  end

  defp unavailable_probe do

    %{

      member: false,

      ready: false,

      boot_converged: false,

      database_ready: false,

      sha: nil,

      image_digest: nil,

      ra_quorum: false

    }

  end

  defp rpc(config, node, module, function, arguments, call_timeout \\ nil) do

    rpc = Keyword.get(config, :rpc, &:erpc.call/5)

    end

  catch

    :exit, reason -> {:error, {:rpc_exit, reason}}

    :error, {:erpc, :noconnection} -> {:error, :noconnection}

    :error, reason -> {:error, {:rpc_error, reason}}

  end

  defp instance(config, node) do

  end

  defp driver(config), do: Keyword.get(config, :driver, Compute)

  defp connected_nodes, do: Enum.uniq(Node.list() ++ Node.list(:hidden))

  defp timeout(config), do: Keyword.get(config, :rpc_timeout_ms, 5_000)

  defp compute_timeout(config), do: Keyword.get(config, :compute_timeout_ms, 300_000)

    with :ok <- validate_request(request),

         {:ok, _receipt} <- gate_verify(request.sha, opts),

         {:ok, provider} <- provider(opts),

         :ok <- initial_membership(request, opts) do

         :ok <- initial_membership(request, provider, opts) do

      replace_nodes(request.expected_nodes, request, provider, opts, %{})

    end

  end

  defp replace_drained_node(node, request, provider, context, opts) do

    with :ok <- provider.replace(node, request.image_digest, context),

         :ok <- wait_for_target(provider, node, request, context, opts),

         :ok <- exact_membership(request, opts) do

         :ok <- exact_membership(request, provider, opts) do

      {:ok, "ready"}

    else

      {:error, reason} ->

    end

  end

  defp initial_membership(request, opts), do: exact_membership(request, opts)

  defp initial_membership(request, provider, opts),

    do: exact_membership(request, provider, opts)

  defp exact_membership(request, opts) do

    if members(opts) == request.expected_nodes,

  defp exact_membership(request, provider, opts) do

    if members(provider, opts) == request.expected_nodes,

      do: :ok,

      else: {:error, :fleet_membership_mismatch}

  end

  defp members(opts) do

    Keyword.get(opts, :members, &OpenAgents.Cluster.members/0).()

  defp members(provider, opts) do

    Keyword.get(opts, :members, &provider.members/0).()

    |> Enum.sort()

  end

    assert {:ok, %{ready: 2, quorum: false}} = Gcp.capacity(tl(@nodes), context())

  end

  test "reports a rebooting node as unavailable when Erlang distribution disconnects" do

    rpc = fn _node, RollingNodeProbe, :status, [_expected], _timeout ->

      :erlang.error({:erpc, :noconnection})

    end

    put_config(rpc)

    assert {:ok,

            %{

              member: false,

              ready: false,

              boot_converged: false,

              database_ready: false,

              sha: nil,

              image_digest: nil

            }} = Gcp.status(hd(@nodes), context())

  end

  test "reports only connected nodes in the configured fleet inventory" do

    rpc = fn _node, _module, _function, _arguments, _timeout -> :ok end

    put_config(rpc,

      node_list: fn -> [hd(@nodes), :unknown@fleet, Enum.at(@nodes, 2)] end

    )

    assert Gcp.members() == [hd(@nodes), Enum.at(@nodes, 2)]

  end

  defp context do

    %{

      sha: @sha,

           ]

  end

  test "uses the provider's exact infrastructure inventory by default" do

    start_provider(nil)

    assert {:ok, %{status: "live"}} =

             RollingReplacement.run(request(),

               provider: RollingProvider,

               gate_verifier: fn @sha -> {:ok, %{}} end,

               wait_attempts: 1,

               wait_interval_ms: 0

             )

  end

  test "restores the prior image and aborts before another replacement when rejoin fails" do

    start_provider(:second@local)

         capacity: capacity,

         events: [],

         fail_node: fail_node,

         members: @nodes,

         previous_sha: @previous_sha,

         rolled_back: MapSet.new(),

         digests: Map.new(@nodes, &{&1, @previous})

  def events, do: Agent.get(__MODULE__, &Enum.reverse(&1.events))

  @impl true

  def members, do: Agent.get(__MODULE__, & &1.members)

  @impl true

  def remove_readiness(node, _context) do

    event({:remove_readiness, node})