test/openagents/forge/rolling_replacement_test.exs

58e6347eeb72 · 5 KB

defmodule OpenAgents.Forge.RollingReplacementTest do
  use ExUnit.Case, async: false

  alias OpenAgents.Forge.RollingReplacement
  alias OpenAgents.Test.RollingAuthority
  alias OpenAgents.Test.RollingProvider

  @sha String.duplicate("a", 40)
  @previous_sha String.duplicate("d", 40)
  @target "sha256:" <> String.duplicate("b", 64)
  @previous "sha256:" <> String.duplicate("c", 64)
  @nodes [:first@local, :second@local, :third@local]
  @target_id "6ba7b810-9dad-11d1-80b4-00c04fd430c8"

  test "replaces one node at a time only after readiness returns" do
    start_provider(nil)

    assert {:ok, %{status: "live"}} = run()

    replace_events =
      RollingProvider.events()
      |> Enum.filter(&match?({:replace, _node, _digest}, &1))

    assert replace_events == [
             {:replace, :first@local, @target},
             {:replace, :second@local, @target},
             {:replace, :third@local, @target}
           ]
  end

  test "uses the provider's exact infrastructure inventory by default" do
    start_provider(nil)

    assert {:ok, %{status: "live"}} =
             RollingReplacement.run(request(),
               provider: RollingProvider,
               authority: RollingAuthority,
               gate_verifier: fn @sha -> {:ok, %{}} end,
               wait_attempts: 1,
               wait_interval_ms: 0
             )
  end

  test "restores the prior image and aborts before another replacement when rejoin fails" do
    start_provider(:second@local)

    assert {:error, %{status: "failed", recovery: "last_known_good_restored"}} = run()

    events = RollingProvider.events()
    assert {:rollback, :second@local, @previous} in events
    refute {:replace, :third@local, @target} in events
  end

  test "refuses deployment without an exact gate receipt" do
    start_provider(nil)

    assert {:error, :missing_gate_receipt} =
             RollingReplacement.run(request(),
               provider: RollingProvider,
               authority: RollingAuthority,
               members: fn -> @nodes end,
               gate_verifier: fn @sha -> {:error, :missing_gate_receipt} end
             )

    assert RollingProvider.events() == []
  end

  test "restores readiness without replacing an image when drained capacity is unsafe" do
    start_provider(nil, {:ok, %{ready: 1, quorum: true}})

    assert {:error, %{status: "failed", recovery: "readiness_restored"}} = run()

    events = RollingProvider.events()
    assert {:restore_readiness, :first@local} in events
    refute Enum.any?(events, &match?({:replace, _node, _digest}, &1))
  end

  test "publishes the authorized rolling identity before the first node is replaced" do
    start_provider(nil)

    assert {:ok, %{status: "live", target_id: @target_id}} = run()

    assert {@target_id,
            %{
              sha: @sha,
              image_digest: @target,
              previous_sha: @previous_sha,
              previous_image_digest: @previous,
              expected_nodes: ["first@local", "second@local", "third@local"]
            }} = RollingAuthority.authorized()

    # The authorization is durable before the fleet is touched at all, so the
    # first replacement node boots into an already-authorized identity.
    assert RollingAuthority.fleet_events_at_authorization() == []
    assert hd(RollingAuthority.calls()) == {:authorize, @target_id}

    assert RollingAuthority.observed() == %{
             "first@local" => %{sha: @sha, image_digest: @target},
             "second@local" => %{sha: @sha, image_digest: @target},
             "third@local" => %{sha: @sha, image_digest: @target}
           }
  end

  test "replaces nothing when the target refuses the rolling authorization" do
    start_supervised!({RollingAuthority, %{refuse: :rolling_authority_conflict}})

    start_supervised!(
      {RollingProvider,
       %{
         capacity: nil,
         events: [],
         fail_node: nil,
         members: @nodes,
         previous_sha: @previous_sha,
         rolled_back: MapSet.new(),
         digests: Map.new(@nodes, &{&1, @previous})
       }}
    )

    assert {:error, {:rolling_authority_refused, :rolling_authority_conflict}} = run()
    assert RollingProvider.events() == []
  end

  test "records the previous identity for a node it rolled back" do
    start_provider(:second@local)

    assert {:error, %{status: "failed", recovery: "last_known_good_restored"}} = run()

    assert RollingAuthority.observed() == %{
             "first@local" => %{sha: @sha, image_digest: @target},
             "second@local" => %{sha: @previous_sha, image_digest: @previous}
           }
  end

  test "refuses a request that names no Forge target" do
    start_provider(nil)

    assert {:error, :invalid_target_id} =
             RollingReplacement.run(Map.delete(request(), :target_id),
               provider: RollingProvider,
               authority: RollingAuthority,
               members: fn -> @nodes end,
               gate_verifier: fn @sha -> {:ok, %{}} end
             )

    assert RollingProvider.events() == []
  end

  defp run do
    RollingReplacement.run(request(),
      provider: RollingProvider,
      authority: RollingAuthority,
      members: fn -> @nodes end,
      gate_verifier: fn @sha -> {:ok, %{}} end,
      wait_attempts: 1,
      wait_interval_ms: 0
    )
  end

  defp request do
    %{
      target_id: @target_id,
      sha: @sha,
      previous_sha: @previous_sha,
      image_digest: @target,
      previous_image_digest: @previous,
      expected_nodes: @nodes,
      expected_fleet_size: 3,
      minimum_ready: 2
    }
  end

  defp start_provider(fail_node, capacity \\ nil) do
    start_supervised!({RollingAuthority, %{}})

    start_supervised!(
      {RollingProvider,
       %{
         capacity: capacity,
         events: [],
         fail_node: fail_node,
         members: @nodes,
         previous_sha: @previous_sha,
         rolled_back: MapSet.new(),
         digests: Map.new(@nodes, &{&1, @previous})
       }}
    )
  end
end