test/openagents/cluster/drain_test.exs

58e6347eeb72 · 3 KB

defmodule OpenAgents.Cluster.DrainTest do
  @moduledoc """
  Proves the rolling-deploy drain: a node leaves the Raft cluster cleanly and
  the surviving members recompute a smaller quorum and keep committing — so a
  deploy that replaces one node at a time never drops the cluster.

  Tagged `:cluster`; run with `mix test --include cluster` (needs epmd).
  """
  use ExUnit.Case, async: false
  @moduletag :cluster

  alias OpenAgents.Cluster.{Drain, Ra}

  setup do
    case ensure_distributed() do
      :ok -> :ok
      :unavailable -> flunk("distribution unavailable — start epmd (`epmd -daemon`)")
    end
  end

  test "a drained node leaves Raft; survivors keep quorum and keep committing" do
    cookie = :erlang.get_cookie()
    {peer1, node1} = start_peer(:openagents_drain1, cookie)
    {peer2, node2} = start_peer(:openagents_drain2, cookie)
    on_exit(fn -> for p <- [peer1, peer2], do: safe_stop(p) end)

    Ra.start_in(data_dir(node()))
    :ok = :erpc.call(node1, Ra, :start_in, [data_dir(node1)])
    :ok = :erpc.call(node2, Ra, :start_in, [data_dir(node2)])

    assert {:ok, started} = Ra.start_cluster([node(), node1, node2])
    assert length(started) == 3

    id = "drain-#{System.unique_integer([:positive])}"
    assert {:ok, 1} = Ra.claim(id, :job)

    # Drain node2 — it leaves the Raft cluster cleanly.
    assert :ok = :erpc.call(node2, Drain, :leave_ra, [])

    assert eventually(fn -> length(Ra.members()) == 2 end),
           "drained node was not removed from the Raft cluster"

    refute node2 in Ra.members()

    # The two remaining members still form a quorum and keep committing.
    assert {:ok, 2} = Ra.claim(id, :job)
    assert {:ok, %{generation: 2}} = Ra.lookup(id)
  end

  # ── helpers ──────────────────────────────────────────────────────────────

  defp data_dir(node) do
    base = Path.join(System.tmp_dir!(), "openagents_drain_test")
    short = node |> Atom.to_string() |> String.replace(~r/[^a-zA-Z0-9]/, "_")
    Path.join(base, "#{short}_#{System.unique_integer([:positive])}")
  end

  defp start_peer(name, cookie) do
    {:ok, peer, node} =
      :peer.start_link(%{
        name: unique_peer_name(name),
        host: ~c"127.0.0.1",
        shutdown: OpenAgents.Test.RemoteCover.shutdown(),
        args: [~c"-setcookie", Atom.to_charlist(cookie)]
      })

    :ok = :erpc.call(node, :code, :add_paths, [:code.get_path()])
    {:ok, _} = :erpc.call(node, Application, :ensure_all_started, [:ra])
    {peer, node}
  end

  defp safe_stop(peer) do
    _ = :peer.stop(peer)
    :ok
  rescue
    _ -> :ok
  catch
    _, _ -> :ok
  end

  defp ensure_distributed do
    cond do
      Node.self() != :nonode@nohost ->
        :ok

      # A fixed node name wedges this whole stage on any machine where an
      # earlier run left that name registered with epmd: net_kernel then
      # refuses to start and every distribution test flunks "unavailable".
      # Unique per run, and OpenAgents-named now that this is not Sarah's BEAM.
      match?({:ok, _}, :net_kernel.start([unique_test_node(), :longnames])) ->
        :erlang.set_cookie(Node.self(), :openagents_cluster_test_cookie)
        :ok

      true ->
        :unavailable
    end
  end

  defp eventually(fun, attempts \\ 100) do
    cond do
      fun.() -> true
      attempts <= 0 -> false
      true -> Process.sleep(100) && eventually(fun, attempts - 1)
    end
  end

  defp unique_test_node do
    :erlang.list_to_atom(~c"openagents_test_#{:erlang.unique_integer([:positive])}@127.0.0.1")
  end

  # Peer node names register with epmd too. A fixed name that a killed peer
  # left behind makes the next run's :peer.start_link fail, so the gate is
  # green once and wedged thereafter. Suffix every peer uniquely.
  defp unique_peer_name(base) do
    :erlang.list_to_atom(~c"#{base}_#{:erlang.unique_integer([:positive])}")
  end
end