test/openagents/cluster_test.exs

main at 58e6347eeb72 · 4 KB

defmodule OpenAgents.ClusterTest do
  # Not async: the last test manipulates node-global distribution state.
  use ExUnit.Case, async: false

  alias OpenAgents.Cluster

  test "single-node health report is well-formed" do
    report = Cluster.local_report()

    assert report["schema"] == "openagents.cluster_health.v1"
    assert report["node"] == to_string(Node.self())
    assert report["revision"] == OpenAgents.BuildInfo.revision()
    assert report["image_digest"] == OpenAgents.BuildInfo.image_digest()
    assert report["admission_ready"] == true
    assert report["version"] == to_string(Application.spec(:openagents, :vsn) || "unknown")
    assert report["live"] == true
    assert report["ready"] == true
    assert is_integer(report["uptime_ms"])
  end

  test "admission fences local readiness" do
    on_exit(&OpenAgents.Cluster.Admission.restore/0)

    assert :ok = OpenAgents.Cluster.Admission.remove()
    refute Cluster.local_report()["ready"]
    refute Cluster.local_report()["admission_ready"]

    assert :ok = OpenAgents.Cluster.Admission.restore()
    assert Cluster.local_report()["admission_ready"]
  end

  test "quorum and snapshot in single-node mode" do
    assert Cluster.size() == 1
    assert Cluster.members() == [Node.self()]
    assert Cluster.distributed?() == false
    assert Cluster.quorum?(1)
    refute Cluster.quorum?(3)

    snapshot = Cluster.snapshot()
    assert snapshot["schema"] == "openagents.cluster_state.v1"
    assert snapshot["size"] == 1
  end

  # Real peer nodes: gated to the `mix test --only cluster` stage, like every
  # other distribution test here. Left in the default suite it both needs epmd
  # and inherits whatever node-global state a previous cluster module left
  # behind — which is exactly how it came to fail on `net_kernel already started`.
  @tag :cluster
  test "three local nodes form a cluster, report a consistent revision, and detect a missing node" do
    start_distribution!()
    peers = start_peers(3)

    try do
      for {_pid, node} <- peers, do: Node.connect(node)

      assert wait_for(fn -> length(Node.list()) == 3 end)
      assert length(Cluster.members()) == 4

      report = Cluster.health_report()
      assert report["consistent"] == true
      assert map_size(report["peers"]) == 3
      assert report["missing"] == []

      # Stop one peer and verify the next report shrinks and marks it missing.
      [{down_pid, down_node} | _] = peers
      :peer.stop(down_pid)

      assert wait_for(fn -> length(Node.list()) == 2 end)

      report2 = Cluster.health_report()
      assert report2["consistent"] == true
      assert map_size(report2["peers"]) == 2
      assert report2["missing"] == []
      refute to_string(down_node) in Enum.map(Cluster.members(), &to_string/1)
    after
      # Stop this test's peers, but leave net_kernel up. Stopping it stranded
      # every cluster module scheduled after this one: their
      # `ensure_distributed/0` then saw an undistributed node whose net_kernel
      # would no longer restart, and flunked "distribution unavailable".
      for {pid, _node} <- peers, do: safe_stop_peer(pid)
      assert wait_for(fn -> Node.list() == [] end)
    end
  end

  # Idempotent: an already-distributed node is a legitimate state in the cluster
  # stage (another module got here first), not a reason to fail.
  defp start_distribution! do
    suffix = :erlang.unique_integer([:positive])
    name = :erlang.list_to_atom(~c"openagents_test_#{suffix}@127.0.0.1")

    case :net_kernel.start([name, :longnames]) do
      :ok -> :ok
      true -> :ok
      {:ok, _pid} -> :ok
      {:error, {:already_started, _pid}} -> :ok
      other -> raise "Failed to start net_kernel: #{inspect(other)}"
    end

    _ = Node.set_cookie(:openagents_test_cookie)
    :ok
  end

  defp start_peers(n) do
    ebins = [
      Application.app_dir(:openagents, "ebin"),
      Application.app_dir(:elixir, "ebin")
    ]

    pa_args =
      Enum.flat_map(ebins, fn path ->
        [~c"-pa", to_charlist(path)]
      end)

    cookie = :openagents_test_cookie

    Enum.map(1..n, fn i ->
      suffix = :erlang.unique_integer([:positive])
      short_name = :erlang.list_to_atom(~c"openagents_peer#{i}_#{suffix}")

      {:ok, pid, node} =
        :peer.start_link(%{
          name: short_name,
          host: ~c"127.0.0.1",
          user: %{},
          shutdown: OpenAgents.Test.RemoteCover.shutdown(),
          args:
            [
              ~c"-setcookie",
              to_charlist(Atom.to_string(cookie))
            ] ++ pa_args
        })

      # Make sure the OpenAgents app is loaded on the peer so version and
      # revision are available for health reports.
      :ok = :erpc.call(node, Application, :load, [:openagents])
      {pid, node}
    end)
  end

  defp safe_stop_peer(pid) do
    :peer.stop(pid)
  catch
    :exit, _ -> :ok
  end

  defp wait_for(fun, retries \\ 50) do
    if fun.() do
      true
    else
      Process.sleep(20)
      if retries > 0, do: wait_for(fun, retries - 1), else: false
    end
  end
end