test/openagents/forge/exit_rehearsal_runbook_test.exs

main at 58e6347eeb72 · 5 KB

defmodule OpenAgents.Forge.ExitRehearsalRunbookTest do
  @moduledoc """
  #189's third acceptance criterion. Rehearsal 3's step 3 told an operator to
  run `OpenAgents.Forge.Sync.rebuild/1` before that function existed, and no
  invariant could notice: invariants read compiled modules, and the procedure
  was a string in a Markdown file. This test reads the strings.

  Three shapes of reference in `docs/forge-exit-rehearsals.md` are checked
  against the compiled code:

  - every `bin/openagents rpc '…'` command in any document under `docs/`
    parses as Elixir, and every `OpenAgents` function it calls is exported at
    the called arity;
  - every qualified `OpenAgents.Module.function/arity` reference resolves;
  - every bare `function/arity` reference in code font is exported by at
    least one module the document names.

  A renamed or removed function now turns this test red instead of leaving a
  runbook step that reads as executable but is not.
  """

  use ExUnit.Case, async: true

  @runbook "docs/forge-exit-rehearsals.md"

  test "every rpc command in the documentation calls functions that exist" do
    commands =
      for path <- Path.wildcard("docs/**/*.md"),
          [command] <-
            Regex.scan(~r{bin/openagents rpc '([^']+)'}, File.read!(path),
              capture: :all_but_first
            ),
          do: {path, command}

    assert Enum.any?(commands, fn {path, _command} -> path == @runbook end),
           "#{@runbook} no longer contains rpc commands; retire this test"

    for {path, command} <- commands do
      calls =
        command
        |> Code.string_to_quoted!()
        |> expand_pipes()
        |> collect_openagents_calls()

      assert calls != [], "rpc command in #{path} calls no OpenAgents function: #{command}"

      for {module, function, arity} <- calls do
        assert Code.ensure_loaded?(module),
               "#{inspect(module)} is named in #{path} but does not exist"

        assert function_exported?(module, function, arity),
               "#{inspect(module)}.#{function}/#{arity} is named in #{path} " <>
                 "but is not exported at that arity"
      end
    end
  end

  test "every qualified function/arity reference in the runbook resolves" do
    references =
      Regex.scan(
        ~r/(OpenAgents(?:\.[A-Z][A-Za-z0-9_]*)+)\.([a-z_][a-z0-9_]*[?!]?)\/(\d+)/,
        runbook(),
        capture: :all_but_first
      )

    for [module_name, function, arity] <- references do
      module = Module.concat([module_name])
      function = String.to_atom(function)
      arity = String.to_integer(arity)

      assert Code.ensure_loaded?(module),
             "#{module_name} is named in #{@runbook} but does not exist"

      assert function_exported?(module, function, arity),
             "#{module_name}.#{function}/#{arity} is named in #{@runbook} but is not exported"
    end
  end

  test "every module the runbook names exists" do
    modules = named_modules()

    assert OpenAgents.Forge.Sync in modules

    for module <- modules do
      assert Code.ensure_loaded?(module),
             "#{inspect(module)} is named in #{@runbook} but does not exist"
    end
  end

  test "every bare function/arity reference is exported by a module the runbook names" do
    # `function_exported?/3` does not load a module, so load them first. The
    # modules-exist test owns the assertion that loading never fails.
    modules = Enum.filter(named_modules(), &Code.ensure_loaded?/1)

    references =
      Regex.scan(~r/`([a-z_][a-z0-9_]*[?!]?)\/(\d+)`/, runbook(), capture: :all_but_first)

    # The reference this test exists for. Its absence means the document
    # changed shape, not that the check passed.
    assert ["rebuild", "1"] in references

    for [function, arity] <- references do
      # Document content is repository-controlled, so minting the atom here
      # is bounded by what review admits into the runbook.
      function = String.to_atom(function)
      arity = String.to_integer(arity)

      assert Enum.any?(modules, &function_exported?(&1, function, arity)),
             "`#{function}/#{arity}` is named in #{@runbook} but no module " <>
               "the document names exports it"
    end
  end

  defp runbook, do: File.read!(@runbook)

  defp named_modules do
    Regex.scan(~r/OpenAgents(?:\.[A-Z][A-Za-z0-9_]*)+/, runbook())
    |> List.flatten()
    |> Enum.uniq()
    |> Enum.map(&Module.concat([&1]))
  end

  # `verify(…) |> IO.inspect()` calls `IO.inspect/1`, not `IO.inspect/0`, so
  # pipes are rewritten into ordinary calls before arities are read.
  defp expand_pipes(ast) do
    Macro.prewalk(ast, fn
      {:|>, _, [_, _]} = pipe ->
        [{first, _} | rest] = Macro.unpipe(pipe)

        Enum.reduce(rest, first, fn {call, position}, piped ->
          Macro.pipe(piped, call, position)
        end)

      node ->
        node
    end)
  end

  defp collect_openagents_calls(ast) do
    {_, calls} =
      Macro.prewalk(ast, [], fn
        {{:., _, [{:__aliases__, _, [:OpenAgents | _] = parts}, function]}, _, args} = node, acc
        when is_list(args) ->
          {node, [{Module.concat(parts), function, length(args)} | acc]}

        node, acc ->
          {node, acc}
      end)

    Enum.reverse(calls)
  end
end