lib/openagents/forge/mirror_watch.ex

main at 58e6347eeb72 · 5 KB

defmodule OpenAgents.Forge.MirrorWatch do
  @moduledoc """
  Mirror drift detection (#127, RELEASE-004: owned infra, no hosted CI).

  Best-effort mirroring means a failed `--mirror` push only logs — this
  watcher closes the gap. Every tick, for each repo with a configured
  mirror: compare the forge's `main` with the mirror's `main`
  (`git ls-remote`). On divergence it retries the mirror push immediately;
  if the mirror is still behind past the lag threshold, it records one
  `forge_mirror_lagging` degraded incident per lag episode (never a
  per-tick storm) and keeps retrying.

  Freshness is published in `state/0` for the public status page:
  `"off"` (no mirror configured), `"current"`, or `"lagging"` with minutes.
  The mirror URL may embed a credential and never appears in state, logs,
  incidents, or output.
  """

  use GenServer

  require Logger

  alias OpenAgents.Forge.Pushes
  alias OpenAgents.Forge.Repos

  @tick_ms 5 * 60 * 1000
  @lag_threshold_ms 15 * 60 * 1000
  @state_key {__MODULE__, :state}

  def start_link(opts), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)

  @impl true
  def init(_opts) do
    send(self(), :tick)
    {:ok, %{lagging_since: %{}, incident_reported: MapSet.new()}}
  end

  @doc "Mirror freshness for the status page: state per configured repo."
  def state do
    :persistent_term.get(@state_key, %{"state" => "off"})
  end

  @impl true
  def handle_info(:tick, state) do
    state = check_all(state)
    schedule()
    {:noreply, state}
  end

  def handle_info(_message, state), do: {:noreply, state}

  @doc "One check pass over every configured repo. Public for tests."
  def check_all(state, now_ms \\ System.monotonic_time(:millisecond)) do
    configured = Enum.filter(Repos.allowed_repos(), &Pushes.mirror_url/1)

    if configured == [] do
      :persistent_term.put(@state_key, %{"state" => "off"})
      state
    else
      Enum.reduce(configured, state, fn repo, acc -> check_repo(repo, acc, now_ms) end)
    end
  rescue
    error ->
      Logger.warning("forge_mirror_watch_failed code=#{OpenAgents.OperationalLog.code(error)}")
      state
  end

  defp check_repo(repo, state, now_ms) do
    case drift?(repo) do
      :current ->
        publish(repo, "current", nil)

        %{
          state
          | lagging_since: Map.delete(state.lagging_since, repo),
            incident_reported: MapSet.delete(state.incident_reported, repo)
        }

      :behind ->
        # Retry immediately — most lag is one missed best-effort push —
        # and re-check: a healed mirror is current, not lagging.
        _retry = Pushes.mirror_now(repo)

        if drift?(repo) == :current do
          publish(repo, "current", nil)

          %{
            state
            | lagging_since: Map.delete(state.lagging_since, repo),
              incident_reported: MapSet.delete(state.incident_reported, repo)
          }
        else
          since = Map.get(state.lagging_since, repo, now_ms)
          lag_ms = now_ms - since
          publish(repo, "lagging", div(lag_ms, 60_000))
          state = %{state | lagging_since: Map.put(state.lagging_since, repo, since)}

          if lag_ms >= @lag_threshold_ms and repo not in state.incident_reported do
            _incident =
              OpenAgents.Incidents.record(%{
                surface: "job",
                origin: "forge_mirror_watch",
                code: "forge_mirror_lagging",
                severity: "degraded",
                summary: "GitHub mirror for #{repo} behind the forge for #{div(lag_ms, 60_000)}m",
                context: %{"repo" => repo}
              })

            %{state | incident_reported: MapSet.put(state.incident_reported, repo)}
          else
            state
          end
        end

      :unknown ->
        # The mirror host is unreachable (GitHub down): the forge is
        # unaffected by design; keep state, retry next tick, report nothing
        # (the roadmap's acceptance line: pushes still succeed, warning
        # logged).
        state
    end
  end

  # main-ref comparison is the drift signal: the forge's cached main vs the
  # mirror's main. (ls-remote against the configured URL — output discarded
  # except the sha, so the credentialed URL never leaks.)
  defp drift?(repo) do
    with url when is_binary(url) <- Pushes.mirror_url(repo),
         {:ok, forge_main} <- forge_main(repo) do
      case remote_main(url) do
        {:ok, ^forge_main} -> :current
        {:ok, _other_sha} -> :behind
        # A reachable mirror with no main at all (fresh/empty) is behind;
        # only an unreachable host is unknown.
        {:error, :no_remote_main} -> :behind
        {:error, _reason} -> :unknown
      end
    else
      _missing -> :unknown
    end
  end

  defp forge_main(repo) do
    case repo |> Pushes.mirror_storage_key() |> Repos.refs() do
      %{"refs/heads/main" => sha} when is_binary(sha) -> {:ok, sha}
      _other -> {:error, :no_main}
    end
  end

  defp remote_main(url) do
    case System.cmd(
           "git",
           ["-c", "credential.helper=", "ls-remote", url, "refs/heads/main"],
           stderr_to_stdout: true,
           env: [{"GIT_TERMINAL_PROMPT", "0"}]
         ) do
      {output, 0} ->
        case String.split(output) do
          [sha | _rest] when byte_size(sha) == 40 -> {:ok, sha}
          _empty -> {:error, :no_remote_main}
        end

      {_output, _status} ->
        {:error, :remote_unreachable}
    end
  end

  defp publish(repo, mirror_state, lagging_minutes) do
    :persistent_term.put(@state_key, %{
      "state" => mirror_state,
      "repo" => repo,
      "lagging_minutes" => lagging_minutes
    })
  end

  defp schedule, do: Process.send_after(self(), :tick, @tick_ms)
end