OpenAgents.Forge.BootConverge,

        {DNSCluster, query: Application.get_env(:openagents, :dns_cluster_query) || :ignore},

        {Phoenix.PubSub, name: OpenAgents.PubSub},

        OpenAgents.Inference.Health,

        OpenAgents.RuntimeSupervisor,

        OpenAgentsWeb.BoxRateLimiter,

        {Registry, keys: :unique, name: OpenAgents.BoxRunRegistry},

defmodule OpenAgents.Inference.Health do

  @moduledoc """

  What each model lane has actually done lately, so the catalog can stop

  claiming a lane is available while every call to it fails.

  The catalog used to answer one question — is a credential configured — and

  publish the answer as `availability`. That is a claim about wiring, not about

  whether the lane answers, and the difference is not academic: a whole

  delegation fleet was blocked by a default model whose credential was present

  and whose every call failed, while `GET /api/v1/models` reported it

  `available` (#238, #199).

  This records the outcome of real calls and lets the catalog say what it

  knows:

  - `:unknown` — nothing has been tried since boot. Not a promise either way.

  - `:healthy` — the last call succeeded.

  - `:degraded` — `@degraded_after` consecutive failures with no success since.

  Deliberately not a circuit breaker. Nothing here refuses a call or routes

  around a lane; a caller that wants the failing lane still gets it. The only

  claim being fixed is the published one, because a client that reads the

  catalog and picks a dead lane was misled by us rather than by the provider.

  State is per node and lives in ETS. It is lost on restart, which is correct:

  health is a statement about now, and a node that has just booted has not

  tried anything yet — that is exactly `:unknown`.

  """

  use GenServer

  @table :openagents_inference_health

  @degraded_after 3

  @type status :: :unknown | :healthy | :degraded

  @doc "How many consecutive failures make a lane degraded."

  @spec degraded_after() :: pos_integer()

  def degraded_after, do: @degraded_after

  def start_link(options) do

    GenServer.start_link(__MODULE__, options, name: __MODULE__)

  end

  @impl true

  def init(_options) do

    :ets.new(@table, [:named_table, :public, :set, read_concurrency: true])

    {:ok, %{}}

  end

  @doc """

  Record that a call to `model_id` answered.

  Success clears the failure run outright rather than decrementing it: a lane

  that answers is working now, and holding a grudge for earlier failures would

  keep publishing `degraded` about a lane that recovered.

  """

  @spec record_success(String.t()) :: :ok

  def record_success(model_id) when is_binary(model_id) do

    if table?(), do: :ets.insert(@table, {model_id, 0, :healthy})

    :ok

  end

  @doc """

  Record that a call to `model_id` failed.

  `status` is the upstream HTTP status when the failure carried one

  (`OpenAgents.OperationalLog.status/1`) and `nil` otherwise. It is kept so an

  operator reading health sees *why*, and never inferred: a failure with no

  status reports none.

  """

  @spec record_failure(String.t(), pos_integer() | nil) :: :ok

  def record_failure(model_id, status \\ nil) when is_binary(model_id) do

    if table?() do

      failures =

        case :ets.lookup(@table, model_id) do

          [{^model_id, count, _}] when is_integer(count) -> count + 1

          _ -> 1

        end

      :ets.insert(@table, {model_id, failures, {:failed, status}})

    end

    :ok

  end

  @doc "The status of one lane, and the last upstream status when it failed."

  @spec status(String.t()) :: {status(), pos_integer() | nil}

  def status(model_id) when is_binary(model_id) do

    case table?() && :ets.lookup(@table, model_id) do

      [{^model_id, 0, :healthy}] ->

        {:healthy, nil}

      [{^model_id, failures, {:failed, upstream}}] when failures >= @degraded_after ->

        {:degraded, upstream}

      [{^model_id, _failures, {:failed, upstream}}] ->

        # Below the threshold a lane is not yet degraded, but the last failure

        # is still worth surfacing to whoever asks.

        {:healthy, upstream}

      _ ->

        {:unknown, nil}

    end

  end

  @doc "Forget everything recorded. For tests and for an operator resetting a lane."

  @spec reset() :: :ok

  def reset do

    if table?(), do: :ets.delete_all_objects(@table)

    :ok

  end

  defp table?, do: :ets.whereis(@table) != :undefined

end

defmodule OpenAgents.Inference.Models do

  alias OpenAgents.Inference.Health

  @moduledoc """

  The typed model catalog: every model this deployment serves, and the

  provider lane that serves each.

  test adapters need no credential, and an adapter that cannot say is refused

  at call time by its own `missing_api_key` rather than guessed at here.

  """

  @doc """

  What a client should believe about a lane, as one word.

  `unavailable` means the deployment cannot call it at all — no credential.

  `degraded` means it is configured and its recent calls have failed, which is

  the case the old two-word answer could not express: a lane whose wiring is

  right and whose every call fails used to publish `available` and mislead the

  caller that trusted it (#238).

  A lane nothing has called since boot is `available`, not `degraded`. Silence

  is not evidence of failure, and refusing to offer an untried lane would make

  every restart look like an outage.

  """

  @spec availability(t()) :: String.t()

  def availability(%{id: id} = model) do

    cond do

      not available?(model) -> "unavailable"

      match?({:degraded, _}, Health.status(id)) -> "degraded"

      true -> "available"

    end

  end

  @spec available?(t()) :: boolean()

  def available?(%{adapter: adapter}) do

    if Code.ensure_loaded?(adapter) and function_exported?(adapter, :configured?, 0) do

        "provider" => Atom.to_string(model.provider),

        "context_window" => model.context_window,

        "max_output" => model.max_output,

        "availability" => if(available?(model), do: "available", else: "unavailable"),

        "availability" => availability(model),

        "default" => model.id == default_id

      }

    end)

        # answered, not what it assumed (PROVIDER-002). Because a mismatched

        # request was refused above, requested and effective are the same

        # name on every 200.

        OpenAgents.Inference.Health.record_success(model.id)

        conn

        |> put_resp_content_type("text/event-stream")

        |> put_resp_header("cache-control", "no-store")

        if usage != %{}, do: meter(grant, usage)

        class = OpenAgents.OperationalLog.code(reason)

        status = OpenAgents.OperationalLog.status(reason)

        # What the catalog publishes about this lane follows from what it

        # actually did, not only from whether a credential is configured.

        OpenAgents.Inference.Health.record_failure(model.id, status)

        Logger.warning(

          "inference_proxy_failed code=#{class}" <>

            if(status == nil, do: "", else: " upstream_status=#{status}")

defmodule OpenAgents.Inference.HealthTest do

  use ExUnit.Case, async: false

  alias OpenAgents.Inference.Health

  alias OpenAgents.Inference.Models

  setup do

    Health.reset()

    on_exit(&Health.reset/0)

    :ok

  end

  describe "what a lane has lately done" do

    test "a lane nothing has called is unknown, which is not a verdict either way" do

      assert Health.status("never-called") == {:unknown, nil}

    end

    test "a success makes a lane healthy" do

      Health.record_success("gpt-5.6-luna")

      assert Health.status("gpt-5.6-luna") == {:healthy, nil}

    end

    test "one failure is not yet degraded, but the upstream status is kept" do

      Health.record_failure("gemini-3.7-flash", 503)

      assert Health.status("gemini-3.7-flash") == {:healthy, 503}

    end

    test "consecutive failures degrade the lane and carry the last status" do

      for _ <- 1..Health.degraded_after(), do: Health.record_failure("gemini-3.7-flash", 429)

      assert Health.status("gemini-3.7-flash") == {:degraded, 429}

    end

    test "a success clears the failure run rather than decrementing it" do

      for _ <- 1..Health.degraded_after(), do: Health.record_failure("gemini-3.7-flash", 500)

      assert {:degraded, _} = Health.status("gemini-3.7-flash")

      Health.record_success("gemini-3.7-flash")

      assert Health.status("gemini-3.7-flash") == {:healthy, nil}

    end

    test "a failure with no upstream status reports none rather than inventing one" do

      for _ <- 1..Health.degraded_after(), do: Health.record_failure("ox-alpha", nil)

      assert Health.status("ox-alpha") == {:degraded, nil}

    end

    test "lanes are tracked apart" do

      for _ <- 1..Health.degraded_after(), do: Health.record_failure("gemini-3.7-flash", 502)

      Health.record_success("gpt-5.6-luna")

      assert {:degraded, 502} = Health.status("gemini-3.7-flash")

      assert {:healthy, nil} = Health.status("gpt-5.6-luna")

    end

  end

  describe "what the catalog publishes" do

    test "an untried lane still reads available, because silence is not failure" do

      entry = Enum.find(Models.catalog(), &(&1["id"] == "gemini-3.7-flash"))

      assert entry["availability"] == "available"

    end

    test "a lane whose calls keep failing stops claiming it is available" do

      for _ <- 1..Health.degraded_after(), do: Health.record_failure("gemini-3.7-flash", 503)

      entry = Enum.find(Models.catalog(), &(&1["id"] == "gemini-3.7-flash"))

      assert entry["availability"] == "degraded"

      # The lanes that are answering are unaffected, so a client can still pick

      # one that works — the whole point of publishing this.

      others = Enum.reject(Models.catalog(), &(&1["id"] == "gemini-3.7-flash"))

      assert Enum.all?(others, &(&1["availability"] in ["available", "unavailable"]))

    end

    test "a recovered lane goes back to available" do

      for _ <- 1..Health.degraded_after(), do: Health.record_failure("gemini-3.7-flash", 503)

      Health.record_success("gemini-3.7-flash")

      entry = Enum.find(Models.catalog(), &(&1["id"] == "gemini-3.7-flash"))

      assert entry["availability"] == "available"

    end

  end

end