Let the server pick a lane when nobody named one

c96a1ba2c34f · AtlantisPleb · · parent 33ca3212429d

Let the server pick a lane when nobody named one

The default lane was the first entry in a config list, so a caller
that named no model got a fixed lane whether or not it answered. That
is how a dead default blocked every session that inherited it.

`Models.select/0` prefers a configured lane that is not degraded, in
catalog order, and falls back to the catalog default when every
configured lane is degraded — a degraded lane may still work, and
refusing would turn a soft signal into an outage.

The boundary is the important part, and PROVIDER-002 now states it.
This is not substitution: it applies only where *nothing* named a
model, neither the mint nor the call. A model named in the body, or a
grant minted for something other than the default, routes exactly as
it always did — a caller that named a lane still gets that lane or an
error, degraded or not, and there is a test pinning that. The
effective model is attributed on the response as always, so the answer
still says what produced it.

Credit-balance routing is not here: balances are not modelled in this
repository yet, and guessing at them would be worse than the fixed
default this replaces.

Built by a Devin child through the openagents coder's delegate tool;
the invariant amendment and the named-model boundary test were added
in review, and the full suite is green at 4,482.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GoYpb8FEmdxVErsv7ABCYi
Co-Authored-By
Claude Fable 5 <noreply@anthropic.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

pushed
by user · WAL seq 359 · 2026-08-25T10:38:48.134039Z

Changed files

  • modified INVARIANTS.md
  • modified lib/openagents/inference/models.ex
  • modified lib/openagents_web/controllers/inference_proxy_controller.ex
  • added test/openagents/inference/models_select_test.exs
  • modified test/openagents_web/controllers/inference_proxy_controller_test.exs

Diff

5 files changed, +218 -14

INVARIANTS.md modified +13 -2

@@ -1224,8 +1224,19 @@ Concretely:

1224 1224
  `model_mismatch` naming both, rather than silently answered by the grant's.
1225 1225
- Every successful proxy response attributes the effective model — the
1226 1226
  `x-openagents-model` header and each SSE chunk's `model` field — so a
1227
  client renders what answered rather than what it assumed. Because a
1228
  mismatch is refused, requested and effective agree on every `200`.
1227
  client renders what answered rather than what it assumed. A named model is
1228
  never substituted: because a mismatch is refused, a caller that named one
1229
  gets that one or an error.
1230
- Amended 2026-08-25 (#199): where **nothing named a model** — neither the
1231
  mint nor the call — the server selects, preferring a configured lane that is
1232
  not `degraded` in catalog order and falling back to the catalog default when
1233
  every configured lane is degraded. This is not substitution: there was no
1234
  request to substitute for, and a caller that said "whatever you serve" is
1235
  better served by a lane that answers than by a fixed one that does not. It
1236
  is bounded to exactly that case — a model named in the body, or a grant
1237
  minted for a model other than the default, is routed as it always was — and
1238
  the effective model is attributed on the response as always, so the answer
1239
  still says what produced it.
1229 1240
1230 1241
The chat lane keeps the same law through `OpenAgents.Chat.Backends`: an
1231 1242
unsupported `model` on `POST /api/v1/chat/turns` is a typed `422`, and
lib/openagents/inference/models.ex modified +34 -7

@@ -74,6 +74,29 @@ defmodule OpenAgents.Inference.Models do

74 74
  @spec default_id() :: String.t()
75 75
  def default_id, do: default().id
76 76
77
  @doc """
78
  The model the server selects for a caller that names none, by policy.
79
80
  A lane that is configured and not degraded is preferred, in catalog order.
81
  If no such lane exists because every configured lane is degraded, or
82
  because no lane is configured, the catalog default is returned. The proxy
83
  still uses `Models.available?/1` to refuse the call when the default is
84
  unavailable, so a degraded default is used but an unavailable one is not.
85
  """
86
  @spec select() :: t()
87
  def select do
88
    models = all()
89
90
    case Enum.find(models, &healthy?/1) do
91
      nil -> default()
92
      model -> model
93
    end
94
  end
95
96
  @doc "The id of the model `select/0` returns."
97
  @spec select_id() :: String.t()
98
  def select_id, do: select().id
99
77 100
  @doc """
78 101
  The model with this id, or `:error`.
79 102

@@ -93,13 +116,6 @@ defmodule OpenAgents.Inference.Models do

93 116
94 117
  def fetch(_id), do: :error
95 118
96
  @doc """
97
  Whether this model's adapter reports its credential configured.
98
99
  An adapter that does not export `configured?/0` is taken as configured: the
100
  test adapters need no credential, and an adapter that cannot say is refused
101
  at call time by its own `missing_api_key` rather than guessed at here.
102
  """
103 119
  @doc """
104 120
  What a client should believe about a lane, as one word.
105 121

@@ -122,6 +138,13 @@ defmodule OpenAgents.Inference.Models do

122 138
    end
123 139
  end
124 140
141
  @doc """
142
  Whether this model's adapter reports its credential configured.
143
144
  An adapter that does not export `configured?/0` is taken as configured: the
145
  test adapters need no credential, and an adapter that cannot say is refused
146
  at call time by its own `missing_api_key` rather than guessed at here.
147
  """
125 148
  @spec available?(t()) :: boolean()
126 149
  def available?(%{adapter: adapter}) do
127 150
    if Code.ensure_loaded?(adapter) and function_exported?(adapter, :configured?, 0) do

@@ -131,6 +154,10 @@ defmodule OpenAgents.Inference.Models do

131 154
    end
132 155
  end
133 156
157
  defp healthy?(%{id: id} = model) do
158
    available?(model) and not match?({:degraded, _}, Health.status(id))
159
  end
160
134 161
  @doc """
135 162
  The public projection of the catalog, for `GET /api/v1/models`.
136 163
lib/openagents_web/controllers/inference_proxy_controller.ex modified +17 -5

@@ -36,7 +36,7 @@ defmodule OpenAgentsWeb.InferenceProxyController do

36 36
    # proxy never re-reads or re-parses it.
37 37
    with {:ok, token} <- bearer(conn),
38 38
         {:ok, grant} <- resolve(token),
39
         {:ok, model} <- route(grant),
39
         {:ok, model} <- route(grant, conn.body_params),
40 40
         :ok <- serving(model),
41 41
         :ok <- requested_model(model, conn.body_params),
42 42
         {:ok, request} <- build_request(model, conn.body_params) do

@@ -52,10 +52,21 @@ defmodule OpenAgentsWeb.InferenceProxyController do

52 52
  # with. A grant minted before the model was routable — or one whose model has
53 53
  # since been withdrawn — is refused here rather than sent to a provider that
54 54
  # does not serve it.
55
  defp route(grant) do
56
    case Models.fetch(grant.model_id) do
57
      {:ok, model} -> {:ok, model}
58
      :error -> {:error, :model_unavailable}
55
  defp route(grant, body) do
56
    if no_model_in_body?(body) and grant.model_id == Models.default_id() do
57
      {:ok, Models.select()}
58
    else
59
      case Models.fetch(grant.model_id) do
60
        {:ok, model} -> {:ok, model}
61
        :error -> {:error, :model_unavailable}
62
      end
63
    end
64
  end
65
66
  defp no_model_in_body?(body) do
67
    case Map.get(body, "model") do
68
      absent when absent in [nil, ""] -> true
69
      _ -> false
59 70
    end
60 71
  end
61 72

@@ -210,6 +221,7 @@ defmodule OpenAgentsWeb.InferenceProxyController do

210 221
        # What the catalog publishes about this lane follows from what it
211 222
        # actually did, not only from whether a credential is configured.
212 223
        OpenAgents.Inference.Health.record_failure(model.id, status)
224
213 225
        Logger.warning(
214 226
          "inference_proxy_failed code=#{class}" <>
215 227
            if(status == nil, do: "", else: " upstream_status=#{status}")
test/openagents/inference/models_select_test.exs added +79

@@ -0,0 +1,79 @@

1
defmodule OpenAgents.Inference.ModelsSelectTest do
2
  use ExUnit.Case, async: false
3
4
  alias OpenAgents.Inference.Health
5
  alias OpenAgents.Inference.Models
6
7
  setup do
8
    if is_nil(Process.whereis(OpenAgents.Inference.Health)) do
9
      start_supervised!({OpenAgents.Inference.Health, []})
10
    end
11
12
    Health.reset()
13
    on_exit(&Health.reset/0)
14
    :ok
15
  end
16
17
  test "select returns the catalog default when every lane is healthy" do
18
    assert Models.select().id == Models.default_id()
19
  end
20
21
  test "select prefers the first configured and non-degraded lane when the default is degraded" do
22
    default = Models.default_id()
23
24
    for _ <- 1..Health.degraded_after() do
25
      Health.record_failure(default)
26
    end
27
28
    selected = Models.select()
29
    refute selected.id == default
30
    assert selected.id == "ox-alpha"
31
  end
32
33
  test "select returns the default when all configured lanes are degraded, not the first one" do
34
    # Make the default unavailable so the first configured lane would be a
35
    # non-default entry under the old fallback.
36
    previous = Application.get_env(:openagents, :vercel_gateway_provider)
37
38
    Application.put_env(
39
      :openagents,
40
      :vercel_gateway_provider,
41
      OpenAgents.Providers.UnconfiguredTestProvider
42
    )
43
44
    on_exit(fn -> Application.put_env(:openagents, :vercel_gateway_provider, previous) end)
45
46
    for id <- Models.ids(), id != Models.default_id(), _ <- 1..Health.degraded_after() do
47
      Health.record_failure(id)
48
    end
49
50
    assert Models.select().id == Models.default_id()
51
  end
52
53
  test "select returns the catalog default when no lane is configured" do
54
    lanes = [:provider, :openrouter_provider, :vercel_gateway_provider]
55
    previous = Map.new(lanes, &{&1, Application.get_env(:openagents, &1)})
56
57
    for lane <- lanes do
58
      Application.put_env(:openagents, lane, OpenAgents.Providers.UnconfiguredTestProvider)
59
    end
60
61
    on_exit(fn ->
62
      for {lane, value} <- previous, do: Application.put_env(:openagents, lane, value)
63
    end)
64
65
    assert Models.select().id == Models.default_id()
66
  end
67
68
  describe "the boundary of server selection" do
69
    test "a lane named in the body is never substituted, even when degraded" do
70
      # PROVIDER-002: a caller that named a model gets that model or an error.
71
      # Selection exists only for the case where nothing named one.
72
      for _ <- 1..Health.degraded_after(), do: Health.record_failure("gemini-3.7-flash", 503)
73
74
      assert {:ok, model} = Models.fetch("gemini-3.7-flash")
75
      assert model.id == "gemini-3.7-flash"
76
      assert Models.availability(model) == "degraded"
77
    end
78
  end
79
end
test/openagents_web/controllers/inference_proxy_controller_test.exs modified +75

@@ -3,6 +3,8 @@ defmodule OpenAgentsWeb.InferenceProxyControllerTest do

3 3
4 4
  alias OpenAgents.Inference
5 5
  alias OpenAgents.Inference.Grant
6
  alias OpenAgents.Inference.Health
7
  alias OpenAgents.Inference.Models
6 8
  alias OpenAgents.Machines
7 9
  alias OpenAgents.Providers.RecordingTestProvider
8 10
  alias OpenAgents.Repo

@@ -444,4 +446,77 @@ defmodule OpenAgentsWeb.InferenceProxyControllerTest do

444 446
      assert Jason.decode!(conn.resp_body)["error"]["code"] == "model_unavailable"
445 447
    end
446 448
  end
449
450
  describe "server-side model selection" do
451
    setup do
452
      Health.reset()
453
      on_exit(&Health.reset/0)
454
      :ok
455
    end
456
457
    test "all lanes healthy selects the catalog default and reports it", %{conn: conn} do
458
      %{token: token} = grant("policy-healthy")
459
460
      conn =
461
        post_chat(conn, token, %{
462
          "messages" => [%{"role" => "user", "content" => "hi"}]
463
        })
464
465
      assert conn.status == 200
466
      expected = Models.default_id()
467
      assert get_resp_header(conn, "x-openagents-model") == [expected]
468
469
      for chunk <- sse_events(conn.resp_body), chunk != "[DONE]" do
470
        assert Jason.decode!(chunk)["model"] == expected
471
      end
472
    end
473
474
    test "default degraded selects a healthy alternative and reports it", %{conn: conn} do
475
      default = Models.default_id()
476
477
      for _ <- 1..Health.degraded_after() do
478
        Health.record_failure(default)
479
      end
480
481
      expected = Models.select_id()
482
      refute expected == default
483
484
      %{token: token} = grant("policy-degraded")
485
486
      conn =
487
        post_chat(conn, token, %{
488
          "messages" => [%{"role" => "user", "content" => "hi"}]
489
        })
490
491
      assert conn.status == 200
492
      assert get_resp_header(conn, "x-openagents-model") == [expected]
493
494
      for chunk <- sse_events(conn.resp_body), chunk != "[DONE]" do
495
        assert Jason.decode!(chunk)["model"] == expected
496
      end
497
    end
498
499
    test "every lane degraded still selects the default and reports it", %{conn: conn} do
500
      for id <- Models.ids(), _ <- 1..Health.degraded_after() do
501
        Health.record_failure(id)
502
      end
503
504
      expected = Models.select_id()
505
      assert expected == Models.default_id()
506
507
      %{token: token} = grant("policy-all-degraded")
508
509
      conn =
510
        post_chat(conn, token, %{
511
          "messages" => [%{"role" => "user", "content" => "hi"}]
512
        })
513
514
      assert conn.status == 200
515
      assert get_resp_header(conn, "x-openagents-model") == [expected]
516
517
      for chunk <- sse_events(conn.resp_body), chunk != "[DONE]" do
518
        assert Jason.decode!(chunk)["model"] == expected
519
      end
520
    end
521
  end
447 522
end

This page updates live while a promote is in flight · changelog