Capture inference model selection

b259fde51b40 · AtlantisPleb · · parent f4bfff587cf6

Capture inference model selection

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

pushed
by user · WAL seq 465 · 2026-08-26T21:38:21.390749Z

Changed files

  • modified lib/openagents_web/controllers/inference_proxy_controller.ex
  • modified test/openagents_web/controllers/inference_proxy_controller_test.exs

Diff

2 files changed, +105 -0

lib/openagents_web/controllers/inference_proxy_controller.ex modified +62

@@ -37,6 +37,7 @@ defmodule OpenAgentsWeb.InferenceProxyController do

37 37
38 38
  require Logger
39 39
40
  alias OpenAgents.Analytics
40 41
  alias OpenAgents.Inference
41 42
  alias OpenAgents.Inference.Models
42 43
  alias OpenAgents.Providers.{Request, ToolDefinition, ToolOutput}

@@ -196,6 +197,9 @@ defmodule OpenAgentsWeb.InferenceProxyController do

196 197
  # ── run + translate ─────────────────────────────────────────────────────
197 198
198 199
  defp run(conn, grant, model, request) do
200
    selection = selection_properties(grant, model, request, conn.body_params)
201
    Analytics.capture("inference_model_selected", analytics_distinct_id(grant), selection)
202
199 203
    parent = self()
200 204
201 205
    # The provider pushes events synchronously; capture them to this process's

@@ -218,6 +222,17 @@ defmodule OpenAgentsWeb.InferenceProxyController do

218 222
        # answered, not what it assumed (PROVIDER-002).
219 223
        label = model_label(model, served)
220 224
225
        Analytics.capture(
226
          "inference_model_served",
227
          analytics_distinct_id(grant),
228
          Map.merge(selection, %{
229
            "served_model" => label,
230
            "served_model_disclosed" => served != :unresolved,
231
            "outcome" => "served",
232
            "usage_reported" => usage != %{}
233
          })
234
        )
235
221 236
        conn
222 237
        |> put_resp_content_type("text/event-stream")
223 238
        |> put_resp_header("cache-control", "no-store")

@@ -241,6 +256,17 @@ defmodule OpenAgentsWeb.InferenceProxyController do

241 256
            if(status == nil, do: "", else: " upstream_status=#{status}")
242 257
        )
243 258
259
        Analytics.capture(
260
          "inference_model_failed",
261
          analytics_distinct_id(grant),
262
          Map.merge(selection, %{
263
            "outcome" => "provider_failed",
264
            "reason_code" => class,
265
            "upstream_status" => status,
266
            "usage_reported" => usage != %{}
267
          })
268
        )
269
244 270
        refuse(conn, {:provider_failed, class, status})
245 271
    end
246 272
  end

@@ -256,6 +282,42 @@ defmodule OpenAgentsWeb.InferenceProxyController do

256 282
  defp meter(grant, usage, _served) when usage == %{}, do: {:ok, grant}
257 283
  defp meter(grant, usage, served), do: Inference.record_usage(grant, usage, served)
258 284
285
  # This is the complete non-secret selection record for one provider call.
286
  # It deliberately excludes the bearer grant, prompts, instructions, and tool
287
  # arguments. The grant's visitor id becomes the analytics distinct id only;
288
  # it is not an event property.
289
  defp selection_properties(grant, model, request, body) do
290
    requested = Map.get(body, "model")
291
292
    %{
293
      "selection_schema" => "inference_model_selection.v1",
294
      "selection_surface" => "inference_proxy",
295
      "requested_model" => requested,
296
      "requested_model_present" => is_binary(requested) and requested != "",
297
      "grant_model" => grant.model_id,
298
      "selected_model" => model.id,
299
      "provider" => Atom.to_string(model.provider),
300
      "provider_model" => model.provider_model,
301
      "model_availability" => Models.availability(model),
302
      "model_available" => Models.available?(model),
303
      "adapter_substitutable" => substitutable?(model.adapter),
304
      "request_model" => request.model_id,
305
      "max_output_tokens" => request.max_output,
306
      "input_message_count" => length(request.input),
307
      "tool_definition_count" => length(request.tool_definitions),
308
      "tool_output_count" => length(request.tool_outputs),
309
      "grant_call_count_before" => grant.call_count,
310
      "grant_max_calls" => grant.max_calls,
311
      "grant_max_total_tokens" => grant.max_total_tokens,
312
      "grant_max_cost_microusd" => grant.max_cost_microusd,
313
      "grant_has_thread_fence" => is_binary(grant.thread_id),
314
      "grant_has_conversation_fence" => is_binary(grant.conversation_id)
315
    }
316
  end
317
318
  defp analytics_distinct_id(grant),
319
    do: Analytics.distinct_id("visitor_" <> grant.owner_visitor_id)
320
259 321
  # Which model actually served this call.
260 322
  #
261 323
  # `:requested` where the response named the model the grant pins, and where a
test/openagents_web/controllers/inference_proxy_controller_test.exs modified +43

@@ -9,6 +9,31 @@ defmodule OpenAgentsWeb.InferenceProxyControllerTest do

9 9
  alias OpenAgents.Providers.RecordingTestProvider
10 10
  alias OpenAgents.Repo
11 11
12
  defmodule AnalyticsSink do
13
    def capture(event, distinct_id, properties) do
14
      send(:inference_proxy_analytics_test, {:analytics, event, distinct_id, properties})
15
      :ok
16
    end
17
  end
18
19
  setup do
20
    Process.register(self(), :inference_proxy_analytics_test)
21
    original_token = Application.get_env(:openagents, :posthog_project_token)
22
    original_sink = Application.get_env(:openagents, :analytics_sink)
23
    Application.put_env(:openagents, :posthog_project_token, "phc_test_token")
24
    Application.put_env(:openagents, :analytics_sink, AnalyticsSink)
25
26
    on_exit(fn ->
27
      if is_nil(original_token),
28
        do: Application.delete_env(:openagents, :posthog_project_token),
29
        else: Application.put_env(:openagents, :posthog_project_token, original_token)
30
31
      if is_nil(original_sink),
32
        do: Application.delete_env(:openagents, :analytics_sink),
33
        else: Application.put_env(:openagents, :analytics_sink, original_sink)
34
    end)
35
  end
36
12 37
  defp grant(key, options \\ []) do
13 38
    owner = github_user("proxy-#{key}")
14 39
    {:ok, conversation} = OpenAgents.Conversations.ensure_conversation(owner)

@@ -56,6 +81,7 @@ defmodule OpenAgentsWeb.InferenceProxyControllerTest do

56 81
57 82
  test "a valid grant proxies a chat completion and meters usage", %{conn: conn} do
58 83
    %{grant: grant, token: token} = grant("ok")
84
    {:ok, default} = Models.fetch(Models.default_id())
59 85
60 86
    conn =
61 87
      post_chat(conn, token, %{

@@ -105,6 +131,23 @@ defmodule OpenAgentsWeb.InferenceProxyControllerTest do

105 131
    assert metered.call_count == 1
106 132
    assert metered.usage["total_tokens"] == 12
107 133
    assert metered.usage["estimated_cost_microusd"] > 0
134
135
    assert_receive {:analytics, "inference_model_selected", distinct_id, selected}
136
    assert distinct_id =~ "visitor_"
137
    assert selected["selection_schema"] == "inference_model_selection.v1"
138
    assert selected["requested_model"] == Models.default_id()
139
    assert selected["grant_model"] == Models.default_id()
140
    assert selected["selected_model"] == Models.default_id()
141
    assert selected["provider_model"] == default.provider_model
142
    assert selected["input_message_count"] == 1
143
    assert selected["tool_definition_count"] == 0
144
    assert selected["tool_output_count"] == 0
145
    assert selected["grant_has_conversation_fence"]
146
    refute selected["grant_has_thread_fence"]
147
148
    assert_receive {:analytics, "inference_model_served", ^distinct_id, served}
149
    assert served["served_model"] == Models.default_id()
150
    assert served["outcome"] == "served"
108 151
  end
109 152
110 153
  test "a reasoning stream survives translation as delta.reasoning", %{conn: conn} do

This page updates live while a promote is in flight · changelog