Keep the reasoning count a turn actually reported

e9c3682629fa · Devin AI · · parent 472cfe3e7827

Keep the reasoning count a turn actually reported

Every tool round of a turn is its own provider request reporting only what
that round consumed, so the last round's usage replaced the earlier counts
and the reasoning a model spent before a tool call never reached the chat
console. Sum the counts each round reports instead.

A model that carries reasoning output while reporting a reasoning count of
zero measures nothing, so that turn now reports no reasoning count rather
than a count of none, and the top bar omits the category instead of
reading Reasoning 0.

Closes #116

Co-Authored-By: Christopher David <chris@openagents.com>
Co-Authored-By
Christopher David <chris@openagents.com>
Closes
#116

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified lib/openagents/chat/account_turns.ex
  • modified lib/openagents/chat/open_router.ex
  • modified test/fixtures/openrouter/responses_tool_call.sse
  • modified test/openagents/chat/open_router_test.exs
  • modified test/openagents_web/live/chat_console_test.exs

Diff

5 files changed, +172 -12

lib/openagents/chat/account_turns.ex modified +27 -8

@@ -373,7 +373,7 @@ defmodule OpenAgents.Chat.AccountTurns do

373 373
  defp finish_run(run_id, {:ok, completion}) do
374 374
    # Token counts are read before redaction, which blanks every field whose
375 375
    # name contains `token`, and are stored beside the redacted completion.
376
    usage = usage_counts(completion["usage"])
376
    usage = usage_counts(completion)
377 377
    completion = OpenAgents.Tools.Redaction.redact(completion)
378 378
379 379
    terminal_update(run_id, "response_completed", completion, %{

@@ -508,16 +508,12 @@ defmodule OpenAgents.Chat.AccountTurns do

508 508
509 509
  # Provider-reported counts only. OpenRouter names them differently across its
510 510
  # two APIs, and a field the provider left out stays `nil` instead of a guess.
511
  defp usage_counts(usage) when is_map(usage) do
511
  defp usage_counts(%{"usage" => usage} = completion) when is_map(usage) do
512 512
    counts = %{
513 513
      "input" => token_count(usage["input_tokens"] || usage["prompt_tokens"]),
514 514
      "output" => token_count(usage["output_tokens"] || usage["completion_tokens"]),
515 515
      "total" => token_count(usage["total_tokens"]),
516
      "reasoning" =>
517
        token_count(
518
          detail(usage, "output_tokens_details", "reasoning_tokens") ||
519
            detail(usage, "completion_tokens_details", "reasoning_tokens")
520
        ),
516
      "reasoning" => reasoning_count(usage, completion),
521 517
      "cached" =>
522 518
        token_count(
523 519
          detail(usage, "input_tokens_details", "cached_tokens") ||

@@ -528,7 +524,30 @@ defmodule OpenAgents.Chat.AccountTurns do

528 524
    if Enum.all?(Map.values(counts), &is_nil/1), do: nil, else: counts
529 525
  end
530 526
531
  defp usage_counts(_usage), do: nil
527
  defp usage_counts(_completion), do: nil
528
529
  # Some models report a reasoning count of zero for a turn that carries
530
  # reasoning output. That zero measures nothing, so the turn reports no
531
  # reasoning count rather than a count of none.
532
  defp reasoning_count(usage, completion) do
533
    count =
534
      token_count(
535
        detail(usage, "output_tokens_details", "reasoning_tokens") ||
536
          detail(usage, "completion_tokens_details", "reasoning_tokens")
537
      )
538
539
    if count == 0 and reasoned?(completion), do: nil, else: count
540
  end
541
542
  defp reasoned?(%{"reasoning_summary" => summary}) when is_binary(summary) and summary != "",
543
    do: true
544
545
  defp reasoned?(%{"reasoning_items" => [_item | _rest]}), do: true
546
547
  defp reasoned?(%{"output" => output}) when is_list(output),
548
    do: Enum.any?(output, &match?(%{"type" => "reasoning"}, &1))
549
550
  defp reasoned?(_completion), do: false
532 551
533 552
  defp usage_view(counts) when is_map(counts),
534 553
    do: %{
lib/openagents/chat/open_router.ex modified +30 -3

@@ -266,7 +266,7 @@ defmodule OpenAgents.Chat.OpenRouter do

266 266
  defp chat_request(request), do: request
267 267
268 268
  defp continue_responses_tool_calls(
269
         {:ok, %{"tool_calls" => tool_calls, "output" => provider_output}},
269
         {:ok, %{"tool_calls" => tool_calls, "output" => provider_output} = completion},
270 270
         api_key,
271 271
         payload,
272 272
         on_event,

@@ -279,8 +279,9 @@ defmodule OpenAgents.Chat.OpenRouter do

279 279
         payload <- Map.update!(payload, "input", &(&1 ++ provider_output ++ tool_outputs)),
280 280
         {:ok, response} <- responses_stream_request(api_key, payload, options),
281 281
         result <- consume_responses_stream(response, on_event, payload["model"]) do
282
      continue_responses_tool_calls(
283
        result,
282
      result
283
      |> carry_usage(Map.get(completion, "usage"))
284
      |> continue_responses_tool_calls(
284 285
        api_key,
285 286
        payload,
286 287
        on_event,

@@ -313,6 +314,32 @@ defmodule OpenAgents.Chat.OpenRouter do

313 314
       ),
314 315
       do: result
315 316
317
  # A turn spends one provider request per tool round, and each response reports
318
  # only what that round consumed. Carrying the earlier counts forward keeps the
319
  # whole turn in the usage the completion reports.
320
  defp carry_usage({:ok, completion}, earlier) when is_map(earlier),
321
    do: {:ok, Map.put(completion, "usage", add_usage(Map.get(completion, "usage"), earlier))}
322
323
  defp carry_usage(result, _earlier), do: result
324
325
  # A count no round reported stays absent instead of becoming a zero.
326
  defp add_usage(usage, earlier) when is_map(usage) and is_map(earlier),
327
    do:
328
      Map.merge(usage, earlier, fn _key, value, earlier_value ->
329
        add_count(value, earlier_value)
330
      end)
331
332
  defp add_usage(nil, earlier), do: earlier
333
  defp add_usage(usage, _earlier), do: usage
334
335
  defp add_count(value, earlier) when is_number(value) and is_number(earlier),
336
    do: value + earlier
337
338
  defp add_count(value, earlier) when is_map(value) and is_map(earlier),
339
    do: add_usage(value, earlier)
340
341
  defp add_count(value, _earlier), do: value
342
316 343
  defp execute_tool_calls(tool_calls, on_event, tool_runtime) do
317 344
    result =
318 345
      Enum.reduce_while(tool_calls, [], fn %{
test/fixtures/openrouter/responses_tool_call.sse modified +1 -1

@@ -20,6 +20,6 @@ data: {"type":"response.function_call_arguments.done","response_id":"resp_demo",

20 20
21 21
data: {"type":"response.output_item.done","response_id":"resp_demo","output_index":2,"item":{"type":"function_call","id":"fc_demo","call_id":"call_demo","name":"read_repository_file","arguments":"{\"repository\":\"OpenAgentsInc/openagents.com\",\"path\":\"README.md\",\"ref\":\"\"}","status":"completed"}}
22 22
23
data: {"type":"response.completed","response":{"id":"resp_demo","object":"response","status":"completed","model":"stealth/ox-alpha","output":[{"type":"reasoning","id":"rs_demo","status":"completed","summary":[{"type":"summary_text","text":"The user asked to read a repository file."}],"encrypted_content":"encrypted-demo-reasoning"},{"type":"message","id":"msg_tool_preamble","role":"assistant","status":"completed","content":[{"type":"output_text","text":"I will inspect the connected repository.","annotations":[]}]},{"type":"function_call","id":"fc_demo","call_id":"call_demo","name":"read_repository_file","arguments":"{\"repository\":\"OpenAgentsInc/openagents.com\",\"path\":\"README.md\",\"ref\":\"\"}","status":"completed"}],"usage":{"input_tokens":18,"output_tokens":11,"total_tokens":29}}}
23
data: {"type":"response.completed","response":{"id":"resp_demo","object":"response","status":"completed","model":"stealth/ox-alpha","output":[{"type":"reasoning","id":"rs_demo","status":"completed","summary":[{"type":"summary_text","text":"The user asked to read a repository file."}],"encrypted_content":"encrypted-demo-reasoning"},{"type":"message","id":"msg_tool_preamble","role":"assistant","status":"completed","content":[{"type":"output_text","text":"I will inspect the connected repository.","annotations":[]}]},{"type":"function_call","id":"fc_demo","call_id":"call_demo","name":"read_repository_file","arguments":"{\"repository\":\"OpenAgentsInc/openagents.com\",\"path\":\"README.md\",\"ref\":\"\"}","status":"completed"}],"usage":{"input_tokens":18,"output_tokens":11,"total_tokens":29,"output_tokens_details":{"reasoning_tokens":7}}}}
24 24
25 25
data: [DONE]
test/openagents/chat/open_router_test.exs modified +75

@@ -965,6 +965,81 @@ defmodule OpenAgents.Chat.OpenRouterTest do

965 965
           } = Jason.decode!(tool_output)
966 966
  end
967 967
968
  test "sums the usage every tool round reported" do
969
    Req.Test.expect(__MODULE__, fn conn ->
970
      body =
971
        Path.expand("../../fixtures/openrouter/responses_tool_call.sse", __DIR__)
972
        |> File.read!()
973
974
      conn
975
      |> Plug.Conn.put_resp_content_type("text/event-stream")
976
      |> Plug.Conn.send_resp(200, body)
977
    end)
978
979
    Req.Test.expect(__MODULE__, fn conn ->
980
      body =
981
        sse(%{
982
          "type" => "response.content_part.delta",
983
          "delta" => "OpenAgents is an agent platform."
984
        }) <>
985
          sse(%{
986
            "type" => "response.completed",
987
            "response" => %{
988
              "object" => "response",
989
              "model" => "stealth/ox-alpha",
990
              "output" => [
991
                %{
992
                  "type" => "message",
993
                  "id" => "msg_after_tool",
994
                  "role" => "assistant",
995
                  "status" => "completed",
996
                  "content" => [
997
                    %{
998
                      "type" => "output_text",
999
                      "text" => "OpenAgents is an agent platform.",
1000
                      "annotations" => []
1001
                    }
1002
                  ]
1003
                }
1004
              ],
1005
              "usage" => %{
1006
                "input_tokens" => 30,
1007
                "output_tokens" => 5,
1008
                "total_tokens" => 35,
1009
                "output_tokens_details" => %{"reasoning_tokens" => 0}
1010
              }
1011
            }
1012
          }) <> "data: [DONE]\n\n"
1013
1014
      conn
1015
      |> Plug.Conn.put_resp_content_type("text/event-stream")
1016
      |> Plug.Conn.send_resp(200, body)
1017
    end)
1018
1019
    parent = self()
1020
    assert {:ok, tool_registry_snapshot} = Registry.build([RepositoryFileToolStub])
1021
1022
    assert {:ok, %{"usage" => usage}} =
1023
             OpenRouter.stream(
1024
               %{
1025
                 "model" => "stealth/ox-alpha",
1026
                 "messages" => [%{"role" => "user", "content" => "Summarize the README."}]
1027
               },
1028
               &send(parent, {:openrouter_event, &1}),
1029
               api_key: "test-openrouter-key",
1030
               tool_registry_snapshot: tool_registry_snapshot,
1031
               tool_execution_context: tool_execution_context(),
1032
               request_options: [plug: {Req.Test, __MODULE__}]
1033
             )
1034
1035
    assert usage == %{
1036
             "input_tokens" => 48,
1037
             "output_tokens" => 16,
1038
             "total_tokens" => 64,
1039
             "output_tokens_details" => %{"reasoning_tokens" => 7}
1040
           }
1041
  end
1042
968 1043
  test "replays provider output around ordered read, write, edit, and reread calls" do
969 1044
    repository_state = start_supervised!({Agent, fn -> "initial" end})
970 1045
test/openagents_web/live/chat_console_test.exs modified +39

@@ -140,6 +140,45 @@ defmodule OpenAgentsWeb.ChatConsoleTest do

140 140
    refute has_element?(view, "#chat-console-evidence-#{run_id}")
141 141
  end
142 142
143
  test "a turn that reasoned without a reasoning count omits the category", %{conn: conn} do
144
    key = "console-unmetered-reasoning-operator"
145
    user = github_user(key)
146
    conn = log_in_admin_user(conn, key)
147
148
    streamer = fn _request, callback, _options ->
149
      callback.({:reasoning_delta, "Weighing the fleet."})
150
      callback.({:text_delta, "The fleet is idle."})
151
152
      {:ok,
153
       %{
154
         "object" => "response",
155
         "model" => "stealth/ox-alpha",
156
         "assistant_content" => "The fleet is idle.",
157
         "reasoning_summary" => "Weighing the fleet.",
158
         "usage" => %{
159
           "input_tokens" => 24,
160
           "output_tokens" => 8,
161
           "total_tokens" => 32,
162
           "output_tokens_details" => %{"reasoning_tokens" => 0}
163
         }
164
       }}
165
    end
166
167
    assert {:ok, %{"id" => run_id}} =
168
             AccountTurns.submit(user, "Report the fleet.",
169
               subscriber: self(),
170
               streamer: streamer
171
             )
172
173
    assert_receive {:account_chat_completed, ^run_id, {:ok, _completion}}
174
    {:ok, view, _html} = live(conn, ~p"/chat")
175
176
    assert has_element?(view, "#chat-console-usage-#{run_id}", "Input 24")
177
    refute has_element?(view, "#chat-console-usage-#{run_id}", "Reasoning")
178
    assert has_element?(view, "#chat-console-token-input", "24")
179
    refute has_element?(view, "#chat-console-token-reasoning")
180
  end
181
143 182
  test "a completed turn shows a context meter where a window is configured", %{conn: conn} do
144 183
    key = "console-context-operator"
145 184
    user = github_user(key)

This page updates live while a promote is in flight · changelog