OpenAgents

Commit

Preserve image input through the inference proxy

13c21a41a546 · AtlantisPleb · · parent 139debc05e69 

Preserve image input through the inference proxy

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

pushed
by user · WAL seq 467 · 2026-08-27T03:51:31.446654Z

Changed files

  • modified lib/openagents/providers/open_ai.ex
  • modified lib/openagents/providers/request.ex
  • modified lib/openagents_web/controllers/inference_proxy_controller.ex
  • modified test/openagents/providers/open_ai/request_payload_test.exs
  • modified test/openagents/providers/open_router/request_payload_test.exs
  • modified test/openagents_web/controllers/inference_proxy_controller_test.exs

Diff

6 files changed, +140 -5

lib/openagents/providers/open_ai.ex modified +10

@@ -136,8 +136,18 @@ defmodule OpenAgents.Providers.OpenAI do

136 136
137 137 
  defp items_for_message(message, _outputs_by_call_id), do: [message_item(message)]
138 138
139 
  defp message_item(%{content: content} = message) when is_list(content) do
140 
    %{role: message.role, content: Enum.map(content, &response_content_part/1)}
141 
  end
142
139 143 
  defp message_item(message), do: %{role: message.role, content: message.content}
140 144
145 
  defp response_content_part(%{type: "text", text: text}),
146 
    do: %{type: "input_text", text: text}
147
148 
  defp response_content_part(%{type: "image_url", image_url: %{url: url}}),
149 
    do: %{type: "input_image", image_url: url}
150
141 151 
  defp function_call_item(call) do
142 152 
    %{
143 153 
      type: "function_call",
lib/openagents/providers/request.ex modified +5 -1

@@ -23,9 +23,13 @@ defmodule OpenAgents.Providers.Request do

23 23 
  """
24 24 
  @type message_tool_call :: %{call_id: String.t(), name: String.t(), arguments: String.t()}
25 25
26 
  @type content_part ::
27 
          %{type: String.t(), text: String.t()}
28 
          | %{type: String.t(), image_url: %{url: String.t()}}
29
26 30 
  @type message :: %{
27 31 
          :role => String.t(),
28 
          :content => String.t(),
32 
          :content => String.t() | [content_part()],
29 33 
          optional(:tool_calls) => [message_tool_call()]
30 34 
        }
31 35 
  @type t :: %__MODULE__{
lib/openagents_web/controllers/inference_proxy_controller.ex modified +38 -4

@@ -136,14 +136,15 @@ defmodule OpenAgentsWeb.InferenceProxyController do

136 136
137 137 
  defp input_message(message) do
138 138 
    tool_calls = message_tool_calls(message["tool_calls"])
139 
    content = message_content(message)
139 140
140 
    case {content_text(message), tool_calls} do
141 
      {"", []} -> []
142 
      {text, []} -> [%{role: role(message), content: text}]
141 
    case {content_empty?(content), tool_calls} do
142 
      {true, []} -> []
143 
      {false, []} -> [%{role: role(message), content: content}]
143 144 
      # An assistant turn that called tools is part of the transcript even
144 145 
      # when it carried no prose: dropping it would orphan the tool outputs
145 146 
      # that answer it.
146 
      {text, calls} -> [%{role: role(message), content: text, tool_calls: calls}]
147 
      {_empty, calls} -> [%{role: role(message), content: content, tool_calls: calls}]
147 148 
    end
148 149 
  end
149 150

@@ -575,6 +576,39 @@ defmodule OpenAgentsWeb.InferenceProxyController do

575 576
576 577 
  defp content_text(_), do: ""
577 578
579 
  defp message_content(%{"content" => content}) when is_binary(content), do: content
580
581 
  defp message_content(%{"content" => parts}) when is_list(parts) do
582 
    Enum.flat_map(parts, fn
583 
      %{"type" => "text", "text" => text} when is_binary(text) ->
584 
        [%{type: "text", text: text}]
585
586 
      %{"type" => "image_url", "image_url" => %{"url" => url}}
587 
      when is_binary(url) ->
588 
        if inline_image?(url),
589 
          do: [%{type: "image_url", image_url: %{url: url}}],
590 
          else: []
591
592 
      _invalid ->
593 
        []
594 
    end)
595 
  end
596
597 
  defp message_content(_), do: ""
598
599 
  defp content_empty?(""), do: true
600 
  defp content_empty?([]), do: true
601 
  defp content_empty?(_content), do: false
602
603 
  defp inline_image?(url) do
604 
    String.starts_with?(url, [
605 
      "data:image/png;base64,",
606 
      "data:image/jpeg;base64,",
607 
      "data:image/gif;base64,",
608 
      "data:image/webp;base64,"
609 
    ])
610 
  end
611
578 612 
  defp join_text(messages) do
579 613 
    messages |> Enum.map(&content_text/1) |> Enum.reject(&(&1 == "")) |> Enum.join("\n")
580 614 
  end
test/openagents/providers/open_ai/request_payload_test.exs modified +28

@@ -114,4 +114,32 @@ defmodule OpenAgents.Providers.OpenAI.RequestPayloadTest do

114 114 
             %{type: "function_call_output", call_id: "call_orphan", output: _output}
115 115 
           ] = OpenAI.request_payload(request).input
116 116 
  end
117
118 
  test "maps multimodal user content to Responses input parts" do
119 
    image_url = "data:image/png;base64,iVBORw0KGgo="
120
121 
    request = %Request{
122 
      model_id: "test-model",
123 
      instructions: "",
124 
      input: [
125 
        %{
126 
          role: "user",
127 
          content: [
128 
            %{type: "text", text: "Describe this image."},
129 
            %{type: "image_url", image_url: %{url: image_url}}
130 
          ]
131 
        }
132 
      ]
133 
    }
134
135 
    assert OpenAI.request_payload(request).input == [
136 
             %{
137 
               role: "user",
138 
               content: [
139 
                 %{type: "input_text", text: "Describe this image."},
140 
                 %{type: "input_image", image_url: image_url}
141 
               ]
142 
             }
143 
           ]
144 
  end
117 145 
end
test/openagents/providers/open_router/request_payload_test.exs modified +28

@@ -43,6 +43,34 @@ defmodule OpenAgents.Providers.OpenRouter.RequestPayloadTest do

43 43 
           ]
44 44 
  end
45 45
46 
  test "preserves multimodal user content" do
47 
    image_url = "data:image/png;base64,iVBORw0KGgo="
48
49 
    request = %Request{
50 
      model_id: "z-ai/glm-5.3-flash",
51 
      instructions: "",
52 
      input: [
53 
        %{
54 
          role: "user",
55 
          content: [
56 
            %{type: "text", text: "Describe this image."},
57 
            %{type: "image_url", image_url: %{url: image_url}}
58 
          ]
59 
        }
60 
      ]
61 
    }
62
63 
    assert OpenRouter.request_payload(request).messages == [
64 
             %{
65 
               role: "user",
66 
               content: [
67 
                 %{type: "text", text: "Describe this image."},
68 
                 %{type: "image_url", image_url: %{url: image_url}}
69 
               ]
70 
             }
71 
           ]
72 
  end
73
46 74 
  test "maps tool definitions to chat-completions functions" do
47 75 
    request = %Request{
48 76 
      model_id: "z-ai/glm-5.3-flash",
test/openagents_web/controllers/inference_proxy_controller_test.exs modified +31

@@ -449,6 +449,37 @@ defmodule OpenAgentsWeb.InferenceProxyControllerTest do

449 449 
      assert output.output == %{"content" => "hello"}
450 450 
    end
451 451
452 
    test "an inline image reaches the provider as multimodal content", %{conn: conn} do
453 
      %{token: token} = grant("image-input", model_id: "glm-5.3-flash")
454 
      image_url = "data:image/png;base64,iVBORw0KGgo="
455
456 
      conn =
457 
        post_chat(conn, token, %{
458 
          "messages" => [
459 
            %{
460 
              "role" => "user",
461 
              "content" => [
462 
                %{"type" => "text", "text" => "Describe this image."},
463 
                %{"type" => "image_url", "image_url" => %{"url" => image_url}}
464 
              ]
465 
            }
466 
          ]
467 
        })
468
469 
      assert conn.status == 200
470 
      assert_received {:recorded_request, "test.recording_provider", request}
471
472 
      assert request.input == [
473 
               %{
474 
                 role: "user",
475 
                 content: [
476 
                   %{type: "text", text: "Describe this image."},
477 
                   %{type: "image_url", image_url: %{url: image_url}}
478 
                 ]
479 
               }
480 
             ]
481 
    end
482
452 483 
    test "a default grant is called with the default model's own vendor string", %{conn: conn} do
453 484 
      # This once asserted the default lane was *not* the recorded one, which
454 485 
      # only held while the default sat on the other adapter. What it was

This page updates live while a promote is in flight · changelog