|
1
|
+ |
defmodule OpenAgents.Providers.OpenRouter.StreamDecoder do
|
|
2
|
+ |
@moduledoc false
|
|
3
|
+ |
|
|
4
|
+ |
alias OpenAgents.Providers.ProviderEvent.ToolCall
|
|
5
|
+ |
|
|
6
|
+ |
@maximum_buffer_bytes 262_144
|
|
7
|
+ |
@maximum_delta_bytes 65_536
|
|
8
|
+ |
@maximum_arguments_bytes 65_536
|
|
9
|
+ |
@identifier_regex ~r/\A[a-zA-Z0-9_.:\/-]+\z/
|
|
10
|
+ |
@tool_name_regex ~r/\A[a-zA-Z0-9_-]+\z/
|
|
11
|
+ |
|
|
12
|
+ |
defstruct buffer: "", response_id: nil, terminal?: false, failed?: false, calls: %{}
|
|
13
|
+ |
|
|
14
|
+ |
@type t :: %__MODULE__{
|
|
15
|
+ |
buffer: String.t(),
|
|
16
|
+ |
response_id: String.t() | nil,
|
|
17
|
+ |
terminal?: boolean(),
|
|
18
|
+ |
failed?: boolean(),
|
|
19
|
+ |
calls: %{optional(integer()) => map()}
|
|
20
|
+ |
}
|
|
21
|
+ |
|
|
22
|
+ |
@spec new() :: t()
|
|
23
|
+ |
def new, do: %__MODULE__{}
|
|
24
|
+ |
|
|
25
|
+ |
@spec feed(t(), binary()) ::
|
|
26
|
+ |
{:ok, t(), [OpenAgents.Providers.ProviderEvent.t()]} | {:error, atom()}
|
|
27
|
+ |
def feed(%__MODULE__{} = state, chunk) when is_binary(chunk) do
|
|
28
|
+ |
buffer = String.replace(state.buffer <> chunk, "\r\n", "\n")
|
|
29
|
+ |
|
|
30
|
+ |
if byte_size(buffer) > @maximum_buffer_bytes do
|
|
31
|
+ |
{:error, :invalid_provider_event}
|
|
32
|
+ |
else
|
|
33
|
+ |
parts = String.split(buffer, "\n\n")
|
|
34
|
+ |
{frames, [remainder]} = Enum.split(parts, -1)
|
|
35
|
+ |
decode_frames(%{state | buffer: remainder}, frames)
|
|
36
|
+ |
end
|
|
37
|
+ |
end
|
|
38
|
+ |
|
|
39
|
+ |
@doc """
|
|
40
|
+ |
Close the stream.
|
|
41
|
+ |
|
|
42
|
+ |
A chat-completions stream reports its end with a `finish_reason`, and the
|
|
43
|
+ |
accumulated tool calls are only whole once that arrives, so they are emitted
|
|
44
|
+ |
here rather than mid-stream. A stream that ends without one is truncated: the
|
|
45
|
+ |
reply it carried is a fragment, and reporting it as complete would hand a
|
|
46
|
+ |
caller half an answer as a whole one.
|
|
47
|
+ |
|
|
48
|
+ |
A stream that already reported a failure is closed without a completion and
|
|
49
|
+ |
without its part-built calls: the failure is the outcome, and a completion
|
|
50
|
+ |
after it would say the reply arrived.
|
|
51
|
+ |
"""
|
|
52
|
+ |
@spec finish(t()) :: {:ok, t(), [OpenAgents.Providers.ProviderEvent.t()]} | {:error, atom()}
|
|
53
|
+ |
def finish(%__MODULE__{} = state) do
|
|
54
|
+ |
with {:ok, state, events} <- decode_final_buffer(state, state.buffer) do
|
|
55
|
+ |
cond do
|
|
56
|
+ |
state.failed? -> {:ok, state, events}
|
|
57
|
+ |
state.terminal? -> {:ok, state, events ++ completion_events(state)}
|
|
58
|
+ |
true -> {:error, :truncated_stream}
|
|
59
|
+ |
end
|
|
60
|
+ |
end
|
|
61
|
+ |
end
|
|
62
|
+ |
|
|
63
|
+ |
defp completion_events(state) do
|
|
64
|
+ |
tool_call_events(state) ++ [{:response_completed, id(state)}]
|
|
65
|
+ |
end
|
|
66
|
+ |
|
|
67
|
+ |
defp decode_final_buffer(state, buffer) do
|
|
68
|
+ |
if String.trim(buffer) == "" do
|
|
69
|
+ |
{:ok, %{state | buffer: ""}, []}
|
|
70
|
+ |
else
|
|
71
|
+ |
decode_frames(%{state | buffer: ""}, [buffer])
|
|
72
|
+ |
end
|
|
73
|
+ |
end
|
|
74
|
+ |
|
|
75
|
+ |
defp decode_frames(state, frames) do
|
|
76
|
+ |
Enum.reduce_while(frames, {:ok, state, []}, fn frame, {:ok, next_state, events} ->
|
|
77
|
+ |
case decode_frame(next_state, frame) do
|
|
78
|
+ |
{:ok, decoded_state, decoded_events} ->
|
|
79
|
+ |
{:cont, {:ok, decoded_state, events ++ decoded_events}}
|
|
80
|
+ |
|
|
81
|
+ |
{:error, reason} ->
|
|
82
|
+ |
{:halt, {:error, reason}}
|
|
83
|
+ |
end
|
|
84
|
+ |
end)
|
|
85
|
+ |
end
|
|
86
|
+ |
|
|
87
|
+ |
defp decode_frame(state, frame) do
|
|
88
|
+ |
data =
|
|
89
|
+ |
frame
|
|
90
|
+ |
|> String.split("\n")
|
|
91
|
+ |
|> Enum.filter(&String.starts_with?(&1, "data:"))
|
|
92
|
+ |
|> Enum.map_join("\n", fn line ->
|
|
93
|
+ |
line |> String.replace_prefix("data:", "") |> String.trim_leading()
|
|
94
|
+ |
end)
|
|
95
|
+ |
|
|
96
|
+ |
case data do
|
|
97
|
+ |
"" -> {:ok, state, []}
|
|
98
|
+ |
"[DONE]" -> {:ok, %{state | terminal?: true}, []}
|
|
99
|
+ |
json -> decode_json(state, Jason.decode(json))
|
|
100
|
+ |
end
|
|
101
|
+ |
end
|
|
102
|
+ |
|
|
103
|
+ |
defp decode_json(_state, {:error, _error}), do: {:error, :invalid_provider_event}
|
|
104
|
+ |
|
|
105
|
+ |
defp decode_json(state, {:ok, %{"error" => error}}) when is_map(error) do
|
|
106
|
+ |
{:ok, %{state | terminal?: true, failed?: true},
|
|
107
|
+ |
[{:failed, {:provider_failed, error_code(error)}}]}
|
|
108
|
+ |
end
|
|
109
|
+ |
|
|
110
|
+ |
defp decode_json(state, {:ok, %{} = chunk}) do
|
|
111
|
+ |
with {:ok, state, start_events} <- start_response(state, chunk["id"]),
|
|
112
|
+ |
{:ok, state, choice_events} <- choices(state, chunk["choices"]),
|
|
113
|
+ |
{:ok, usage_events} <- usage(chunk["usage"]) do
|
|
114
|
+ |
{:ok, state, start_events ++ choice_events ++ usage_events}
|
|
115
|
+ |
end
|
|
116
|
+ |
end
|
|
117
|
+ |
|
|
118
|
+ |
defp decode_json(_state, {:ok, _invalid}), do: {:error, :invalid_provider_event}
|
|
119
|
+ |
|
|
120
|
+ |
defp choices(state, nil), do: {:ok, state, []}
|
|
121
|
+ |
|
|
122
|
+ |
defp choices(state, choices) when is_list(choices) do
|
|
123
|
+ |
Enum.reduce_while(choices, {:ok, state, []}, fn choice, {:ok, next_state, events} ->
|
|
124
|
+ |
case choice(next_state, choice) do
|
|
125
|
+ |
{:ok, decoded_state, decoded_events} ->
|
|
126
|
+ |
{:cont, {:ok, decoded_state, events ++ decoded_events}}
|
|
127
|
+ |
|
|
128
|
+ |
{:error, reason} ->
|
|
129
|
+ |
{:halt, {:error, reason}}
|
|
130
|
+ |
end
|
|
131
|
+ |
end)
|
|
132
|
+ |
end
|
|
133
|
+ |
|
|
134
|
+ |
defp choices(_state, _invalid), do: {:error, :invalid_provider_event}
|
|
135
|
+ |
|
|
136
|
+ |
defp choice(state, %{} = choice) do
|
|
137
|
+ |
delta = if is_map(choice["delta"]), do: choice["delta"], else: %{}
|
|
138
|
+ |
state = if is_binary(choice["finish_reason"]), do: %{state | terminal?: true}, else: state
|
|
139
|
+ |
|
|
140
|
+ |
with {:ok, text_events} <- text(delta["content"]),
|
|
141
|
+ |
{:ok, state} <- tool_calls(state, delta["tool_calls"]) do
|
|
142
|
+ |
{:ok, state, text_events}
|
|
143
|
+ |
end
|
|
144
|
+ |
end
|
|
145
|
+ |
|
|
146
|
+ |
defp choice(_state, _invalid), do: {:error, :invalid_provider_event}
|
|
147
|
+ |
|
|
148
|
+ |
defp text(nil), do: {:ok, []}
|
|
149
|
+ |
defp text(""), do: {:ok, []}
|
|
150
|
+ |
|
|
151
|
+ |
defp text(content) when is_binary(content) and byte_size(content) <= @maximum_delta_bytes,
|
|
152
|
+ |
do: {:ok, [{:text_delta, content}]}
|
|
153
|
+ |
|
|
154
|
+ |
defp text(_content), do: {:error, :invalid_provider_event}
|
|
155
|
+ |
|
|
156
|
+ |
# A chat-completions tool call streams as fragments keyed by index: the name
|
|
157
|
+ |
# arrives once and the arguments arrive as a string built up over chunks, so
|
|
158
|
+ |
# the fragments are accumulated and emitted whole at the end of the stream.
|
|
159
|
+ |
defp tool_calls(state, nil), do: {:ok, state}
|
|
160
|
+ |
|
|
161
|
+ |
defp tool_calls(state, calls) when is_list(calls) do
|
|
162
|
+ |
Enum.reduce_while(calls, {:ok, state}, fn call, {:ok, next_state} ->
|
|
163
|
+ |
case tool_call(next_state, call) do
|
|
164
|
+ |
{:ok, decoded} -> {:cont, {:ok, decoded}}
|
|
165
|
+ |
{:error, reason} -> {:halt, {:error, reason}}
|
|
166
|
+ |
end
|
|
167
|
+ |
end)
|
|
168
|
+ |
end
|
|
169
|
+ |
|
|
170
|
+ |
defp tool_calls(_state, _invalid), do: {:error, :invalid_provider_event}
|
|
171
|
+ |
|
|
172
|
+ |
defp tool_call(state, %{} = call) do
|
|
173
|
+ |
index = if is_integer(call["index"]), do: call["index"], else: 0
|
|
174
|
+ |
function = if is_map(call["function"]), do: call["function"], else: %{}
|
|
175
|
+ |
held = Map.get(state.calls, index, %{id: nil, name: nil, arguments: ""})
|
|
176
|
+ |
|
|
177
|
+ |
arguments =
|
|
178
|
+ |
case function["arguments"] do
|
|
179
|
+ |
fragment when is_binary(fragment) -> held.arguments <> fragment
|
|
180
|
+ |
_absent -> held.arguments
|
|
181
|
+ |
end
|
|
182
|
+ |
|
|
183
|
+ |
if byte_size(arguments) > @maximum_arguments_bytes do
|
|
184
|
+ |
{:error, :invalid_provider_event}
|
|
185
|
+ |
else
|
|
186
|
+ |
merged = %{
|
|
187
|
+ |
id: text_or(call["id"], held.id),
|
|
188
|
+ |
name: text_or(function["name"], held.name),
|
|
189
|
+ |
arguments: arguments
|
|
190
|
+ |
}
|
|
191
|
+ |
|
|
192
|
+ |
{:ok, %{state | calls: Map.put(state.calls, index, merged)}}
|
|
193
|
+ |
end
|
|
194
|
+ |
end
|
|
195
|
+ |
|
|
196
|
+ |
defp tool_call(_state, _invalid), do: {:error, :invalid_provider_event}
|
|
197
|
+ |
|
|
198
|
+ |
defp text_or(value, _fallback) when is_binary(value) and value != "", do: value
|
|
199
|
+ |
defp text_or(_value, fallback), do: fallback
|
|
200
|
+ |
|
|
201
|
+ |
defp tool_call_events(%__MODULE__{calls: calls} = state) do
|
|
202
|
+ |
calls
|
|
203
|
+ |
|> Enum.sort_by(fn {index, _call} -> index end)
|
|
204
|
+ |
|> Enum.flat_map(fn {index, call} -> tool_call_event(state, index, call) end)
|
|
205
|
+ |
end
|
|
206
|
+ |
|
|
207
|
+ |
defp tool_call_event(state, index, %{name: name} = call) when is_binary(name) do
|
|
208
|
+ |
if Regex.match?(@tool_name_regex, name) and byte_size(name) <= 128 do
|
|
209
|
+ |
call_id = call.id || "#{id(state)}-#{index}"
|
|
210
|
+ |
|
|
211
|
+ |
[
|
|
212
|
+ |
{:tool_call,
|
|
213
|
+ |
%ToolCall{
|
|
214
|
+ |
item_id: call_id,
|
|
215
|
+ |
call_id: call_id,
|
|
216
|
+ |
name: name,
|
|
217
|
+ |
raw_arguments: if(call.arguments == "", do: "{}", else: call.arguments)
|
|
218
|
+ |
}}
|
|
219
|
+ |
]
|
|
220
|
+ |
else
|
|
221
|
+ |
[]
|
|
222
|
+ |
end
|
|
223
|
+ |
end
|
|
224
|
+ |
|
|
225
|
+ |
defp tool_call_event(_state, _index, _call), do: []
|
|
226
|
+ |
|
|
227
|
+ |
defp start_response(%__MODULE__{response_id: nil} = state, response_id) do
|
|
228
|
+ |
if valid_identifier?(response_id) do
|
|
229
|
+ |
{:ok, %{state | response_id: response_id}, [{:response_started, response_id}]}
|
|
230
|
+ |
else
|
|
231
|
+ |
{:ok, %{state | response_id: nil}, []}
|
|
232
|
+ |
end
|
|
233
|
+ |
end
|
|
234
|
+ |
|
|
235
|
+ |
defp start_response(%__MODULE__{} = state, _response_id), do: {:ok, state, []}
|
|
236
|
+ |
|
|
237
|
+ |
defp usage(nil), do: {:ok, []}
|
|
238
|
+ |
|
|
239
|
+ |
defp usage(usage) when is_map(usage) do
|
|
240
|
+ |
input = integer(usage["prompt_tokens"])
|
|
241
|
+ |
output = integer(usage["completion_tokens"])
|
|
242
|
+ |
total = integer(usage["total_tokens"])
|
|
243
|
+ |
|
|
244
|
+ |
if is_nil(input) and is_nil(output) and is_nil(total) do
|
|
245
|
+ |
{:ok, []}
|
|
246
|
+ |
else
|
|
247
|
+ |
normalized =
|
|
248
|
+ |
%{
|
|
249
|
+ |
"input_tokens" => input || 0,
|
|
250
|
+ |
"output_tokens" => output || 0,
|
|
251
|
+ |
"total_tokens" => total || (input || 0) + (output || 0)
|
|
252
|
+ |
}
|
|
253
|
+ |
|
|
254
|
+ |
{:ok, [{:usage, normalized}]}
|
|
255
|
+ |
end
|
|
256
|
+ |
end
|
|
257
|
+ |
|
|
258
|
+ |
defp usage(_usage), do: {:error, :invalid_provider_event}
|
|
259
|
+ |
|
|
260
|
+ |
defp integer(value) when is_integer(value) and value >= 0, do: value
|
|
261
|
+ |
defp integer(_value), do: nil
|
|
262
|
+ |
|
|
263
|
+ |
defp id(%__MODULE__{response_id: nil}), do: "openrouter-response"
|
|
264
|
+ |
defp id(%__MODULE__{response_id: response_id}), do: response_id
|
|
265
|
+ |
|
|
266
|
+ |
defp valid_identifier?(value) when is_binary(value) and byte_size(value) in 1..256,
|
|
267
|
+ |
do: Regex.match?(@identifier_regex, value)
|
|
268
|
+ |
|
|
269
|
+ |
defp valid_identifier?(_value), do: false
|
|
270
|
+ |
|
|
271
|
+ |
defp error_code(error) do
|
|
272
|
+ |
code = error["code"] || error["type"]
|
|
273
|
+ |
|
|
274
|
+ |
cond do
|
|
275
|
+ |
is_binary(code) and byte_size(code) <= 128 and Regex.match?(@tool_name_regex, code) -> code
|
|
276
|
+ |
is_integer(code) -> Integer.to_string(code)
|
|
277
|
+ |
true -> nil
|
|
278
|
+ |
end
|
|
279
|
+ |
end
|
|
280
|
+ |
end
|