|
1
|
+ |
defmodule OpenAgentsWeb.ReleaseController do
|
|
2
|
+ |
@moduledoc """
|
|
3
|
+ |
Serves CLI release artifacts out of the public release bucket.
|
|
4
|
+ |
|
|
5
|
+ |
The binaries are not in the repository and never will be: a 40 MB executable
|
|
6
|
+ |
per platform per version turns a clone into a download. They live in a
|
|
7
|
+ |
world-readable Cloud Storage bucket, and this route is the one durable name
|
|
8
|
+ |
in front of them, so `https://openagents.com/releases/<name>` keeps working
|
|
9
|
+ |
when the bucket behind it is renamed or replaced.
|
|
10
|
+ |
|
|
11
|
+ |
The proxy therefore adds no authority. Every object it serves is already
|
|
12
|
+ |
readable by anyone who knows the bucket URL, so there is nothing here to
|
|
13
|
+ |
authenticate and nothing to withhold. It adds a stable URL, the content type
|
|
14
|
+ |
and cache lifetime the object deserves, and a strict allowlist on the object
|
|
15
|
+ |
name so that the one path segment a caller controls cannot address anything
|
|
16
|
+ |
outside the bucket.
|
|
17
|
+ |
|
|
18
|
+ |
`priv/static/install.sh` is the client this contract exists for, and it is
|
|
19
|
+ |
the reason `Range` is honoured rather than ignored. For an artifact of 16 MiB
|
|
20
|
+ |
or more the installer reads `Content-Length` with a `HEAD`, then fetches
|
|
21
|
+ |
eight ranges concurrently and concatenates them. A ranged request answered
|
|
22
|
+ |
with `200` and the whole body would produce eight complete copies
|
|
23
|
+ |
concatenated into one corrupt binary that still passes as a download, so the
|
|
24
|
+ |
`206` is a correctness requirement rather than an optimization.
|
|
25
|
+ |
"""
|
|
26
|
+ |
|
|
27
|
+ |
use OpenAgentsWeb, :controller
|
|
28
|
+ |
|
|
29
|
+ |
import Plug.Conn
|
|
30
|
+ |
|
|
31
|
+ |
require Logger
|
|
32
|
+ |
|
|
33
|
+ |
@storage_host "https://storage.googleapis.com"
|
|
34
|
+ |
|
|
35
|
+ |
# One segment, and a conservative one. Letters, digits, dot, underscore, and
|
|
36
|
+ |
# hyphen cover every object we publish (`openagents-0.1.0-rc.1-macos-aarch64`,
|
|
37
|
+ |
# `SHA256SUMS-0.1.0-rc.1`, `stable`) and admit no slash, no percent escape,
|
|
38
|
+ |
# and no leading dot. The caller controls this segment and it is interpolated
|
|
39
|
+ |
# into an outbound URL, so the allowlist is the boundary that keeps the
|
|
40
|
+ |
# request inside the bucket.
|
|
41
|
+ |
@name_pattern ~r/\A[A-Za-z0-9][A-Za-z0-9._-]*\z/
|
|
42
|
+ |
@name_max_bytes 128
|
|
43
|
+ |
|
|
44
|
+ |
# A ranged request is passed through verbatim, so its shape is checked here
|
|
45
|
+ |
# rather than trusted. Only the byte-range syntax the installer sends is
|
|
46
|
+ |
# forwarded; anything else is dropped and the whole object is served.
|
|
47
|
+ |
@range_pattern ~r/\Abytes=[0-9,\- ]+\z/
|
|
48
|
+ |
|
|
49
|
+ |
# A version-pinned artifact and its sums file are the same bytes forever, so
|
|
50
|
+ |
# a client that has one never needs to ask again. A channel pointer is the
|
|
51
|
+ |
# opposite: `stable` is a name that moves, and caching it for a year would
|
|
52
|
+ |
# pin every installer that read it to the release it named that day. Getting
|
|
53
|
+ |
# these two the wrong way round is how a fleet ends up stuck on an old
|
|
54
|
+ |
# version with no way to tell it otherwise.
|
|
55
|
+ |
@immutable_cache "public, max-age=31536000, immutable"
|
|
56
|
+ |
@pointer_cache "public, max-age=60"
|
|
57
|
+ |
|
|
58
|
+ |
@streamable_statuses [200, 206]
|
|
59
|
+ |
|
|
60
|
+ |
def show(conn, %{"name" => name}) when is_binary(name) do
|
|
61
|
+ |
if admitted_name?(name) do
|
|
62
|
+ |
proxy(conn, name)
|
|
63
|
+ |
else
|
|
64
|
+ |
not_found(conn)
|
|
65
|
+ |
end
|
|
66
|
+ |
end
|
|
67
|
+ |
|
|
68
|
+ |
# Anything that does not arrive as one string names no object. The bucket is
|
|
69
|
+ |
# flat, and a deeper path never reaches here: `releases` is a reserved slug,
|
|
70
|
+ |
# so the router answers `/releases/a/b` with the site's own 404 page.
|
|
71
|
+ |
def show(conn, _params), do: not_found(conn)
|
|
72
|
+ |
|
|
73
|
+ |
defp admitted_name?(name) do
|
|
74
|
+ |
byte_size(name) <= @name_max_bytes and
|
|
75
|
+ |
Regex.match?(@name_pattern, name) and
|
|
76
|
+ |
not String.contains?(name, "..")
|
|
77
|
+ |
end
|
|
78
|
+ |
|
|
79
|
+ |
defp proxy(conn, name) do
|
|
80
|
+ |
url = object_url(name)
|
|
81
|
+ |
headers = upstream_headers(conn)
|
|
82
|
+ |
|
|
83
|
+ |
if head_request?(conn) do
|
|
84
|
+ |
case Req.head(url, request_options(headers)) do
|
|
85
|
+ |
{:ok, response} -> answer_head(conn, name, response)
|
|
86
|
+ |
{:error, reason} -> unavailable(conn, name, reason)
|
|
87
|
+ |
end
|
|
88
|
+ |
else
|
|
89
|
+ |
case Req.get(url, request_options(headers) ++ [into: :self]) do
|
|
90
|
+ |
{:ok, response} -> answer_get(conn, name, response)
|
|
91
|
+ |
{:error, reason} -> unavailable(conn, name, reason)
|
|
92
|
+ |
end
|
|
93
|
+ |
end
|
|
94
|
+ |
end
|
|
95
|
+ |
|
|
96
|
+ |
defp answer_head(conn, name, %Req.Response{status: status} = response)
|
|
97
|
+ |
when status in @streamable_statuses do
|
|
98
|
+ |
conn
|
|
99
|
+ |
|> put_artifact_headers(name, response)
|
|
100
|
+ |
|> send_resp(status, "")
|
|
101
|
+ |
end
|
|
102
|
+ |
|
|
103
|
+ |
defp answer_head(conn, name, response), do: refuse(conn, name, response)
|
|
104
|
+ |
|
|
105
|
+ |
defp answer_get(conn, name, %Req.Response{status: status, body: body} = response)
|
|
106
|
+ |
when status in @streamable_statuses do
|
|
107
|
+ |
conn =
|
|
108
|
+ |
conn
|
|
109
|
+ |
|> put_artifact_headers(name, response)
|
|
110
|
+ |
|> send_chunked(status)
|
|
111
|
+ |
|
|
112
|
+ |
# `body` is a `Req.Response.Async`, so the artifact crosses this process one
|
|
113
|
+ |
# chunk at a time and a 40 MB binary is never a 40 MB message. A client that
|
|
114
|
+ |
# walks away mid-download closes the socket; halting cancels the upstream
|
|
115
|
+ |
# read rather than draining the rest of the object into a closed connection.
|
|
116
|
+ |
Enum.reduce_while(body, conn, fn data, current_conn ->
|
|
117
|
+ |
case chunk(current_conn, data) do
|
|
118
|
+ |
{:ok, next_conn} -> {:cont, next_conn}
|
|
119
|
+ |
{:error, :closed} -> {:halt, current_conn}
|
|
120
|
+ |
end
|
|
121
|
+ |
end)
|
|
122
|
+ |
end
|
|
123
|
+ |
|
|
124
|
+ |
defp answer_get(conn, name, %Req.Response{body: %Req.Response.Async{}} = response) do
|
|
125
|
+ |
Req.cancel_async_response(response)
|
|
126
|
+ |
refuse(conn, name, response)
|
|
127
|
+ |
end
|
|
128
|
+ |
|
|
129
|
+ |
defp answer_get(conn, name, response), do: refuse(conn, name, response)
|
|
130
|
+ |
|
|
131
|
+ |
defp refuse(conn, _name, %Req.Response{status: 404}), do: not_found(conn)
|
|
132
|
+ |
|
|
133
|
+ |
# The bucket rejected the range the client asked for. Say so, rather than
|
|
134
|
+ |
# reporting it as a fault of ours: the client chose the range and is the only
|
|
135
|
+ |
# party that can choose a different one.
|
|
136
|
+ |
defp refuse(conn, _name, %Req.Response{status: 416}) do
|
|
137
|
+ |
conn
|
|
138
|
+ |
|> put_resp_header("accept-ranges", "bytes")
|
|
139
|
+ |
|> put_resp_header("x-content-type-options", "nosniff")
|
|
140
|
+ |
|> send_resp(416, "")
|
|
141
|
+ |
end
|
|
142
|
+ |
|
|
143
|
+ |
defp refuse(conn, name, %Req.Response{status: status}) do
|
|
144
|
+ |
Logger.warning("release object #{name} answered #{status} from the bucket")
|
|
145
|
+ |
|
|
146
|
+ |
conn
|
|
147
|
+ |
|> put_resp_content_type("text/plain", "utf-8")
|
|
148
|
+ |
|> put_resp_header("x-content-type-options", "nosniff")
|
|
149
|
+ |
|> send_resp(502, "The release store did not answer.\n")
|
|
150
|
+ |
end
|
|
151
|
+ |
|
|
152
|
+ |
defp unavailable(conn, name, reason) do
|
|
153
|
+ |
Logger.warning("release object #{name} could not be read: #{failure_kind(reason)}")
|
|
154
|
+ |
|
|
155
|
+ |
conn
|
|
156
|
+ |
|> put_resp_content_type("text/plain", "utf-8")
|
|
157
|
+ |
|> put_resp_header("x-content-type-options", "nosniff")
|
|
158
|
+ |
|> send_resp(502, "The release store did not answer.\n")
|
|
159
|
+ |
end
|
|
160
|
+ |
|
|
161
|
+ |
# The kind of failure, never the payload that carried it. A log line records
|
|
162
|
+ |
# that the bucket was unreachable and how; the exception itself may quote a
|
|
163
|
+ |
# URL or a header, and `test/openagents/log_safety_test.exs` refuses the shape
|
|
164
|
+ |
# that would put one in a log.
|
|
165
|
+ |
defp failure_kind(%Req.TransportError{reason: reason}), do: "transport #{reason}"
|
|
166
|
+ |
defp failure_kind(%{__struct__: module}), do: inspect(module)
|
|
167
|
+ |
defp failure_kind(_other), do: "unknown"
|
|
168
|
+ |
|
|
169
|
+ |
defp not_found(conn) do
|
|
170
|
+ |
conn
|
|
171
|
+ |
|> put_resp_content_type("text/plain", "utf-8")
|
|
172
|
+ |
|> put_resp_header("x-content-type-options", "nosniff")
|
|
173
|
+ |
|> send_resp(404, "Not found.\n")
|
|
174
|
+ |
end
|
|
175
|
+ |
|
|
176
|
+ |
defp put_artifact_headers(conn, name, response) do
|
|
177
|
+ |
{type, charset} = content_type(name)
|
|
178
|
+ |
|
|
179
|
+ |
conn
|
|
180
|
+ |
|> put_resp_content_type(type, charset)
|
|
181
|
+ |
|> put_resp_header("cache-control", cache_control(name))
|
|
182
|
+ |
|> put_resp_header("x-content-type-options", "nosniff")
|
|
183
|
+ |
|> put_resp_header("accept-ranges", "bytes")
|
|
184
|
+ |
|> copy_upstream_header(response, "content-length")
|
|
185
|
+ |
|> copy_upstream_header(response, "content-range")
|
|
186
|
+ |
end
|
|
187
|
+ |
|
|
188
|
+ |
# `content-length` is set before the response is sent, which is what makes
|
|
189
|
+ |
# `HEAD` answer with the artifact's real size. Bandit streams a
|
|
190
|
+ |
# length-delimited body rather than a chunked one once the length is
|
|
191
|
+ |
# declared, and suppresses the body entirely for a `HEAD`, so the same code
|
|
192
|
+ |
# path serves both the installer's size probe and its download.
|
|
193
|
+ |
defp copy_upstream_header(conn, response, name) do
|
|
194
|
+ |
case Req.Response.get_header(response, name) do
|
|
195
|
+ |
[value | _rest] -> put_resp_header(conn, name, value)
|
|
196
|
+ |
[] -> conn
|
|
197
|
+ |
end
|
|
198
|
+ |
end
|
|
199
|
+ |
|
|
200
|
+ |
# `Plug.Head` rewrites `HEAD` to `GET` before the router sees the request, so
|
|
201
|
+ |
# `conn.method` cannot answer this. The adapter still holds the method the
|
|
202
|
+ |
# client actually sent. An adapter that stops carrying it falls through to
|
|
203
|
+ |
# `false`, which costs a discarded body rather than a wrong answer.
|
|
204
|
+ |
defp head_request?(%Plug.Conn{adapter: {_adapter, %{method: "HEAD"}}}), do: true
|
|
205
|
+ |
defp head_request?(%Plug.Conn{}), do: false
|
|
206
|
+ |
|
|
207
|
+ |
defp content_type("openagents-" <> _rest), do: {"application/octet-stream", nil}
|
|
208
|
+ |
defp content_type(_name), do: {"text/plain", "utf-8"}
|
|
209
|
+ |
|
|
210
|
+ |
defp cache_control("openagents-" <> _rest), do: @immutable_cache
|
|
211
|
+ |
defp cache_control("SHA256SUMS-" <> _rest), do: @immutable_cache
|
|
212
|
+ |
defp cache_control(_name), do: @pointer_cache
|
|
213
|
+ |
|
|
214
|
+ |
defp upstream_headers(conn) do
|
|
215
|
+ |
# Ask the bucket for the stored bytes. Req negotiates compression by
|
|
216
|
+ |
# default, and a compressed transfer would make the `content-length` this
|
|
217
|
+ |
# route forwards describe the encoded body rather than the artifact the
|
|
218
|
+ |
# client is about to checksum.
|
|
219
|
+ |
identity = [{"accept-encoding", "identity"}]
|
|
220
|
+ |
|
|
221
|
+ |
case get_req_header(conn, "range") do
|
|
222
|
+ |
[range | _rest] ->
|
|
223
|
+ |
if Regex.match?(@range_pattern, range), do: [{"range", range} | identity], else: identity
|
|
224
|
+ |
|
|
225
|
+ |
[] ->
|
|
226
|
+ |
identity
|
|
227
|
+ |
end
|
|
228
|
+ |
end
|
|
229
|
+ |
|
|
230
|
+ |
defp request_options(headers) do
|
|
231
|
+ |
Keyword.merge(
|
|
232
|
+ |
[
|
|
233
|
+ |
headers: headers,
|
|
234
|
+ |
decode_body: false,
|
|
235
|
+ |
# The installer already retries by falling back to a serial download,
|
|
236
|
+ |
# and a retry of a partially streamed body would restart it from the
|
|
237
|
+ |
# top into a response that is already open.
|
|
238
|
+ |
retry: false,
|
|
239
|
+ |
receive_timeout: 60_000
|
|
240
|
+ |
],
|
|
241
|
+ |
Application.get_env(:openagents, :releases_request_options, [])
|
|
242
|
+ |
)
|
|
243
|
+ |
end
|
|
244
|
+ |
|
|
245
|
+ |
defp object_url(name) do
|
|
246
|
+ |
bucket = Application.fetch_env!(:openagents, :releases_bucket)
|
|
247
|
+ |
"#{@storage_host}/#{bucket}/#{name}"
|
|
248
|
+ |
end
|
|
249
|
+ |
end
|