OpenAgents

Commit

Connect relup packages and SSH mirrors

da8dce1b3c64 · AtlantisPleb · · parent b01780f305dc 

Connect relup packages and SSH mirrors

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified Dockerfile
  • modified lib/openagents/forge/pushes.ex
  • modified lib/openagents/forge/relup_node.ex
  • added lib/openagents/forge/relup_package.ex
  • modified lib/openagents/runtime_config.ex
  • modified ops/forge/package-relup.sh
  • modified test/openagents/forge/mirror_watch_test.exs
  • modified test/openagents/forge/relup_deployment_test.exs
  • modified test/openagents/forge/relup_node_test.exs
  • added test/openagents/forge/relup_package_test.exs
  • modified test/openagents/runtime_config_test.exs

Diff

11 files changed, +292 -8

Dockerfile modified +1 -1

@@ -163,7 +163,7 @@ RUN sed -i \

163 163 
      /etc/apt/sources.list.d/debian.sources \
164 164 
  && printf 'Acquire::Check-Valid-Until "false";\n' > /etc/apt/apt.conf.d/99snapshot \
165 165 
  && apt-get update \
166 
  && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates curl git unzip fontconfig librsvg2-bin \
166 
  && apt-get install -y --no-install-recommends libstdc++6 openssl libncurses6 locales ca-certificates curl git openssh-client unzip fontconfig librsvg2-bin \
167 167 
  && rm -rf /var/lib/apt/lists/*
168 168
169 169 
# Server-side image rendering (Open Graph cards) reads system fonts through
lib/openagents/forge/pushes.ex modified +9 -2

@@ -268,9 +268,11 @@ defmodule OpenAgents.Forge.Pushes do

268 268
269 269 
      true ->
270 270 
        case URI.new(url) do
271 
          {:ok, %URI{scheme: scheme, host: host, userinfo: nil}}
271 
          {:ok, %URI{scheme: scheme, host: host, userinfo: userinfo}}
272 272 
          when scheme in ["http", "https", "git", "ssh"] and is_binary(host) ->
273 
            url
273 
            if is_nil(userinfo) or (scheme == "ssh" and clean_ssh_username?(userinfo)),
274 
              do: url,
275 
              else: nil
274 276
275 277 
          _credentialed_or_invalid ->
276 278 
            nil

@@ -279,4 +281,9 @@ defmodule OpenAgents.Forge.Pushes do

279 281 
  end
280 282
281 283 
  defp clean_mirror_url(_invalid), do: nil
284
285 
  defp clean_ssh_username?(userinfo) do
286 
    is_binary(userinfo) and userinfo != "" and
287 
      not String.contains?(userinfo, [":", "@", "/", "\\"])
288 
  end
282 289 
end
lib/openagents/forge/relup_node.ex modified +2

@@ -141,6 +141,7 @@ defmodule OpenAgents.Forge.RelupNode do

141 141 
    with true <-
142 142 
           release_status(request.to_version, opts) == expected_status or
143 143 
             {:error, :release_status_mismatch},
144 
         true <- build_revision(opts) == request.sha or {:error, :revision_mismatch},
144 145 
         %{"ready" => true} <- health_report(opts),
145 146 
         %{schema_version: schema} <- state_snapshot(opts),
146 147 
         true <- schema == request.to_state_version or {:error, :state_version_mismatch} do

@@ -325,6 +326,7 @@ defmodule OpenAgents.Forge.RelupNode do

325 326 
  end
326 327
327 328 
  defp health_report(opts), do: Keyword.get(opts, :health, &OpenAgents.Cluster.local_report/0).()
329 
  defp build_revision(opts), do: Keyword.get(opts, :revision, &OpenAgents.BuildInfo.revision/0).()
328 330 
  defp state_snapshot(opts), do: Keyword.get(opts, :state, &ReleaseState.snapshot/0).()
329 331
330 332 
  defp release_basename(request), do: "#{request.release_name}-#{request.to_version}"
lib/openagents/forge/relup_package.ex added +155

@@ -0,0 +1,155 @@

1 
defmodule OpenAgents.Forge.RelupPackage do
2 
  @moduledoc """
3 
  Validates a digest-addressed relup package and deploys it to the current fleet.
4
5 
  The package must match the running revision and the target node operating
6 
  system and architecture. This prevents an operator from passing a macOS or
7 
  ARM release to Linux/AMD64 nodes.
8 
  """
9
10 
  alias OpenAgents.Forge.RelupDeployment
11
12 
  @schema "openagents.relup-package.v1"
13 
  @sha_pattern ~r/\A[0-9a-f]{40}\z/
14 
  @digest_pattern ~r/\A[0-9a-f]{64}\z/
15 
  @version_pattern ~r/\A[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?\z/
16 
  @maximum_manifest_bytes 65_536
17 
  @maximum_artifact_bytes 536_870_912
18
19 
  @doc "Validate and load the target release from a relup package directory."
20 
  def load(directory, opts \\ [])
21
22 
  def load(directory, opts) when is_binary(directory) do
23 
    with {:ok, manifest} <- read_manifest(directory),
24 
         :ok <- validate_manifest(manifest, opts),
25 
         {:ok, artifact_bytes} <- read_artifact(directory, manifest),
26 
         :ok <- verify_artifact(artifact_bytes, manifest["to_artifact_digest"]) do
27 
      nodes =
28 
        Keyword.get_lazy(opts, :expected_nodes, &OpenAgents.Cluster.members/0) |> Enum.sort()
29
30 
      {:ok,
31 
       %{
32 
         sha: manifest["to_revision"],
33 
         release_name: manifest["release_name"],
34 
         from_version: manifest["from_version"],
35 
         to_version: manifest["to_version"],
36 
         from_state_version: manifest["from_state_version"],
37 
         to_state_version: manifest["to_state_version"],
38 
         artifact_bytes: artifact_bytes,
39 
         artifact_digest: manifest["to_artifact_digest"],
40 
         expected_nodes: nodes,
41 
         expected_fleet_size: length(nodes)
42 
       }}
43 
    end
44 
  end
45
46 
  def load(_directory, _opts), do: {:error, :invalid_package_directory}
47
48 
  @doc "Validate a relup package and deploy it one node at a time."
49 
  def deploy(directory, opts \\ []) do
50 
    with {:ok, request} <- load(directory, opts) do
51 
      deployment_opts =
52 
        Keyword.drop(opts, [:current_revision, :expected_nodes, :system_architecture])
53
54 
      RelupDeployment.run(request, deployment_opts)
55 
    end
56 
  end
57
58 
  defp read_manifest(directory) do
59 
    path = Path.join(directory, "package.json")
60
61 
    with {:ok, stat} <- File.stat(path),
62 
         true <- stat.type == :regular and stat.size <= @maximum_manifest_bytes,
63 
         {:ok, bytes} <- File.read(path),
64 
         {:ok, manifest} <- Jason.decode(bytes) do
65 
      {:ok, manifest}
66 
    else
67 
      {:error, :enoent} -> {:error, :missing_package_manifest}
68 
      {:error, %Jason.DecodeError{}} -> {:error, :invalid_package_manifest_json}
69 
      {:error, reason} -> {:error, {:package_manifest_unreadable, safe_code(reason)}}
70 
      false -> {:error, :invalid_package_manifest_file}
71 
    end
72 
  end
73
74 
  defp validate_manifest(manifest, opts) when is_map(manifest) do
75 
    current_revision =
76 
      Keyword.get_lazy(opts, :current_revision, fn -> OpenAgents.BuildInfo.revision() end)
77
78 
    cond do
79 
      manifest["schema"] != @schema ->
80 
        {:error, :wrong_package_schema}
81
82 
      manifest["release_name"] != "openagents" ->
83 
        {:error, :invalid_release_name}
84
85 
      not sha?(manifest["from_revision"]) ->
86 
        {:error, :invalid_from_revision}
87
88 
      not sha?(manifest["to_revision"]) ->
89 
        {:error, :invalid_to_revision}
90
91 
      manifest["from_revision"] != current_revision ->
92 
        {:error, :running_revision_mismatch}
93
94 
      not version?(manifest["from_version"]) ->
95 
        {:error, :invalid_from_version}
96
97 
      not version?(manifest["to_version"]) ->
98 
        {:error, :invalid_to_version}
99
100 
      manifest["from_version"] == manifest["to_version"] ->
101 
        {:error, :degenerate_version_transition}
102
103 
      manifest["from_state_version"] not in [1, 2] ->
104 
        {:error, :invalid_from_state_version}
105
106 
      manifest["to_state_version"] not in [1, 2] ->
107 
        {:error, :invalid_to_state_version}
108
109 
      manifest["to_state_version"] < manifest["from_state_version"] ->
110 
        {:error, :state_version_regression}
111
112 
      not digest?(manifest["to_artifact_digest"]) ->
113 
        {:error, :invalid_target_artifact_digest}
114
115 
      manifest["target_system"] != expected_system(opts) ->
116 
        {:error, :target_system_mismatch}
117
118 
      true ->
119 
        :ok
120 
    end
121 
  end
122
123 
  defp validate_manifest(_manifest, _opts), do: {:error, :invalid_package_manifest}
124
125 
  defp read_artifact(directory, manifest) do
126 
    filename = "openagents-#{manifest["to_version"]}.tar.gz"
127 
    path = Path.join(directory, filename)
128
129 
    with {:ok, stat} <- File.stat(path),
130 
         true <- stat.type == :regular and stat.size <= @maximum_artifact_bytes,
131 
         {:ok, bytes} <- File.read(path) do
132 
      {:ok, bytes}
133 
    else
134 
      {:error, :enoent} -> {:error, :missing_target_artifact}
135 
      {:error, reason} -> {:error, {:target_artifact_unreadable, safe_code(reason)}}
136 
      false -> {:error, :invalid_target_artifact_file}
137 
    end
138 
  end
139
140 
  defp verify_artifact(bytes, expected_digest) do
141 
    if digest(bytes) == expected_digest, do: :ok, else: {:error, :target_artifact_digest_mismatch}
142 
  end
143
144 
  defp expected_system(opts) do
145 
    Keyword.get_lazy(opts, :system_architecture, fn ->
146 
      :erlang.system_info(:system_architecture) |> to_string()
147 
    end)
148 
  end
149
150 
  defp sha?(value), do: is_binary(value) and Regex.match?(@sha_pattern, value)
151 
  defp digest?(value), do: is_binary(value) and Regex.match?(@digest_pattern, value)
152 
  defp version?(value), do: is_binary(value) and Regex.match?(@version_pattern, value)
153 
  defp digest(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower)
154 
  defp safe_code(reason), do: OpenAgents.OperationalLog.code(reason)
155 
end
lib/openagents/runtime_config.ex modified +13 -5

@@ -956,16 +956,24 @@ defmodule OpenAgents.RuntimeConfig do

956 956 
        true
957 957
958 958 
      true ->
959 
        match?(
960 
          {:ok, %URI{scheme: scheme, host: host, userinfo: nil}}
961 
          when scheme in ["http", "https", "git", "ssh"] and is_binary(host),
962 
          URI.new(url)
963 
        )
959 
        case URI.new(url) do
960 
          {:ok, %URI{scheme: scheme, host: host, userinfo: userinfo}}
961 
          when scheme in ["http", "https", "git", "ssh"] and is_binary(host) ->
962 
            is_nil(userinfo) or (scheme == "ssh" and clean_ssh_username?(userinfo))
963
964 
          _invalid ->
965 
            false
966 
        end
964 967 
    end
965 968 
  end
966 969
967 970 
  defp clean_mirror_url?(_url), do: false
968 971
972 
  defp clean_ssh_username?(userinfo) do
973 
    is_binary(userinfo) and userinfo != "" and
974 
      not String.contains?(userinfo, [":", "@", "/", "\\"])
975 
  end
976
969 977 
  defp clean_service_url?(url) do
970 978 
    case URI.new(url) do
971 979 
      {:ok, %URI{scheme: "https", host: host, userinfo: nil}} when is_binary(host) -> true
ops/forge/package-relup.sh modified +7

@@ -72,6 +72,9 @@ fi

72 72
73 73 
from_sha=$(git -C "$repo_root" rev-parse --verify "${from_rev}^{commit}")
74 74 
to_sha=$(git -C "$repo_root" rev-parse --verify "${to_rev}^{commit}")
75 
target_system=$(erl -noshell -eval 'io:format("~s", [erlang:system_info(system_architecture)]), halt().' 2>/dev/null)
76
77 
[ -n "$target_system" ] || { echo "could not determine the release target system" >&2; exit 1; }
75 78
76 79 
build_root=$(mktemp -d "${TMPDIR:-/tmp}/openagents-relup-package.XXXXXX")
77 80 
assets_digested=0

@@ -113,6 +116,7 @@ env -u RELUP_FROM -u RELUP_TO -u OPENAGENTS_RELUP_PATH \

113 116 
  OPENAGENTS_RELEASE_PATH="$build_root/from/release" \
114 117 
  OPENAGENTS_RELEASE_VSN="$from_version" \
115 118 
  OPENAGENTS_RELUP_STATE_VERSION="$from_state" \
119 
  OPENAGENTS_BUILD_REVISION="$from_sha" \
116 120 
  sh -c 'cd "$1" && mix do compile --force --warnings-as-errors + release --overwrite' sh "$build_root/from"
117 121
118 122 
# The appup derives its instruction list from these two module sets, so every

@@ -129,6 +133,7 @@ env -u OPENAGENTS_RELUP_PATH \

129 133 
  OPENAGENTS_RELEASE_PATH="$build_root/to/release" \
130 134 
  OPENAGENTS_RELEASE_VSN="$to_version" \
131 135 
  OPENAGENTS_RELUP_STATE_VERSION="$to_state" \
136 
  OPENAGENTS_BUILD_REVISION="$to_sha" \
132 137 
  RELUP_FROM="$from_version" \
133 138 
  RELUP_TO="$to_version" \
134 139 
  RELUP_FROM_EBIN="$from_ebin" \

@@ -156,6 +161,7 @@ env \

156 161 
  OPENAGENTS_RELEASE_VSN="$to_version" \
157 162 
  OPENAGENTS_RELUP_PATH="$build_root/relup" \
158 163 
  OPENAGENTS_RELUP_STATE_VERSION="$to_state" \
164 
  OPENAGENTS_BUILD_REVISION="$to_sha" \
159 165 
  RELUP_FROM="$from_version" \
160 166 
  RELUP_TO="$to_version" \
161 167 
  RELUP_FROM_EBIN="$from_ebin" \

@@ -190,6 +196,7 @@ cat >"$publish_root/package.json" <<EOF

190 196 
  "to_version": "$to_version",
191 197 
  "from_state_version": $from_state,
192 198 
  "to_state_version": $to_state,
199 
  "target_system": "$target_system",
193 200 
  "from_artifact_digest": "$from_digest",
194 201 
  "to_artifact_digest": "$to_digest",
195 202 
  "relup_digest": "$relup_digest"
test/openagents/forge/mirror_watch_test.exs modified +7

@@ -149,6 +149,13 @@ defmodule OpenAgents.Forge.MirrorWatchTest do

149 149 
      Application.put_env(:openagents, :forge_mirror_urls, %{"openagents.com" => url})
150 150 
      assert OpenAgents.Forge.Pushes.mirror_url("openagents.com") == nil
151 151 
    end
152
153 
    Application.put_env(:openagents, :forge_mirror_urls, %{
154 
      "openagents.com" => "ssh://git@github.com/OpenAgentsInc/openagents.com.git"
155 
    })
156
157 
    assert OpenAgents.Forge.Pushes.mirror_url("openagents.com") ==
158 
             "ssh://git@github.com/OpenAgentsInc/openagents.com.git"
152 159 
  end
153 160
154 161 
  defp incident_count do
test/openagents/forge/relup_deployment_test.exs modified +1

@@ -221,6 +221,7 @@ defmodule OpenAgents.Forge.RelupDeploymentTest do

221 221 
        release_root: root,
222 222 
        release_handler: ReleaseHandler,
223 223 
        generate_config: fn _version -> :ok end,
224 
        revision: fn -> request().sha end,
224 225 
        health: &unready_once/0,
225 226 
        state: fn -> ReleaseState.snapshot(state) end
226 227 
      ]
test/openagents/forge/relup_node_test.exs modified +6

@@ -6,6 +6,8 @@ defmodule OpenAgents.Forge.RelupNodeTest do

6 6 
  alias OpenAgents.ReleaseState.State
7 7 
  alias OpenAgents.Test.ReleaseHandler
8 8
9 
  @sha String.duplicate("a", 40)
10
9 11 
  describe "a schema 1 to schema 2 upgrade" do
10 12 
    setup do
11 13 
      start_supervised!(

@@ -17,6 +19,7 @@ defmodule OpenAgents.Forge.RelupNodeTest do

17 19 
      artifact = "immutable release artifact"
18 20
19 21 
      request = %{
22 
        sha: @sha,
20 23 
        release_name: "openagents",
21 24 
        from_version: "0.1.0",
22 25 
        to_version: "0.2.0",

@@ -30,6 +33,7 @@ defmodule OpenAgents.Forge.RelupNodeTest do

30 33 
        release_root: root,
31 34 
        release_handler: ReleaseHandler,
32 35 
        generate_config: fn _version -> :ok end,
36 
        revision: fn -> @sha end,
33 37 
        health: fn -> %{"ready" => true} end,
34 38 
        state: fn -> %{schema_version: 2} end
35 39 
      ]

@@ -113,6 +117,7 @@ defmodule OpenAgents.Forge.RelupNodeTest do

113 117 
      artifact = "immutable 0.3.0 artifact"
114 118
115 119 
      request = %{
120 
        sha: @sha,
116 121 
        release_name: "openagents",
117 122 
        from_version: "0.2.0",
118 123 
        to_version: "0.3.0",

@@ -126,6 +131,7 @@ defmodule OpenAgents.Forge.RelupNodeTest do

126 131 
        release_root: release_root(),
127 132 
        release_handler: ReleaseHandler,
128 133 
        generate_config: fn _version -> :ok end,
134 
        revision: fn -> @sha end,
129 135 
        health: fn -> %{"ready" => true} end,
130 136 
        state: fn -> ReleaseState.snapshot(state) end
131 137 
      ]
test/openagents/forge/relup_package_test.exs added +84

@@ -0,0 +1,84 @@

1 
defmodule OpenAgents.Forge.RelupPackageTest do
2 
  use ExUnit.Case, async: true
3
4 
  alias OpenAgents.Forge.RelupPackage
5
6 
  @from_sha String.duplicate("a", 40)
7 
  @to_sha String.duplicate("b", 40)
8 
  @system "x86_64-pc-linux-gnu"
9
10 
  test "loads a package bound to the running revision and target system" do
11 
    directory = package_directory()
12 
    on_exit(fn -> File.rm_rf!(directory) end)
13
14 
    assert {:ok, request} =
15 
             RelupPackage.load(directory,
16 
               current_revision: @from_sha,
17 
               system_architecture: @system,
18 
               expected_nodes: [:second@local, :first@local]
19 
             )
20
21 
    assert request.sha == @to_sha
22 
    assert request.from_version == "0.2.0"
23 
    assert request.to_version == "0.3.0"
24 
    assert request.expected_nodes == [:first@local, :second@local]
25 
    assert request.artifact_bytes == "target release"
26 
  end
27
28 
  test "refuses a package for another revision, platform, or artifact" do
29 
    directory = package_directory()
30 
    on_exit(fn -> File.rm_rf!(directory) end)
31
32 
    assert {:error, :running_revision_mismatch} =
33 
             RelupPackage.load(directory,
34 
               current_revision: String.duplicate("c", 40),
35 
               system_architecture: @system
36 
             )
37
38 
    assert {:error, :target_system_mismatch} =
39 
             RelupPackage.load(directory,
40 
               current_revision: @from_sha,
41 
               system_architecture: "aarch64-apple-darwin"
42 
             )
43
44 
    File.write!(Path.join(directory, "openagents-0.3.0.tar.gz"), "changed")
45
46 
    assert {:error, :target_artifact_digest_mismatch} =
47 
             RelupPackage.load(directory,
48 
               current_revision: @from_sha,
49 
               system_architecture: @system
50 
             )
51 
  end
52
53 
  defp package_directory do
54 
    directory =
55 
      Path.join(
56 
        System.tmp_dir!(),
57 
        "openagents-relup-package-#{System.unique_integer([:positive])}"
58 
      )
59
60 
    File.mkdir_p!(directory)
61 
    artifact = "target release"
62 
    File.write!(Path.join(directory, "openagents-0.3.0.tar.gz"), artifact)
63
64 
    manifest = %{
65 
      "schema" => "openagents.relup-package.v1",
66 
      "release_name" => "openagents",
67 
      "from_revision" => @from_sha,
68 
      "to_revision" => @to_sha,
69 
      "from_version" => "0.2.0",
70 
      "to_version" => "0.3.0",
71 
      "from_state_version" => 2,
72 
      "to_state_version" => 2,
73 
      "from_artifact_digest" => String.duplicate("d", 64),
74 
      "to_artifact_digest" => sha256(artifact),
75 
      "relup_digest" => String.duplicate("e", 64),
76 
      "target_system" => @system
77 
    }
78
79 
    File.write!(Path.join(directory, "package.json"), Jason.encode!(manifest))
80 
    directory
81 
  end
82
83 
  defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower)
84 
end
test/openagents/runtime_config_test.exs modified +7

@@ -229,6 +229,13 @@ defmodule OpenAgents.RuntimeConfigTest do

229 229 
             staging_settings()
230 230 
             |> Map.put(:forge_mirror_urls, %{"openagents.com" => "/var/lib/openagents/mirror"})
231 231 
             |> RuntimeConfig.validate()
232
233 
    assert {:ok, _config} =
234 
             staging_settings()
235 
             |> Map.put(:forge_mirror_urls, %{
236 
               "openagents.com" => "ssh://git@github.com/OpenAgentsInc/openagents.com.git"
237 
             })
238 
             |> RuntimeConfig.validate()
232 239 
  end
233 240
234 241 
  test "startup refuses an empty tool catalog when tools are enabled" do

This page updates live while a promote is in flight · changelog