OpenAgents

Commit

Activate the Forge hot deployment loop

d112a5753469 · AtlantisPleb · · parent bf68015dd0bf 

Activate the Forge hot deployment loop

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified config/config.exs
  • modified infra/staging/templates/fleet-startup.sh.tftpl
  • modified lib/openagents/forge/build_executor.ex
  • modified lib/openagents/forge/build_worker.ex
  • modified lib/openagents/forge/git_http.ex
  • modified lib/openagents/forge/targets.ex
  • modified lib/openagents/network_status.ex
  • modified lib/openagents/runtime_config.ex
  • modified lib/openagents_web/live/network_status_live.ex
  • modified ops/staging/gate-5-profile.sh
  • modified test/openagents/forge/build_worker_test.exs
  • modified test/openagents/forge/git_http_test.exs
  • modified test/openagents/forge/target_lifecycle_test.exs
  • modified test/openagents/network_status_test.exs
  • modified test/openagents/runtime_config_test.exs
  • modified test/openagents_web/live/network_status_live_test.exs

Diff

16 files changed, +241 -18

config/config.exs modified +1 -1

@@ -236,7 +236,7 @@ config :openagents,

236 236 
  forge_build_executor: OpenAgents.Forge.BuildExecutor.Sidecar,
237 237 
  forge_expected_fleet_size: 1,
238 238 
  forge_repos: ["openagents.com"],
239 
  forge_internal_git_url: "http://127.0.0.1:8080/git",
239 
  forge_internal_git_url: "http://127.0.0.1:8080/OpenAgentsInc",
240 240 
  forge_operator_token: nil,
241 241 
  forge_mirror_urls: %{},
242 242 
  forge_wal_adapter: OpenAgents.Forge.WAL.Local,
infra/staging/templates/fleet-startup.sh.tftpl modified +1

@@ -245,6 +245,7 @@ if [ -n "$builder_image" ] || [ -n "$builder_digest" ]; then

245 245 
    --name openagents-builder \
246 246 
    --network host \
247 247 
    --restart always \
248 
    --user 0:65534 \
248 249 
    --env-file /run/openagents/builder.env \
249 250 
    --volume "$state_root/workspace:$state_root/workspace" \
250 251 
    --volume "$state_root/artifacts:$state_root/artifacts" \
lib/openagents/forge/build_executor.ex modified +7 -1

@@ -118,7 +118,13 @@ defmodule OpenAgents.Forge.BuildExecutor.Sidecar do

118 118
119 119 
  @doc "Repository URL with no embedded credential, query, or fragment."
120 120 
  def repo_url(repo) do
121 
    base = Application.get_env(:openagents, :forge_internal_git_url, "http://127.0.0.1:8080/git")
121 
    base =
122 
      Application.get_env(
123 
        :openagents,
124 
        :forge_internal_git_url,
125 
        "http://127.0.0.1:8080/OpenAgentsInc"
126 
      )
127
122 128 
    URI.to_string(URI.parse(base)) <> "/" <> repo <> ".git"
123 129 
  end
124 130
lib/openagents/forge/build_worker.ex modified +47 -5

@@ -22,6 +22,7 @@ defmodule OpenAgents.Forge.BuildWorker do

22 22 
    artifacts = required_env!("OPENAGENTS_FORGE_ARTIFACT_DIR")
23 23 
    builds = required_env!("OPENAGENTS_FORGE_BUILD_DIR")
24 24 
    ensure_builder_paths!(queue, artifacts, builds)
25 
    seed_cache!(builds, System.get_env("OPENAGENTS_FORGE_BUILD_CACHE_SEED_DIR") || File.cwd!())
25 26 
    loop(queue, artifacts, builds)
26 27 
  end
27 28

@@ -177,7 +178,14 @@ defmodule OpenAgents.Forge.BuildWorker do

177 178 
  end
178 179
179 180 
  defp prepare_production_candidate(request, workspace, builds, output, opts) do
181 
    cache = cache_paths(builds)
182
180 183 
    with {:ok, env} <- command_env(opts),
184 
         env =
185 
           [
186 
             {"MIX_BUILD_PATH", cache.build},
187 
             {"MIX_DEPS_PATH", cache.deps}
188 
           ] ++ env,
181 189 
         :ok <- run_ok("git", ["init", "--quiet", workspace], builds, env, output),
182 190 
         :ok <-
183 191 
           run_ok(

@@ -206,11 +214,11 @@ defmodule OpenAgents.Forge.BuildWorker do

206 214 
             [{"MIX_ENV", "prod"} | env],
207 215 
             output
208 216 
           ),
209 
         {:ok, beams} <- read_candidate_beams(workspace) do
217 
         {:ok, beams} <- read_candidate_beams(cache.build) do
210 218 
      toolchain =
211 219 
        BuildArtifact.current_toolchain(
212 220 
          lock_path: Path.join(workspace, "mix.lock"),
213 
          app_file: Path.join(workspace, "_build/prod/lib/openagents/ebin/openagents.app")
221 
          app_file: Path.join(cache.build, "lib/openagents/ebin/openagents.app")
214 222 
        )
215 223
216 224 
      {:ok, beams, toolchain, structural_reasons}

@@ -305,8 +313,8 @@ defmodule OpenAgents.Forge.BuildWorker do

305 313 
    end
306 314 
  end
307 315
308 
  defp read_candidate_beams(workspace) do
309 
    paths = Path.wildcard(Path.join(workspace, "_build/prod/lib/openagents/ebin/*.beam"))
316 
  defp read_candidate_beams(build_path) do
317 
    paths = Path.wildcard(Path.join(build_path, "lib/openagents/ebin/*.beam"))
310 318
311 319 
    if paths == [] do
312 320 
      {:error, :application_beams_missing}

@@ -595,11 +603,45 @@ defmodule OpenAgents.Forge.BuildWorker do

595 603 
          Path.join(queue, "responses"),
596 604 
          Path.join(artifacts, "artifacts"),
597 605 
          Path.join(artifacts, "output"),
598 
          Path.join(builds, "jobs")
606 
          Path.join(builds, "jobs"),
607 
          cache_paths(builds).build,
608 
          cache_paths(builds).deps
599 609 
        ],
600 610 
        do: File.mkdir_p!(path)
601 611 
  end
602 612
613 
  @doc false
614 
  def cache_paths(builds) do
615 
    %{
616 
      build: Path.join([builds, "cache", "_build", "prod"]),
617 
      deps: Path.join([builds, "cache", "deps"])
618 
    }
619 
  end
620
621 
  @doc false
622 
  def seed_cache!(builds, source_root) do
623 
    cache = cache_paths(builds)
624 
    marker = Path.join([builds, "cache", ".seeded"])
625
626 
    unless File.exists?(marker) do
627 
      copy_cache_entries(Path.join([source_root, "_build", "prod"]), cache.build)
628 
      copy_cache_entries(Path.join(source_root, "deps"), cache.deps)
629 
      File.write!(marker, "seeded\n")
630 
    end
631
632 
    :ok
633 
  end
634
635 
  defp copy_cache_entries(source, destination) do
636 
    if File.dir?(source) do
637 
      source
638 
      |> File.ls!()
639 
      |> Enum.each(fn entry ->
640 
        File.cp_r!(Path.join(source, entry), Path.join(destination, entry))
641 
      end)
642 
    end
643 
  end
644
603 645 
  defp required_env!(name) do
604 646 
    case System.get_env(name) do
605 647 
      value when is_binary(value) and value != "" -> value
lib/openagents/forge/git_http.ex modified +16 -1

@@ -261,7 +261,22 @@ defmodule OpenAgents.Forge.GitHTTP do

261 261 
  end
262 262
263 263 
  defp operational_access(repository) do
264 
    if repository.storage_key in Repos.allowed_repos(),
264 
    owner =
265 
      case repository.namespace do
266 
        %{slug: slug} when is_binary(slug) -> slug
267 
        _not_loaded -> Application.get_env(:openagents, :forge_url_owner, "OpenAgentsInc")
268 
      end
269
270 
    configured? =
271 
      Enum.any?(Repos.allowed_repos(), fn allowed ->
272 
        allowed in [
273 
          repository.storage_key,
274 
          repository.name,
275 
          "#{owner}/#{repository.name}"
276 
        ]
277 
      end)
278
279 
    if configured?,
265 280 
      do: :ok,
266 281 
      else: {:error, 404, "unknown repository"}
267 282 
  end
lib/openagents/forge/targets.ex modified +4 -3

@@ -18,7 +18,7 @@ defmodule OpenAgents.Forge.Targets do

18 18 
  alias OpenAgents.Analytics
19 19 
  alias OpenAgents.Forge.BuildReceipt
20 20 
  alias OpenAgents.Forge.DeployReceipt
21 
  alias OpenAgents.Forge.Target
21 
  alias OpenAgents.Forge.{Pushes, Target}
22 22 
  alias OpenAgents.Repo
23 23
24 24 
  @statuses ~w(promoted building built deploying live failed reverted needs_rolling_replace)

@@ -492,8 +492,9 @@ defmodule OpenAgents.Forge.Targets do

492 492 
  end
493 493
494 494 
  defp commit_exists?(repo, sha) do
495 
    OpenAgents.Forge.Sync.ensure_fresh(repo)
496 
    path = OpenAgents.Forge.Repos.bare_path(repo)
495 
    storage_key = Pushes.mirror_storage_key(repo)
496 
    OpenAgents.Forge.Sync.ensure_fresh(storage_key)
497 
    path = OpenAgents.Forge.Repos.bare_path(storage_key)
497 498
498 499 
    case OpenAgents.Forge.Repos.git(path, ["cat-file", "-e", sha <> "^{commit}"]) do
499 500 
      {_, 0} -> true
lib/openagents/network_status.ex modified +5

@@ -144,6 +144,11 @@ defmodule OpenAgents.NetworkStatus do

144 144
145 145 
    %{
146 146 
      "repo" => repo,
147 
      "state" =>
148 
        if(Application.get_env(:openagents, :forge_deploy_lane_enabled, false),
149 
          do: "active",
150 
          else: "off"
151 
        ),
147 152 
      "target" => targets |> List.first() |> public_target(),
148 153 
      "recent_targets" => Enum.map(targets, &public_target/1),
149 154 
      "recent_deploys" => Enum.map(deploys, &public_deploy/1),
lib/openagents/runtime_config.ex modified +13 -4

@@ -627,7 +627,13 @@ defmodule OpenAgents.RuntimeConfig do

627 627 
             :forge_expected_fleet_size,
628 628 
             "must include a canary and peer for deployment"
629 629 
           ),
630 
         :ok <- validate_rolling_provider(rolling_provider, rolling_provider_config, features),
630 
         :ok <-
631 
           validate_rolling_provider(
632 
             rolling_provider,
633 
             rolling_provider_config,
634 
             features,
635 
             environment
636 
           ),
631 637 
         :ok <- validate_forge_secrets(operator_token, durable_required? or features.forge_deploy),
632 638 
         :ok <- validate_forge_paths(settings, durable_required? or features.forge_deploy),
633 639 
         :ok <- validate_wal(settings, durable_required? or features.forge_deploy) do

@@ -635,16 +641,19 @@ defmodule OpenAgents.RuntimeConfig do

635 641 
    end
636 642 
  end
637 643
638 
  defp validate_rolling_provider(_provider, _config, %{forge_deploy: false}), do: :ok
644 
  defp validate_rolling_provider(_provider, _config, %{forge_deploy: false}, _environment),
645 
    do: :ok
646
647 
  defp validate_rolling_provider(nil, _config, %{forge_deploy: true}, :production), do: :ok
639 648
640 
  defp validate_rolling_provider(Gcp, config, %{forge_deploy: true}) do
649 
  defp validate_rolling_provider(Gcp, config, %{forge_deploy: true}, _environment) do
641 650 
    case Gcp.validate_config(config) do
642 651 
      :ok -> :ok
643 652 
      {:error, _reason} -> error(:forge_rolling_provider, "must use an isolated staging project")
644 653 
    end
645 654 
  end
646 655
647 
  defp validate_rolling_provider(_provider, _config, %{forge_deploy: true}),
656 
  defp validate_rolling_provider(_provider, _config, %{forge_deploy: true}, _environment),
648 657 
    do: error(:forge_rolling_provider, "must be the admitted infrastructure provider")
649 658
650 659 
  defp validate_forge_secrets(operator_token, true) do
lib/openagents_web/live/network_status_live.ex modified +10 -1

@@ -103,6 +103,7 @@ defmodule OpenAgentsWeb.NetworkStatusLive do

103 103 
    projection =
104 104 
      projection
105 105 
      |> Map.put_new("forge", %{
106 
        "state" => "off",
106 107 
        "target" => nil,
107 108 
        "recent_targets" => [],
108 109 
        "recent_deploys" => [],

@@ -414,7 +415,15 @@ defmodule OpenAgentsWeb.NetworkStatusLive do

414 415 
          </div>
415 416
416 417 
          <.card id="status-forge">
417 
            <h2>Rapid deploys</h2>
418 
            <div class="status-node__head">
419 
              <h2>Rapid deploys</h2>
420 
              <.badge
421 
                id="status-forge-state"
422 
                variant={if @projection["forge"]["state"] == "active", do: :success, else: :warning}
423 
              >
424 
                {@projection["forge"]["state"]}
425 
              </.badge>
426 
            </div>
418 427 
            <p class="status-forge__intro">
419 428 
              Code moves through the OpenAgents forge: a push is promoted by an
420 429 
              operator, built into just the changed modules, and hot-loaded
ops/staging/gate-5-profile.sh modified +1 -1

@@ -59,7 +59,7 @@ export OPENAGENTS_FORGE_BOOT_RETRY_MIN_MS="1000"

59 59 
export OPENAGENTS_FORGE_BOOT_RETRY_MAX_MS="30000"
60 60 
export OPENAGENTS_FORGE_DATA_DIR="/var/lib/openagents/forge"
61 61 
export OPENAGENTS_FORGE_EXPECTED_FLEET_SIZE="1"
62 
export OPENAGENTS_FORGE_INTERNAL_GIT_URL="http://127.0.0.1:8080/git"
62 
export OPENAGENTS_FORGE_INTERNAL_GIT_URL="http://127.0.0.1:8080/OpenAgentsInc"
63 63 
export OPENAGENTS_FORGE_OPERATOR_TOKEN=""
64 64 
export OPENAGENTS_FORGE_OWNER="OpenAgentsInc"
65 65 
export OPENAGENTS_FORGE_REPOSITORIES="openagents.com"
test/openagents/forge/build_worker_test.exs modified +32

@@ -122,6 +122,38 @@ defmodule OpenAgents.Forge.BuildWorkerTest do

122 122 
    refute File.exists?(old_log)
123 123 
  end
124 124
125 
  test "creates durable Mix cache paths outside disposable job workspaces", context do
126 
    assert :idle =
127 
             BuildWorker.run_once(context.queue, context.artifacts, context.builds,
128 
               build_fun: fn _request, _workspace -> flunk("no request should run") end
129 
             )
130
131 
    cache = BuildWorker.cache_paths(context.builds)
132
133 
    assert File.dir?(cache.build)
134 
    assert File.dir?(cache.deps)
135 
    refute String.starts_with?(cache.build, Path.join(context.builds, "jobs"))
136 
  end
137
138 
  test "seeds the durable Mix cache from the pinned builder image once", context do
139 
    source = Path.join(context.builds, "image")
140 
    File.mkdir_p!(Path.join([source, "_build", "prod", "lib", "sample"]))
141 
    File.mkdir_p!(Path.join([source, "deps", "sample"]))
142 
    File.write!(Path.join([source, "_build", "prod", "lib", "sample", "sample.app"]), "app")
143 
    File.write!(Path.join([source, "deps", "sample", "mix.exs"]), "dep")
144
145 
    assert :idle = BuildWorker.run_once(context.queue, context.artifacts, context.builds)
146 
    assert :ok = BuildWorker.seed_cache!(context.builds, source)
147
148 
    cache = BuildWorker.cache_paths(context.builds)
149 
    assert File.read!(Path.join([cache.build, "lib", "sample", "sample.app"])) == "app"
150 
    assert File.read!(Path.join([cache.deps, "sample", "mix.exs"])) == "dep"
151
152 
    File.write!(Path.join([source, "deps", "sample", "mix.exs"]), "changed")
153 
    assert :ok = BuildWorker.seed_cache!(context.builds, source)
154 
    assert File.read!(Path.join([cache.deps, "sample", "mix.exs"])) == "dep"
155 
  end
156
125 157 
  test "expired request IDs cannot be revived by a later attempt", context do
126 158 
    expired_id = Ecto.UUID.generate()
127 159 
    fresh_id = Ecto.UUID.generate()
test/openagents/forge/git_http_test.exs modified +26

@@ -229,6 +229,32 @@ defmodule OpenAgents.Forge.GitHTTPTest do

229 229 
    assert output =~ "404" or output =~ "not found" or output =~ "unknown"
230 230 
  end
231 231
232 
  test "the operator can access a configured UUID-backed repository", %{
233 
    base: base,
234 
    port: port,
235 
    repository: repository
236 
  } do
237 
    operator_token = "forge_operator_test_token_0123456789"
238 
    previous_token = Application.get_env(:openagents, :forge_operator_token)
239 
    previous_repos = Application.get_env(:openagents, :forge_repos)
240
241 
    Application.put_env(:openagents, :forge_operator_token, operator_token)
242 
    Application.put_env(:openagents, :forge_repos, [repository.name])
243
244 
    on_exit(fn ->
245 
      Application.put_env(:openagents, :forge_operator_token, previous_token)
246 
      Application.put_env(:openagents, :forge_repos, previous_repos)
247 
    end)
248
249 
    url =
250 
      "http://x:#{operator_token}@127.0.0.1:#{port}/git-http-owner/demo.git"
251
252 
    work = seed_clone!(base, url)
253 
    commit_and_push!(work, "operator.txt", "operator\n", "Operator commit")
254
255 
    assert [_receipt] = Forge.recent_pushes(repository.storage_key)
256 
  end
257
232 258 
  test "Git RPC reauthenticates a token after ref advertisement", %{
233 259 
    api_token: api_token,
234 260 
    token: token,
test/openagents/forge/target_lifecycle_test.exs modified +27

@@ -1,5 +1,7 @@

1 1 
defmodule OpenAgents.Forge.TargetLifecycleTest do
2 2 
  use OpenAgents.DataCase, async: false
3 
  import OpenAgents.AccountsFixtures
4
3 5 
  alias OpenAgents.Forge.{BuildReceipt, DeployReceipt, Repos, Targets}
4 6
5 7 
  setup do

@@ -77,6 +79,31 @@ defmodule OpenAgents.Forge.TargetLifecycleTest do

77 79 
    assert {:error, :invalid_sha} = Targets.promote("demo", "not-a-sha!", "operator:test")
78 80 
  end
79 81
82 
  test "promotion resolves a repository name to its UUID storage key" do
83 
    user = repository_user_fixture("target-storage-owner")
84
85 
    {:ok, repository, :created} =
86 
      OpenAgents.Repositories.create_user_repository(
87 
        user,
88 
        %{name: "mapped-target"},
89 
        "target-storage-key"
90 
      )
91
92 
    repository =
93 
      repository
94 
      |> Ecto.Changeset.change(lifecycle_state: "ready", ready_at: DateTime.utc_now())
95 
      |> OpenAgents.Repo.update!()
96
97 
    sha = seeded_commit(repository.storage_key)
98 
    previous_repos = Application.get_env(:openagents, :forge_repos)
99 
    Application.put_env(:openagents, :forge_repos, [repository.name])
100
101 
    on_exit(fn -> Application.put_env(:openagents, :forge_repos, previous_repos) end)
102
103 
    assert {:ok, target} = Targets.promote(repository.name, sha, "operator:test")
104 
    assert target.sha == sha
105 
  end
106
80 107 
  test "advance walks the lifecycle, bounds details, and refuses terminal rows", %{sha: sha} do
81 108 
    {:ok, target} = Targets.promote("demo", sha, "operator:test")
82 109
test/openagents/network_status_test.exs modified +13

@@ -141,6 +141,7 @@ defmodule OpenAgents.NetworkStatusTest do

141 141
142 142 
      forge = NetworkStatus.projection(refresh: true)["forge"]
143 143
144 
      assert forge["state"] in ["active", "off"]
144 145 
      assert forge["target"] == nil
145 146 
      assert forge["recent_deploys"] == []
146 147 
      assert forge["loop"] == %{"last_ms" => nil, "median_ms" => nil}

@@ -187,6 +188,18 @@ defmodule OpenAgents.NetworkStatusTest do

187 188 
      assert forge["loop"]["median_ms"] == 13_242
188 189 
    end
189 190
191 
    test "reports whether the preferred deploy loop is active" do
192 
      previous = Application.get_env(:openagents, :forge_deploy_lane_enabled)
193
194 
      on_exit(fn -> restore_env(:forge_deploy_lane_enabled, previous) end)
195
196 
      Application.put_env(:openagents, :forge_deploy_lane_enabled, true)
197 
      assert NetworkStatus.projection(refresh: true)["forge"]["state"] == "active"
198
199 
      Application.put_env(:openagents, :forge_deploy_lane_enabled, false)
200 
      assert NetworkStatus.projection(refresh: true)["forge"]["state"] == "off"
201 
    end
202
190 203 
    test "loop metrics: last is newest, median over live deploys only" do
191 204 
      sha = String.duplicate("b", 40)
192 205
test/openagents/runtime_config_test.exs modified +36

@@ -126,6 +126,42 @@ defmodule OpenAgents.RuntimeConfigTest do

126 126 
             |> RuntimeConfig.validate()
127 127 
  end
128 128
129 
  test "production admits direct deployment without an automatic rolling provider" do
130 
    settings =
131 
      staging_settings()
132 
      |> Map.merge(%{
133 
        runtime_environment: :production,
134 
        staging_gate: 16,
135 
        production_deploy_enabled: true,
136 
        build_revision: String.duplicate("a", 40),
137 
        image_digest: "sha256:" <> String.duplicate("b", 64),
138 
        forge_enabled: true,
139 
        forge_deploy_lane_enabled: true,
140 
        forge_boot_converge_enabled: false,
141 
        forge_expected_fleet_size: 3,
142 
        forge_operator_token: "production-operator-token",
143 
        forge_rolling_provider: nil,
144 
        forge_wal_dir: "/var/lib/openagents/forge-wal",
145 
        github_token_encryption_key_id: "production-2026-08",
146 
        ra_enabled: true,
147 
        dns_cluster_query: "openagents.fleet.internal",
148 
        distribution: [
149 
          enabled: true,
150 
          node_configured: true,
151 
          cookie_configured: true,
152 
          port_min: 9_100,
153 
          port_max: 9_115
154 
        ]
155 
      })
156 
      |> Map.put(OpenAgentsWeb.Endpoint,
157 
        url: [host: "openagents.com", port: 443, scheme: "https"],
158 
        check_origin: ["https://openagents.com"]
159 
      )
160 
      |> update_oauth(:redirect_uri, "https://openagents.com/auth/github/callback")
161
162 
    assert {:ok, _config} = RuntimeConfig.validate(settings)
163 
  end
164
129 165 
  test "enabled OpenAI features require the centralized provider secret" do
130 166 
    settings =
131 167 
      staging_settings()
test/openagents_web/live/network_status_live_test.exs modified +2 -1

@@ -127,9 +127,10 @@ defmodule OpenAgentsWeb.NetworkStatusLiveTest do

127 127 
          else: Application.delete_env(:openagents, :forge_repos)
128 128 
      end)
129 129
130 
      {:ok, _view, html} = live(conn, ~p"/status")
130 
      {:ok, view, html} = live(conn, ~p"/status")
131 131
132 132 
      assert html =~ "Rapid deploys"
133 
      assert has_element?(view, "#status-forge-state")
133 134 
      assert html =~ "No deploys yet"
134 135 
    end
135 136

This page updates live while a promote is in flight · changelog