Quarantine structurally invalid forge repository caches

5852d4d22a47 · Devin AI · · parent edccc45b8d74

Quarantine structurally invalid forge repository caches

Production node sarah-fleet-2 served HTTP 500 for every git read of one
repository because its bare cache had a HEAD file but no refs directory,
so git refused the directory and Repos.set_default_branch!/2 crashed
with a MatchError on every request. The load balancer alternated between
the two healthy nodes and the broken one, which surfaced as intermittent
500s on push and fetch.

The cache is a disposable projection of the WAL, so ensure_repo_at! now
moves a cache that git refuses as a bare repository aside to
<path>.corrupt-<n> and reinitializes it; WAL replay re-materializes the
refs on the next read.

Also classify the deploy runbook's fleet references and record the two
failure modes diagnosed tonight (invalid_module_name at target build,
and the invalid-cache 500 loop) in the runbook.

Co-Authored-By: Christopher David <chris@openagents.com>
Co-Authored-By
Christopher David <chris@openagents.com>

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified docs/operations/production-deploy-runbook.md
  • modified lib/openagents/forge/repos.ex
  • modified ops/ci/allowed-sarah-references.txt
  • added test/openagents/forge/repos_test.exs

Diff

4 files changed, +66 -0

docs/operations/production-deploy-runbook.md modified +2

@@ -203,3 +203,5 @@ Do not report success without all of the following:

203 203
| Push rejected (non-fast-forward) | Forge `main` advanced | Fetch, rebase your commit, push, restart the gate on the new SHA |
204 204
| `gate.sh --verify` fails in `build-image.sh` | Receipt is for a different SHA | Rerun the gate on the exact candidate |
205 205
| App boots without a new variable | Name missing from `ENV_NAMES` in the startup script | Add the export and the `ENV_NAMES` entry, re-apply metadata |
206
| Target build fails with `invalid_module_name` | A compiled module falls outside the artifact allowlist in `OpenAgents.Forge.BuildArtifact` (`OpenAgents.*`, `OpenAgentsWeb.*`, allowlisted protocol implementations, `Mix.Tasks.Openagents.*`) | Rename the module into an allowlisted namespace, or extend the pattern for a new generated-implementation family. `test/openagents/forge/build_artifact_namespace_test.exs` catches this in precommit |
207
| Intermittent git-over-HTTP `500` on push and fetch | One fleet node has a structurally invalid bare-repository cache (for example `HEAD` present but `refs/` missing), and `Repos.ensure_repo_at!` crashed on it. The load balancer alternates between healthy nodes and the broken one | Since the quarantine fix in `OpenAgents.Forge.Repos`, the node moves the invalid cache to `<repo>.git.corrupt-<n>` and re-materializes from the WAL on the next read. On an older build, find the node whose log shows `fatal: not a git repository` with a `MatchError` from `Repos.set_default_branch!/2`, move the cache directory aside inside the container, and let WAL replay rebuild it (`docs/operations/forge-cache-recovery.md`) |
lib/openagents/forge/repos.ex modified +34

@@ -8,6 +8,8 @@ defmodule OpenAgents.Forge.Repos do

8 8
  ever carry request data.
9 9
  """
10 10
11
  require Logger
12
11 13
  @name_pattern ~r/^[a-z0-9](?:[a-z0-9_-]|\.(?=[a-z0-9])){0,63}$/
12 14
  @storage_key_pattern ~r/\A[A-Za-z0-9][A-Za-z0-9._-]{0,127}\z/
13 15

@@ -56,6 +58,8 @@ defmodule OpenAgents.Forge.Repos do

56 58
57 59
  @doc false
58 60
  def ensure_repo_at!(path, default_branch \\ "main") do
61
    quarantine_invalid_cache!(path)
62
59 63
    unless File.exists?(Path.join(path, "HEAD")) do
60 64
      File.mkdir_p!(path)
61 65
      {_, 0} = git(path, ["init", "--bare", "--initial-branch=#{default_branch}", path])

@@ -67,6 +71,36 @@ defmodule OpenAgents.Forge.Repos do

67 71
    path
68 72
  end
69 73
74
  # The cache is a disposable projection of the WAL. A directory that has a
75
  # HEAD file but that git refuses as a bare repository (for example one whose
76
  # refs directory was lost mid-write) would otherwise crash every read of the
77
  # repository forever. Move it aside so the caller reinitializes an empty
78
  # repository and WAL replay re-materializes every ref.
79
  defp quarantine_invalid_cache!(path) do
80
    if File.exists?(Path.join(path, "HEAD")) and not bare_repository_at?(path) do
81
      suffix = System.unique_integer([:positive, :monotonic])
82
      quarantine_path = path <> ".corrupt-#{suffix}"
83
84
      case File.rename(path, quarantine_path) do
85
        :ok ->
86
          Logger.warning(
87
            "forge_repo_cache_quarantined path=#{path} quarantine=#{quarantine_path}"
88
          )
89
90
          :ok
91
92
        {:error, reason} ->
93
          raise "cannot quarantine invalid repository cache #{path}: #{inspect(reason)}"
94
      end
95
    end
96
97
    :ok
98
  end
99
100
  defp bare_repository_at?(path) do
101
    match?({"true" <> _rest, 0}, git(path, ["rev-parse", "--is-bare-repository"]))
102
  end
103
70 104
  def set_default_branch!(repo, default_branch) do
71 105
    repo |> bare_path() |> set_default_branch_at!(default_branch)
72 106
  end
ops/ci/allowed-sarah-references.txt modified +1

@@ -22,6 +22,7 @@

22 22
# The production cutover retains current provider resource names until the
23 23
# OpenAgents observation window completes.
24 24
^docs/operations/production-cutover\.md:
25
^docs/operations/production-deploy-runbook\.md:
25 26
^ops/production/preflight\.sh:
26 27
27 28
# Current architecture and decisions discuss the Sarah persona boundary.
test/openagents/forge/repos_test.exs added +29

@@ -0,0 +1,29 @@

1
defmodule OpenAgents.Forge.ReposTest do
2
  use ExUnit.Case, async: true
3
4
  alias OpenAgents.Forge.Repos
5
6
  test "a structurally invalid cache is quarantined and reinitialized" do
7
    path =
8
      Path.join(
9
        System.tmp_dir!(),
10
        "forge-repos-#{System.unique_integer([:positive, :monotonic])}.git"
11
      )
12
13
    on_exit(fn ->
14
      File.rm_rf!(path)
15
      Enum.each(Path.wildcard(path <> ".corrupt-*"), &File.rm_rf!/1)
16
    end)
17
18
    assert ^path = Repos.ensure_repo_at!(path)
19
20
    # Simulate the production failure: HEAD survives but the refs directory
21
    # is lost, so git refuses the directory as a repository.
22
    File.rm_rf!(Path.join(path, "refs"))
23
    assert {_output, 128} = Repos.git(path, ["rev-parse", "--is-bare-repository"])
24
25
    assert ^path = Repos.ensure_repo_at!(path)
26
    assert {"true" <> _rest, 0} = Repos.git(path, ["rev-parse", "--is-bare-repository"])
27
    assert [_quarantined] = Path.wildcard(path <> ".corrupt-*")
28
  end
29
end

This page updates live while a promote is in flight · changelog