OpenAgents

Commit

Allow OAuth-backed repository imports

a80ab5914818 · AtlantisPleb · · parent 5722913cb1b9 

Allow OAuth-backed repository imports

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified lib/openagents/repositories/importer.ex
  • modified test/openagents/repositories/provisioner_test.exs

Diff

2 files changed, +180 -19

lib/openagents/repositories/importer.ex modified +103 -19

@@ -1,6 +1,8 @@

1 1 
defmodule OpenAgents.Repositories.Importer do
2 2 
  @moduledoc "Copies one accepted GitHub ref snapshot into the durable forge WAL."
3 3
4 
  require Logger
5
4 6 
  alias OpenAgents.{Accounts, Audit, GitHubOAuth, Repo}
5 7 
  alias OpenAgents.Forge.{Repos, Sync, WAL}
6 8 
  alias OpenAgents.Repositories.{Repository, RepositoryImport}

@@ -39,18 +41,24 @@ defmodule OpenAgents.Repositories.Importer do

39 41 
          :ok
40 42 
        end
41 43 
      rescue
42 
        _error -> {:error, :import_failed}
44 
        _error ->
45 
          log_stage(repository, running_import, "import", "failed", :import_exception)
46 
          {:error, :import_failed}
43 47 
      catch
44 
        _kind, _reason -> {:error, :import_failed}
48 
        _kind, _reason ->
49 
          log_stage(repository, running_import, "import", "failed", :import_exception)
50 
          {:error, :import_failed}
45 51 
      after
46 52 
        File.rm_rf(temporary_directory)
47 53 
      end
48 54
49 55 
    case result do
50 56 
      :ok ->
57 
        log_stage(repository, running_import, "import", "completed")
51 58 
        :ok
52 59
53 60 
      {:error, reason} ->
61 
        log_stage(repository, running_import, "import", "failed", reason)
54 62 
        mark_failed!(running_import, error_code(reason))
55 63 
        {:error, reason}
56 64 
    end

@@ -83,7 +91,8 @@ defmodule OpenAgents.Repositories.Importer do

83 91 
            repository_import,
84 92 
            source_url,
85 93 
            credential,
86 
            temporary_directory
94 
            temporary_directory,
95 
            options
87 96 
          )
88 97 
        end)
89 98

@@ -108,16 +117,46 @@ defmodule OpenAgents.Repositories.Importer do

108 117 
         repository_import,
109 118 
         source_url,
110 119 
         credential,
111 
         temporary_directory
120 
         temporary_directory,
121 
         options
112 122 
       ) do
113 
    with :ok <- File.mkdir_p(temporary_directory),
114 
         :ok <- File.chmod(temporary_directory, 0o700),
115 
         {:ok, source_repository} <- initialize_source(temporary_directory),
116 
         :ok <- fetch_source(source_repository, source_url, credential, temporary_directory),
117 
         {:ok, refs} <- verify_snapshot(source_repository, repository_import),
118 
         {:ok, payload, format} <- create_payload(source_repository, refs, temporary_directory),
119 
         :ok <- append_import(repository, repository_import, payload, format, refs, 0),
120 
         :ok <- Sync.ensure_fresh(repository.storage_key, repository.default_branch) do
123 
    with :ok <-
124 
           import_stage(repository, repository_import, "prepare_workspace", fn ->
125 
             with :ok <- File.mkdir_p(temporary_directory),
126 
                  :ok <- File.chmod(temporary_directory, 0o700) do
127 
               :ok
128 
             end
129 
           end),
130 
         {:ok, source_repository} <-
131 
           import_stage(repository, repository_import, "initialize_source", fn ->
132 
             initialize_source(temporary_directory)
133 
           end),
134 
         :ok <-
135 
           import_stage(repository, repository_import, "fetch_source", fn ->
136 
             fetch_source(
137 
               source_repository,
138 
               source_url,
139 
               credential,
140 
               temporary_directory,
141 
               options
142 
             )
143 
           end),
144 
         {:ok, refs} <-
145 
           import_stage(repository, repository_import, "verify_snapshot", fn ->
146 
             verify_snapshot(source_repository, repository_import)
147 
           end),
148 
         {:ok, payload, format} <-
149 
           import_stage(repository, repository_import, "create_payload", fn ->
150 
             create_payload(source_repository, refs, temporary_directory)
151 
           end),
152 
         :ok <-
153 
           import_stage(repository, repository_import, "append_wal", fn ->
154 
             append_import(repository, repository_import, payload, format, refs, 0)
155 
           end),
156 
         :ok <-
157 
           import_stage(repository, repository_import, "materialize_cache", fn ->
158 
             Sync.ensure_fresh(repository.storage_key, repository.default_branch)
159 
           end) do
121 160 
      :ok
122 161 
    end
123 162 
  end

@@ -125,9 +164,13 @@ defmodule OpenAgents.Repositories.Importer do

125 164 
  defp source_access(repository, repository_import, options) do
126 165 
    case Keyword.get(options, :source_url) do
127 166 
      source_url when is_binary(source_url) ->
128 
        if Path.type(source_url) == :absolute,
129 
          do: {:ok, source_url, nil},
130 
          else: {:error, :invalid_source_url}
167 
        credential = Keyword.get(options, :source_credential)
168
169 
        cond do
170 
          Path.type(source_url) != :absolute -> {:error, :invalid_source_url}
171 
          is_nil(credential) or is_binary(credential) -> {:ok, source_url, credential}
172 
          true -> {:error, :invalid_source_credential}
173 
        end
131 174
132 175 
      nil ->
133 176 
        with true <-

@@ -148,13 +191,13 @@ defmodule OpenAgents.Repositories.Importer do

148 191 
    end
149 192 
  end
150 193
151 
  defp fetch_source(source_repository, source_url, credential, temporary_directory) do
194 
  defp fetch_source(source_repository, source_url, credential, temporary_directory, options) do
152 195 
    with {:ok, environment} <- credential_environment(credential, temporary_directory) do
196 
      git_runner = Keyword.get(options, :git_runner, &Repos.git/3)
197
153 198 
      args = [
154 199 
        "-c",
155 200 
        "credential.helper=",
156 
        "-c",
157 
        "credential.interactive=never",
158 201 
        "fetch",
159 202 
        "--force",
160 203 
        "--prune",

@@ -164,7 +207,7 @@ defmodule OpenAgents.Repositories.Importer do

164 207 
        "+refs/tags/*:refs/tags/*"
165 208 
      ]
166 209
167 
      case Repos.git(source_repository, args, env: environment) do
210 
      case git_runner.(source_repository, args, env: environment) do
168 211 
        {_output, 0} -> :ok
169 212 
        {_output, _status} -> {:error, :source_fetch_failed}
170 213 
      end

@@ -423,6 +466,47 @@ defmodule OpenAgents.Repositories.Importer do

423 466 
  defp error_code(:import_timeout), do: "import_timeout"
424 467 
  defp error_code(_reason), do: "import_failed"
425 468
469 
  defp import_stage(repository, repository_import, stage, operation) do
470 
    log_stage(repository, repository_import, stage, "started")
471
472 
    case operation.() do
473 
      :ok = result ->
474 
        log_stage(repository, repository_import, stage, "completed")
475 
        result
476
477 
      {:ok, _value} = result ->
478 
        log_stage(repository, repository_import, stage, "completed")
479 
        result
480
481 
      {:ok, _value, _metadata} = result ->
482 
        log_stage(repository, repository_import, stage, "completed")
483 
        result
484
485 
      {:error, reason} = result ->
486 
        log_stage(repository, repository_import, stage, "failed", reason)
487 
        result
488 
    end
489 
  end
490
491 
  defp log_stage(repository, repository_import, stage, state, reason \\ nil) do
492 
    message =
493 
      "repository_import_stage" <>
494 
        " repository_id=#{repository.id}" <>
495 
        " repository_import_id=#{repository_import.id}" <>
496 
        " stage=#{stage}" <>
497 
        " state=#{state}" <>
498 
        diagnostic_error(reason)
499
500 
    if state == "failed", do: Logger.warning(message), else: Logger.info(message)
501 
  end
502
503 
  defp diagnostic_error(nil), do: ""
504
505 
  defp diagnostic_error(reason) when is_atom(reason),
506 
    do: " error_code=#{reason |> Atom.to_string() |> String.slice(0, 80)}"
507
508 
  defp diagnostic_error(_reason), do: " error_code=unexpected_error"
509
426 510 
  defp temporary_directory(import_id) do
427 511 
    root = Application.get_env(:openagents, :repository_import_temp_dir, System.tmp_dir!())
428 512 
    Path.join(root, "openagents-import-#{import_id}-#{System.unique_integer([:positive])}")
test/openagents/repositories/provisioner_test.exs modified +77

@@ -178,6 +178,83 @@ defmodule OpenAgents.Repositories.ProvisionerTest do

178 178 
             "accepted snapshot"
179 179 
  end
180 180
181 
  test "a credential-backed import keeps the askpass helper available during fetch" do
182 
    root = Application.fetch_env!(:openagents, :forge_data_dir) |> Path.dirname()
183 
    source = Path.join(root, "credential-source")
184 
    File.mkdir_p!(source)
185 
    git!(source, ["init", "--initial-branch=main"])
186 
    git!(source, ["config", "user.email", "test@example.com"])
187 
    git!(source, ["config", "user.name", "Import test"])
188 
    File.write!(Path.join(source, "README.md"), "credential boundary\n")
189 
    git!(source, ["add", "README.md"])
190 
    git!(source, ["commit", "-m", "Credential boundary"])
191
192 
    sha = source |> git!(["rev-parse", "HEAD"]) |> String.trim()
193 
    refs = %{"refs/heads/main" => sha}
194 
    user = repository_user_fixture("credential-import-owner")
195
196 
    source_record = %{
197 
      source_repository_id: 502,
198 
      source_owner_id: user.github_id,
199 
      source_full_name: "credential-import-owner/source",
200 
      source_default_branch: "main",
201 
      source_ref_digest: ref_digest(source, refs),
202 
      source_head_sha: sha,
203 
      source_refs: refs,
204 
      source_uses_lfs: false
205 
    }
206
207 
    assert {:ok, repository, _repository_import, :created} =
208 
             Repositories.create_user_import(
209 
               user,
210 
               source_record,
211 
               %{name: "credential-import", default_branch: "main"},
212 
               "credential-import-key"
213 
             )
214
215 
    test_process = self()
216
217 
    git_runner = fn git_directory, arguments, options ->
218 
      environment = Keyword.fetch!(options, :env)
219 
      askpass = List.keyfind!(environment, "GIT_ASKPASS", 0) |> elem(1)
220 
      token_file = List.keyfind!(environment, "OPENAGENTS_GITHUB_TOKEN_FILE", 0) |> elem(1)
221
222 
      {username, 0} =
223 
        System.cmd(askpass, ["Username for 'https://github.com':"],
224 
          env: environment,
225 
          stderr_to_stdout: true
226 
        )
227
228 
      {password, 0} =
229 
        System.cmd(askpass, ["Password for 'https://github.com':"],
230 
          env: environment,
231 
          stderr_to_stdout: true
232 
        )
233
234 
      send(test_process, {
235 
        :credential_fetch,
236 
        arguments,
237 
        String.trim(username) == "x-access-token",
238 
        password == "fixture-credential",
239 
        File.stat!(token_file).mode |> Bitwise.band(0o777)
240 
      })
241
242 
      Repos.git(git_directory, arguments, options)
243 
    end
244
245 
    assert :ok =
246 
             Importer.import(repository,
247 
               source_url: source,
248 
               source_credential: "fixture-credential",
249 
               git_runner: git_runner
250 
             )
251
252 
    assert_receive {:credential_fetch, arguments, true, true, 0o600}
253 
    refute "credential.interactive=never" in arguments
254 
    assert "credential.helper=" in arguments
255 
    assert "fetch" in arguments
256 
  end
257
181 258 
  defp bare_git!(storage_key, args) do
182 259 
    {output, 0} = Repos.git(Repos.bare_path(storage_key), args)
183 260 
    output

This page updates live while a promote is in flight · changelog