Import a public repository owned by another account as an upstream mirror.

7d21158a1d11 · AtlantisPleb · · parent 9be6dddf7b0c

Import a public repository owned by another account as an upstream mirror.

Closes #194

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-Authored-By
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Closes
#194

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

pushed
by user · WAL seq 280 · 2026-08-24T05:10:44.760692Z

Changed files

  • modified lib/openagents/forge/git_http.ex
  • modified lib/openagents/github.ex
  • modified lib/openagents/repositories.ex
  • modified lib/openagents/repositories/github_projection.ex
  • modified lib/openagents/repositories/importer.ex
  • modified lib/openagents/repositories/repository.ex
  • modified lib/openagents_web/controllers/repository_import_controller.ex
  • modified lib/openagents_web/controllers/repository_json.ex
  • modified lib/openagents_web/live/code_repo_live.ex
  • modified lib/openagents_web/live/repository_index_live.ex
  • modified priv/migration_lineages/prior-2026-08-19.json
  • added priv/repo/migrations/20260824043735_add_upstream_mirror_to_repositories.exs
  • modified test/openagents/forge/git_http_test.exs
  • added test/openagents/repositories/upstream_mirror_test.exs
  • modified test/openagents_web/controllers/repository_import_controller_test.exs
  • modified test/openagents_web/live/code_live_test.exs

Diff

16 files changed, +1135 -54

lib/openagents/forge/git_http.ex modified +20 -1

@@ -255,6 +255,25 @@ defmodule OpenAgents.Forge.GitHTTP do

255 255
    end
256 256
  end
257 257
258
  # An upstream mirror is one-way by construction. There is no push to the
259
  # upstream it names, so accepting a push here would produce a copy that
260
  # claims an origin it has silently diverged from — worse than a copy that
261
  # will not move.
262
  #
263
  # The refusal sits on the Git plane rather than in a controller because this
264
  # is where a push actually lands: `advertise/4` and `receive_pack/3` are the
265
  # only two callers, both reach it through this clause, and the clause is
266
  # ahead of every principal. An operator token, a paired computer's grant,
267
  # and an assignment credential are refused the same way an account is, so
268
  # there is no principal for which the mirror is writable.
269
  defp authorize(_conn, %{upstream_url: upstream_url}, :write) when is_binary(upstream_url) do
270
    {:error, 403,
271
     "this repository is an upstream mirror of #{upstream_url} and accepts no pushes. " <>
272
       "A mirror is one-way: it carries the upstream's history and its license, " <>
273
       "and nothing here can be pushed back to the upstream. " <>
274
       "Create your own repository if you want to push."}
275
  end
276
258 277
  defp authorize(conn, repository, :write) do
259 278
    case conn.assigns[:forge_principal] do
260 279
      nil ->

@@ -357,7 +376,7 @@ defmodule OpenAgents.Forge.GitHTTP do

357 376
       [{"www-authenticate", ~s(Basic realm="openagents-forge")}]}
358 377
359 378
  defp send_git_error(conn, {:error, status, message}) do
360
    conn |> send_resp(status, message) |> halt()
379
    conn |> put_resp_content_type("text/plain") |> send_resp(status, message) |> halt()
361 380
  end
362 381
363 382
  defp send_git_error(conn, {:error, %OpenAgents.Forge.SyncError{}}) do
lib/openagents/github.ex modified +12

@@ -347,6 +347,7 @@ defmodule OpenAgents.GitHub do

347 347
         "archived" => repository["archived"] == true,
348 348
         "default_branch" => bounded_string(default_branch, 255),
349 349
         "language" => bounded_string(repository["language"], 60),
350
         "license" => bounded_string(license_spdx_id(repository["license"]), 60),
350 351
         "pushed_at" => bounded_string(repository["pushed_at"], 32),
351 352
         "size_kb" => bounded_size(repository["size"]),
352 353
         "permissions" => permissions,

@@ -357,6 +358,17 @@ defmodule OpenAgents.GitHub do

357 358
358 359
  defp project_repository(_body), do: {:error, :github_response_invalid}
359 360
361
  # GitHub answers `null` for a repository with no license file, and a
362
  # `spdx_id` of `NOASSERTION` for one whose license it could not identify.
363
  # Both are "no license this projection can name", and both must stay
364
  # distinguishable from a real identifier downstream, so neither is
365
  # smuggled through as a string here.
366
  defp license_spdx_id(%{"spdx_id" => spdx_id})
367
       when is_binary(spdx_id) and spdx_id != "" and spdx_id != "NOASSERTION",
368
       do: spdx_id
369
370
  defp license_spdx_id(_license), do: nil
371
360 372
  defp project_organization_membership(%{
361 373
         "state" => "active",
362 374
         "role" => role,
lib/openagents/repositories.ex modified +132 -7

@@ -322,6 +322,120 @@ defmodule OpenAgents.Repositories do

322 322
    end
323 323
  end
324 324
325
  @doc """
326
  Bring in a public repository this account does not own, as an upstream
327
  mirror in the account's own namespace.
328
329
  This is not the import gate relaxed. `create_user_import/4` still refuses a
330
  source owned by anyone else, because an import claims the snapshot as this
331
  account's own repository and says nothing about where it came from. A mirror
332
  makes the opposite claim: it names the upstream on every surface that shows
333
  it, carries the upstream's license or records that there is none, and
334
  refuses every push, because there is no path back to the upstream and a
335
  copy that silently diverges from the source it names is worse than one that
336
  will not move.
337
338
  The source must be public. A private repository this account can read
339
  through its own GitHub grant is not something the forge may republish under
340
  a mirror's provenance, so `:source_repository_not_public` refuses it before
341
  any row exists.
342
  """
343
  def create_user_mirror(%User{} = user, source, attrs, idempotency_key)
344
      when is_map(source) and is_map(attrs) and is_binary(idempotency_key) do
345
    with {:ok, namespace} <- ensure_user_namespace(user),
346
         {:ok, mirror} <- mirror_provenance(source) do
347
      create_repository_transaction(
348
        user,
349
        namespace,
350
        attrs,
351
        source,
352
        "github_mirror",
353
        "github_import",
354
        idempotency_key,
355
        mirror
356
      )
357
    end
358
  end
359
360
  @doc "Bring in a public repository as an upstream mirror in an organization namespace."
361
  def create_organization_mirror(
362
        %User{} = user,
363
        %Namespace{kind: "organization"} = namespace,
364
        source,
365
        attrs,
366
        idempotency_key
367
      )
368
      when is_map(source) and is_map(attrs) and is_binary(idempotency_key) do
369
    with {:ok, mirror} <- mirror_provenance(source) do
370
      create_repository_transaction(
371
        user,
372
        namespace,
373
        attrs,
374
        source,
375
        "github_mirror",
376
        "github_import",
377
        idempotency_key,
378
        mirror
379
      )
380
    end
381
  end
382
383
  @doc """
384
  Whether this repository is an upstream mirror.
385
386
  One predicate, so the Git plane, the API projection, and the page all decide
387
  mirror-ness the same way and a fifth reader cannot invent a sixth rule.
388
  """
389
  def mirror?(%Repository{} = repository), do: Repository.mirror?(repository)
390
391
  defp mirror_provenance(source) do
392
    full_name = fetch_attr!(source, :source_full_name)
393
    license = fetch_attr(source, :source_license)
394
395
    cond do
396
      fetch_attr(source, :source_public) != true ->
397
        {:error, :source_repository_not_public}
398
399
      not is_binary(full_name) ->
400
        {:error, :invalid_import}
401
402
      true ->
403
        {:ok, {"https://github.com/" <> full_name, normalize_license(license)}}
404
    end
405
  end
406
407
  # An upstream with no license is not an upstream whose license is unknown.
408
  # GitHub answers `null` for a repository with no license file and
409
  # `NOASSERTION` for one whose license it cannot identify; both become the
410
  # literal "none", which the surfaces render as a statement rather than a
411
  # blank.
412
  defp normalize_license(license)
413
       when is_binary(license) and license != "" and license != "NOASSERTION",
414
       do: String.slice(license, 0, 60)
415
416
  defp normalize_license(_license), do: "none"
417
418
  defp repository_creation_changeset(repository, attrs, namespace, user_id, kind, nil),
419
    do: Repository.creation_changeset(repository, attrs, namespace, user_id, kind)
420
421
  defp repository_creation_changeset(
422
         repository,
423
         attrs,
424
         namespace,
425
         user_id,
426
         _kind,
427
         {upstream_url, license}
428
       ),
429
       do:
430
         Repository.mirror_creation_changeset(
431
           repository,
432
           attrs,
433
           namespace,
434
           user_id,
435
           upstream_url,
436
           license
437
         )
438
325 439
  def list_visible_repositories(%User{} = user) do
326 440
    Repo.all(
327 441
      from repository in readable_by(Repository, user),

@@ -1162,12 +1276,14 @@ defmodule OpenAgents.Repositories do

1162 1276
         source,
1163 1277
         operation,
1164 1278
         provisioning_kind,
1165
         idempotency_key
1279
         idempotency_key,
1280
         mirror \\ nil
1166 1281
       ) do
1167 1282
    normalized_request = %{
1168 1283
      namespace_id: namespace.id,
1169 1284
      repository: normalize_repository_attrs(attrs),
1170
      source: normalize_source(source)
1285
      source: normalize_source(source),
1286
      mirror: mirror
1171 1287
    }
1172 1288
1173 1289
    request_digest = digest(normalized_request)

@@ -1190,7 +1306,8 @@ defmodule OpenAgents.Repositories do

1190 1306
              operation,
1191 1307
              provisioning_kind,
1192 1308
              idempotency_key,
1193
              request_digest
1309
              request_digest,
1310
              normalized_request.mirror
1194 1311
            )
1195 1312
        end
1196 1313
      end)

@@ -1222,13 +1339,14 @@ defmodule OpenAgents.Repositories do

1222 1339
         operation,
1223 1340
         provisioning_kind,
1224 1341
         idempotency_key,
1225
         request_digest
1342
         request_digest,
1343
         mirror
1226 1344
       ) do
1227 1345
    lock_and_validate_quota!(namespace.id)
1228 1346
1229 1347
    repository =
1230 1348
      %Repository{}
1231
      |> Repository.creation_changeset(attrs, namespace, user.id, provisioning_kind)
1349
      |> repository_creation_changeset(attrs, namespace, user.id, provisioning_kind, mirror)
1232 1350
      |> Repo.insert!()
1233 1351
1234 1352
    Audit.record!("repository.created", {:user, user.id}, "repository", repository.id,

@@ -1278,12 +1396,16 @@ defmodule OpenAgents.Repositories do

1278 1396
        created_import
1279 1397
      end
1280 1398
1399
    # The outbox names the executor, not the caller's intent. A mirror and an
1400
    # import are copied by the same provisioning path, so both queue
1401
    # `github_import` here while the idempotency record keeps them apart: one
1402
    # key must never replay an import as a mirror or the reverse.
1281 1403
    outbox =
1282 1404
      %ProvisioningOutbox{}
1283 1405
      |> ProvisioningOutbox.changeset(
1284 1406
        repository.id,
1285 1407
        repository_import && repository_import.id,
1286
        operation
1408
        outbox_operation(provisioning_kind)
1287 1409
      )
1288 1410
      |> Repo.insert!()
1289 1411

@@ -1318,7 +1440,10 @@ defmodule OpenAgents.Repositories do

1318 1440
    end
1319 1441
  end
1320 1442
1321
  defp replay_result(request, "github_import") do
1443
  defp outbox_operation("empty"), do: "create"
1444
  defp outbox_operation("github_import"), do: "github_import"
1445
1446
  defp replay_result(request, operation) when operation in ["github_import", "github_mirror"] do
1322 1447
    repository =
1323 1448
      Repository
1324 1449
      |> Repo.get!(request.repository_id)
lib/openagents/repositories/github_projection.ex modified +3 -1

@@ -86,7 +86,9 @@ defmodule OpenAgents.Repositories.GitHubProjection do

86 86
         source_ref_digest: references["digest"],
87 87
         source_head_sha: refs["refs/heads/#{default_branch}"],
88 88
         source_refs: refs,
89
         source_uses_lfs: lfs_warning?
89
         source_uses_lfs: lfs_warning?,
90
         source_public: repository["private"] == false,
91
         source_license: repository["license"]
90 92
       }, repository}
91 93
    else
92 94
      {:error, :github_token_missing} -> {:error, :github_connection_required}
lib/openagents/repositories/importer.ex modified +37 -14

@@ -136,6 +136,7 @@ defmodule OpenAgents.Repositories.Importer do

136 136
         :ok <-
137 137
           import_stage(repository, repository_import, "fetch_source", fn ->
138 138
             fetch_source(
139
               repository,
139 140
               source_repository,
140 141
               source_url,
141 142
               credential,

@@ -220,23 +221,33 @@ defmodule OpenAgents.Repositories.Importer do

220 221
    end
221 222
  end
222 223
223
  defp fetch_source(source_repository, source_url, credential, temporary_directory, options) do
224
  defp fetch_source(
225
         repository,
226
         source_repository,
227
         source_url,
228
         credential,
229
         temporary_directory,
230
         options
231
       ) do
224 232
    with {:ok, environment} <- credential_environment(credential, temporary_directory) do
225 233
      git_runner = Keyword.get(options, :git_runner, &Repos.git/3)
226 234
227
      args = [
228
        "-c",
229
        "credential.helper=",
230
        "fetch",
231
        "--force",
232
        "--prune",
233
        "--depth=1",
234
        "--no-tags",
235
        "--no-recurse-submodules",
236
        source_url,
237
        "+refs/heads/*:refs/heads/*",
238
        "+refs/tags/*:refs/tags/*"
239
      ]
235
      args =
236
        [
237
          "-c",
238
          "credential.helper=",
239
          "fetch",
240
          "--force",
241
          "--prune"
242
        ] ++
243
          depth_arguments(repository) ++
244
          [
245
            "--no-tags",
246
            "--no-recurse-submodules",
247
            source_url,
248
            "+refs/heads/*:refs/heads/*",
249
            "+refs/tags/*:refs/tags/*"
250
          ]
240 251
241 252
      case git_runner.(source_repository, args, env: environment) do
242 253
        {_output, 0} ->

@@ -250,6 +261,18 @@ defmodule OpenAgents.Repositories.Importer do

250 261
    end
251 262
  end
252 263
264
  # An owned import takes the tip and states its boundary; the account can push
265
  # the rest of the history it already has. A mirror has no such recourse: no
266
  # push reaches it, and there is no path back to the upstream, so a boundary
267
  # here would be permanent and every clone of the mirror would carry it. The
268
  # copy is therefore full, and `shallow_boundaries/1` then records the empty
269
  # boundary set the fetch actually produced rather than assuming it.
270
  #
271
  # Both branches record what happened. Neither leaves the WAL silent about
272
  # its boundary, which is the failure #179 found.
273
  defp depth_arguments(%Repository{upstream_url: url}) when is_binary(url), do: []
274
  defp depth_arguments(_repository), do: ["--depth=1"]
275
253 276
  defp credential_environment(nil, _temporary_directory),
254 277
    do: {:ok, [{"GIT_TERMINAL_PROMPT", "0"}]}
255 278
lib/openagents/repositories/repository.ex modified +39

@@ -25,6 +25,8 @@ defmodule OpenAgents.Repositories.Repository do

25 25
    field :provision_error_code, :string
26 26
    field :storage_key, :string
27 27
    field :ready_at, :utc_datetime_usec
28
    field :upstream_url, :string
29
    field :upstream_license, :string
28 30
29 31
    belongs_to :namespace, OpenAgents.Repositories.Namespace
30 32
    belongs_to :created_by_user, OpenAgents.Accounts.User

@@ -81,6 +83,7 @@ defmodule OpenAgents.Repositories.Repository do

81 83
    |> check_constraint(:lifecycle_state, name: :repositories_lifecycle_state_check)
82 84
    |> check_constraint(:provisioning_kind, name: :repositories_provisioning_kind_check)
83 85
    |> check_constraint(:ready_at, name: :repositories_ready_state_check)
86
    |> check_constraint(:upstream_url, name: :repositories_upstream_mirror_check)
84 87
  end
85 88
86 89
  def creation_changeset(repository, attrs, namespace, created_by_user_id, provisioning_kind) do

@@ -103,6 +106,42 @@ defmodule OpenAgents.Repositories.Repository do

103 106
    |> put_change(:created_by_user_id, created_by_user_id)
104 107
  end
105 108
109
  @doc """
110
  Build an upstream mirror: a repository whose content comes from a public
111
  source this forge does not own.
112
113
  The upstream fields are deliberately absent from `changeset/2`'s `cast`
114
  list and are written only here, through `put_change/3`. Caller-supplied
115
  attributes therefore cannot make a repository claim an upstream, and the
116
  ordinary import path cannot produce a mirror however its attributes are
117
  shaped. A mirror exists only because a caller asked this function for one.
118
119
  `license` is the upstream's SPDX identifier, or the literal `"none"` when
120
  the upstream publishes no license. `nil` is not accepted: the database
121
  constraint pairs the two columns, and a mirror that says nothing about its
122
  license is exactly the state this refuses to represent.
123
  """
124
  def mirror_creation_changeset(
125
        repository,
126
        attrs,
127
        namespace,
128
        created_by_user_id,
129
        upstream_url,
130
        license
131
      )
132
      when is_binary(upstream_url) and is_binary(license) do
133
    repository
134
    |> creation_changeset(attrs, namespace, created_by_user_id, "github_import")
135
    |> put_change(:upstream_url, upstream_url)
136
    |> put_change(:upstream_license, license)
137
    |> validate_length(:upstream_url, min: 12, max: 500)
138
    |> validate_format(:upstream_url, ~r{\Ahttps://[a-z0-9.-]+/[^\s]+\z})
139
    |> validate_length(:upstream_license, min: 1, max: 60)
140
  end
141
142
  @doc "Whether this repository is an upstream mirror rather than an owned repository."
143
  def mirror?(%__MODULE__{upstream_url: url}), do: is_binary(url)
144
106 145
  defp normalize_creation_name(attrs) do
107 146
    case Map.get(attrs, :name, Map.get(attrs, "name")) do
108 147
      name when is_binary(name) ->
lib/openagents_web/controllers/repository_import_controller.ex modified +99 -26

@@ -1,5 +1,18 @@

1 1
defmodule OpenAgentsWeb.RepositoryImportController do
2
  @moduledoc "Accepts and reports one-time GitHub repository imports."
2
  @moduledoc """
3
  Accepts and reports one-time GitHub repository copies.
4
5
  Two kinds arrive here and they are not the same claim. An **import** copies a
6
  repository this account owns and becomes this account's own repository. An
7
  **upstream mirror** (`"mirror": true`) copies a public repository this
8
  account does not own, names the upstream, carries its license, and refuses
9
  every push.
10
11
  Which one you get is never inferred. A foreign source owner without
12
  `"mirror": true` is still refused, and the refusal names the source owner
13
  and the field that would admit it, because becoming a mirror is a decision
14
  the caller makes rather than one this controller makes for them.
15
  """
3 16
4 17
  use OpenAgentsWeb, :controller
5 18

@@ -10,11 +23,13 @@ defmodule OpenAgentsWeb.RepositoryImportController do

10 23
  def create_user(conn, params) do
11 24
    with {:ok, idempotency_key} <- idempotency_key(conn),
12 25
         {:ok, full_name} <- source_repository(params),
26
         {:ok, mirror?} <- mirror_requested(params),
13 27
         {:ok, source, github_repository} <-
14 28
           GitHubProjection.import_source(conn.assigns.current_user, full_name),
15 29
         {:ok, attrs} <- import_attrs(params, github_repository),
16 30
         {:ok, repository, repository_import, replay_state} <-
17
           Repositories.create_user_import(
31
           create_user_copy(
32
             mirror?,
18 33
             conn.assigns.current_user,
19 34
             source,
20 35
             attrs,

@@ -22,20 +37,38 @@ defmodule OpenAgentsWeb.RepositoryImportController do

22 37
           ) do
23 38
      render_import(conn, repository, repository_import, replay_state)
24 39
    else
25
      {:error, reason} -> render_error(conn, reason)
40
      {:error, reason} -> render_error(conn, reason, params)
26 41
    end
27 42
  end
28 43
44
  defp create_user_copy(true, user, source, attrs, idempotency_key),
45
    do: Repositories.create_user_mirror(user, source, attrs, idempotency_key)
46
47
  defp create_user_copy(false, user, source, attrs, idempotency_key),
48
    do: Repositories.create_user_import(user, source, attrs, idempotency_key)
49
50
  defp create_organization_copy(true, user, namespace, source, attrs, idempotency_key),
51
    do: Repositories.create_organization_mirror(user, namespace, source, attrs, idempotency_key)
52
53
  defp create_organization_copy(false, user, namespace, source, attrs, idempotency_key),
54
    do: Repositories.create_organization_import(user, namespace, source, attrs, idempotency_key)
55
56
  defp mirror_requested(%{"mirror" => mirror}) when is_boolean(mirror), do: {:ok, mirror}
57
  defp mirror_requested(%{"mirror" => _invalid}), do: {:error, :invalid_import}
58
  defp mirror_requested(_params), do: {:ok, false}
59
29 60
  def create_organization(conn, %{"org" => org} = params) do
30 61
    with {:ok, idempotency_key} <- idempotency_key(conn),
31 62
         {:ok, namespace} <-
32 63
           GitHubProjection.authorized_organization(conn.assigns.current_user, org),
33 64
         {:ok, full_name} <- source_repository(params),
65
         {:ok, mirror?} <- mirror_requested(params),
34 66
         {:ok, source, github_repository} <-
35 67
           GitHubProjection.import_source(conn.assigns.current_user, full_name),
36 68
         {:ok, attrs} <- import_attrs(params, github_repository),
37 69
         {:ok, repository, repository_import, replay_state} <-
38
           Repositories.create_organization_import(
70
           create_organization_copy(
71
             mirror?,
39 72
             conn.assigns.current_user,
40 73
             namespace,
41 74
             source,

@@ -44,7 +77,7 @@ defmodule OpenAgentsWeb.RepositoryImportController do

44 77
           ) do
45 78
      render_import(conn, repository, repository_import, replay_state)
46 79
    else
47
      {:error, reason} -> render_error(conn, reason)
80
      {:error, reason} -> render_error(conn, reason, params)
48 81
    end
49 82
  end
50 83

@@ -63,7 +96,7 @@ defmodule OpenAgentsWeb.RepositoryImportController do

63 96
      "import" => RepositoryImportJSON.import(repository_import)
64 97
    })
65 98
  rescue
66
    Ecto.NoResultsError -> render_error(conn, :not_found)
99
    Ecto.NoResultsError -> render_error(conn, :not_found, %{})
67 100
  end
68 101
69 102
  defp render_import(conn, repository, repository_import, replay_state) do

@@ -124,28 +157,55 @@ defmodule OpenAgentsWeb.RepositoryImportController do

124 157
    end
125 158
  end
126 159
127
  defp render_error(conn, :not_found),
128
    do: error(conn, :not_found, "not_found", "Repository import not found")
160
  # The refusal used to say "Source namespace is not eligible", which names
161
  # nothing: the caller cannot tell whether the destination namespace or the
162
  # source owner failed, and the destination is usually their own account. The
163
  # source owner is what failed, so the source owner is what the message says,
164
  # along with the one field that admits a public repository owned by someone
165
  # else.
166
  defp render_error(conn, :source_namespace_mismatch, params) do
167
    error(
168
      conn,
169
      :forbidden,
170
      "source_namespace_mismatch",
171
      "The source repository #{source_name(params)} is owned by another GitHub account, " <>
172
        "so it cannot be imported as your own repository. " <>
173
        "If it is public, send \"mirror\": true to bring it in as an upstream mirror, " <>
174
        "which names the upstream, carries its license, and accepts no pushes.",
175
      %{"source" => source_name(params), "destination" => "eligible"}
176
    )
177
  end
178
179
  defp render_error(conn, :source_repository_not_public, params),
180
    do:
181
      error(
182
        conn,
183
        :forbidden,
184
        "source_repository_not_public",
185
        "The source repository #{source_name(params)} is not public, " <>
186
          "so it cannot be brought in as an upstream mirror.",
187
        %{"source" => source_name(params)}
188
      )
129 189
130
  defp render_error(conn, :source_namespace_mismatch),
131
    do: error(conn, :forbidden, "source_namespace_mismatch", "Source namespace is not eligible")
190
  defp render_error(conn, :not_found, _params),
191
    do: error(conn, :not_found, "not_found", "Repository import not found")
132 192
133
  defp render_error(conn, :namespace_not_allowed),
193
  defp render_error(conn, :namespace_not_allowed, _params),
134 194
    do: error(conn, :forbidden, "namespace_not_allowed", "Namespace is not eligible")
135 195
136
  defp render_error(conn, :github_connection_required),
196
  defp render_error(conn, :github_connection_required, _params),
137 197
    do: error(conn, :forbidden, "github_connection_required", "Connect GitHub before importing")
138 198
139
  defp render_error(conn, :github_scope_required),
199
  defp render_error(conn, :github_scope_required, _params),
140 200
    do: error(conn, :forbidden, "github_scope_required", "Reconnect GitHub with required access")
141 201
142
  defp render_error(conn, :source_repository_not_accessible),
202
  defp render_error(conn, :source_repository_not_accessible, _params),
143 203
    do: error(conn, :forbidden, "source_repository_not_accessible", "Source is not accessible")
144 204
145
  defp render_error(conn, :idempotency_conflict),
205
  defp render_error(conn, :idempotency_conflict, _params),
146 206
    do: error(conn, :conflict, "idempotency_conflict", "The idempotency key is already in use")
147 207
148
  defp render_error(conn, :repository_quota_exceeded),
208
  defp render_error(conn, :repository_quota_exceeded, _params),
149 209
    do:
150 210
      error(
151 211
        conn,

@@ -154,13 +214,13 @@ defmodule OpenAgentsWeb.RepositoryImportController do

154 214
        "The namespace repository quota is exhausted"
155 215
      )
156 216
157
  defp render_error(conn, :invalid_idempotency_key),
217
  defp render_error(conn, :invalid_idempotency_key, _params),
158 218
    do: error(conn, :bad_request, "invalid_idempotency_key", "Provide one Idempotency-Key header")
159 219
160
  defp render_error(conn, %Ecto.Changeset{}),
220
  defp render_error(conn, %Ecto.Changeset{}, _params),
161 221
    do: error(conn, :unprocessable_entity, "invalid_import", "Repository import is invalid")
162 222
163
  defp render_error(conn, reason)
223
  defp render_error(conn, reason, _params)
164 224
       when reason in [
165 225
              :github_unavailable,
166 226
              :github_request_failed,

@@ -168,18 +228,31 @@ defmodule OpenAgentsWeb.RepositoryImportController do

168 228
            ],
169 229
       do: error(conn, :service_unavailable, "github_unavailable", "GitHub is unavailable")
170 230
171
  defp render_error(conn, _reason),
231
  defp render_error(conn, _reason, _params),
172 232
    do: error(conn, :unprocessable_entity, "invalid_import", "Repository import is invalid")
173 233
174
  defp error(conn, status, code, message) do
175
    conn
176
    |> put_status(status)
177
    |> put_resp_header("cache-control", "no-store")
178
    |> json(%{
234
  defp source_name(%{"source" => %{"repository" => full_name}}) when is_binary(full_name),
235
    do: bounded_source_name(full_name)
236
237
  defp source_name(_params), do: "requested by this call"
238
239
  defp bounded_source_name(full_name) do
240
    full_name
241
    |> String.replace(~r/[^A-Za-z0-9._\/-]/, "")
242
    |> String.slice(0, 140)
243
  end
244
245
  defp error(conn, status, code, message, failed \\ nil) do
246
    body = %{
179 247
      "code" => code,
180 248
      "message" => message,
181 249
      "request_id" => List.first(get_resp_header(conn, "x-request-id"))
182
    })
250
    }
251
252
    conn
253
    |> put_status(status)
254
    |> put_resp_header("cache-control", "no-store")
255
    |> json(if failed, do: Map.put(body, "failed", failed), else: body)
183 256
  end
184 257
185 258
  defp base_url(conn) do
lib/openagents_web/controllers/repository_json.ex modified +25 -1

@@ -25,15 +25,39 @@ defmodule OpenAgentsWeb.RepositoryJSON do

25 25
      "clone_url" => base_url <> "/#{owner}/#{repository.name}.git",
26 26
      "html_url" => base_url <> "/#{owner}/#{repository.name}",
27 27
      "permissions" => permissions,
28
      "mirror" => Repository.mirror?(repository),
29
      "upstream" => upstream(repository),
28 30
      "created_at" => DateTime.to_iso8601(repository.inserted_at),
29 31
      "updated_at" => DateTime.to_iso8601(repository.updated_at)
30 32
    }
31 33
  end
32 34
35
  # A mirror names its upstream in the response body, not only in the
36
  # database. `"mirror" => false` and `"upstream" => nil` are published for
37
  # every owned repository too, so a reader learns the distinction exists
38
  # from any repository rather than only from a mirror.
39
  #
40
  # `"license"` here is what the upstream published at the moment the copy was
41
  # taken, or the literal `"none"`. It is a record of the upstream's terms,
42
  # never a claim about the copy.
43
  defp upstream(%Repository{upstream_url: nil}), do: nil
44
45
  defp upstream(%Repository{} = repository) do
46
    %{
47
      "url" => repository.upstream_url,
48
      "license" => repository.upstream_license,
49
      "direction" => "one_way",
50
      "accepts_pushes" => false
51
    }
52
  end
53
54
  # `"push"` is a claim about what the Git plane will accept, so a mirror
55
  # reports `false` for every role. An owner who reads `true` here and then
56
  # has the push refused has been told two different things by one system.
33 57
  def permissions(repository, role) do
34 58
    %{
35 59
      "admin" => role in ~w(owner maintainer),
36
      "push" => role in ~w(owner maintainer contributor),
60
      "push" => role in ~w(owner maintainer contributor) and not Repository.mirror?(repository),
37 61
      "pull" => repository.visibility == "public" or not is_nil(role)
38 62
    }
39 63
  end
lib/openagents_web/live/code_repo_live.ex modified +52 -2

@@ -55,6 +55,7 @@ defmodule OpenAgentsWeb.CodeRepoLive do

55 55
     |> assign(:page_title, "#{repository.name} · code")
56 56
     |> assign(:repository, repository)
57 57
     |> assign(:repository_import, repository.repository_import)
58
     |> assign(:mirror?, Repositories.mirror?(repository))
58 59
     |> assign(:repo, repository.name)
59 60
     |> assign(:owner, repository.namespace.slug)
60 61
     |> assign(:base, base)

@@ -203,6 +204,11 @@ defmodule OpenAgentsWeb.CodeRepoLive do

203 204
    end
204 205
  end
205 206
207
  # "none" is the recorded absence of a license, not a missing value, and it
208
  # is rendered as a sentence rather than as the bare token the database holds.
209
  defp upstream_license_label(%{upstream_license: "none"}), do: "No license found upstream"
210
  defp upstream_license_label(%{upstream_license: license}), do: license
211
206 212
  defp short(sha), do: String.slice(sha, 0, 12)
207 213
208 214
  # The tab carries a count only when there is something to count, the way the

@@ -458,7 +464,10 @@ defmodule OpenAgentsWeb.CodeRepoLive do

458 464
          </.card>
459 465
460 466
          <:about>
461
            <.repo_about description={@repository.description}>
467
            <.repo_about
468
              description={@repository.description}
469
              license={if @mirror?, do: upstream_license_label(@repository)}
470
            >
462 471
              <%!-- The file that is actually there, under the name it actually
463 472
              has. A fixed `README.md` is a guess, and a repository whose readme
464 473
              is named anything else gets a rail link to a 404. --%>

@@ -482,8 +491,49 @@ defmodule OpenAgentsWeb.CodeRepoLive do

482 491
            <%!-- REPOSITORY-001: an import freezes one authorized ref map and
483 492
            schedules no later synchronization. Keep that provenance beside
484 493
            the repository metadata instead of interrupting the code tree. --%>
494
            <%!-- REPOSITORY-001: a mirror is not an owned repository, and the
495
            page says so where a reader is looking at the repository rather
496
            than only in the API. The upstream is named, its license travels
497
            or its absence is stated, and the one-way rule is written down
498
            beside the clone URL that would otherwise imply a push. --%>
499
            <section
500
              :if={@mirror?}
501
              id="repo-upstream-mirror"
502
              class="repo-import-provenance"
503
              aria-labelledby="repo-upstream-mirror-title"
504
            >
505
              <h2 id="repo-upstream-mirror-title">Upstream mirror</h2>
506
              <dl>
507
                <div>
508
                  <dt>Upstream</dt>
509
                  <dd>
510
                    <.link href={@repository.upstream_url} rel="noopener nofollow">
511
                      {@repository.upstream_url}
512
                    </.link>
513
                  </dd>
514
                </div>
515
                <div>
516
                  <dt>Upstream license</dt>
517
                  <dd>{upstream_license_label(@repository)}</dd>
518
                </div>
519
                <div :if={@repository_import && @repository_import.source_head_sha}>
520
                  <dt>Snapshot</dt>
521
                  <dd><code>{short(@repository_import.source_head_sha)}</code></dd>
522
                </div>
523
                <div :if={@repository_import}>
524
                  <dt>State</dt>
525
                  <dd>{@repository_import.state}</dd>
526
                </div>
527
              </dl>
528
              <p>
529
                OpenAgents does not own this repository. It is a one-way copy of the
530
                upstream above, taken once, and it accepts no pushes. Contribute
531
                to the upstream instead.
532
              </p>
533
            </section>
534
485 535
            <section
486
              :if={@repository_import}
536
              :if={@repository_import && not @mirror?}
487 537
              id="repo-import-provenance"
488 538
              class="repo-import-provenance"
489 539
              aria-labelledby="repo-import-provenance-title"
lib/openagents_web/live/repository_index_live.ex modified +18 -1

@@ -237,6 +237,11 @@ defmodule OpenAgentsWeb.RepositoryIndexLive do

237 237
          <span class="font-normal text-muted-foreground">{@repository.namespace.slug}/</span>{@repository.name}
238 238
        </.link>
239 239
        <.badge variant={:dim}>{@repository.visibility}</.badge>
240
        <%!-- A mirror in a list of owned repositories is the easiest place to
241
        misread one for the other, so the row says which it is. --%>
242
        <.badge :if={@repository.upstream_url} id={"#{@id}-mirror"} variant={:dim}>
243
          upstream mirror
244
        </.badge>
240 245
        <.badge :if={@repository.lifecycle_state != "ready"} variant={state_variant(@repository)}>
241 246
          {@repository.lifecycle_state}
242 247
        </.badge>

@@ -259,7 +264,19 @@ defmodule OpenAgentsWeb.RepositoryIndexLive do

259 264
        <span :if={@source} aria-hidden="true">·</span>
260 265
        <%!-- REPOSITORY-001: the snapshot was copied once and is never
261 266
        resynchronized, so the row says so rather than implying a mirror. --%>
262
        <span :if={@source} id={"#{@id}-provenance"} data-source={@source}>
267
        <span
268
          :if={@source && @repository.upstream_url}
269
          id={"#{@id}-provenance"}
270
          data-source={@source}
271
        >
272
          One-way mirror of <code class="text-foreground">{@repository.upstream_url}</code>,
273
          licensed {@repository.upstream_license}
274
        </span>
275
        <span
276
          :if={@source && is_nil(@repository.upstream_url)}
277
          id={"#{@id}-provenance"}
278
          data-source={@source}
279
        >
263 280
          Imported once from GitHub, from <code class="text-foreground">{@source}</code>
264 281
        </span>
265 282
      </div>
priv/migration_lineages/prior-2026-08-19.json modified +2 -1

@@ -288,7 +288,8 @@

288 288
    20260824032226,
289 289
    20260824035934,
290 290
    20260824040140,
291
    20260824042729
291
    20260824042729,
292
    20260824043735
292 293
  ],
293 294
  "required_tables": [
294 295
    "users",
priv/repo/migrations/20260824043735_add_upstream_mirror_to_repositories.exs added +26

@@ -0,0 +1,26 @@

1
defmodule OpenAgents.Repo.Migrations.AddUpstreamMirrorToRepositories do
2
  use Ecto.Migration
3
4
  # An upstream mirror is a repository whose content comes from a public
5
  # source this forge does not own. `upstream_url` names that source and is
6
  # the whole distinction: NULL is an owned repository, non-NULL is a mirror.
7
  #
8
  # `upstream_license` travels with it, and the check constraint is why a
9
  # mirror can never be silent about its license: the two columns are NULL
10
  # together or set together, so recording an upstream without recording what
11
  # it is licensed under is not a representable state. An upstream that
12
  # publishes no license records the literal "none", which is a statement
13
  # rather than an omission.
14
  def change do
15
    alter table(:repositories) do
16
      add :upstream_url, :string
17
      add :upstream_license, :string
18
    end
19
20
    create constraint(:repositories, :repositories_upstream_mirror_check,
21
             check: "(upstream_url IS NULL) = (upstream_license IS NULL)"
22
           )
23
24
    create index(:repositories, [:upstream_url], where: "upstream_url IS NOT NULL")
25
  end
26
end
test/openagents/forge/git_http_test.exs modified +119

@@ -456,6 +456,125 @@ defmodule OpenAgents.Forge.GitHTTPTest do

456 456
    sh!(machine_clone, "git", ["push", "origin", "HEAD:main"])
457 457
  end
458 458
459
  test "an upstream mirror serves a complete clone and refuses every push", %{
460
    base: base,
461
    port: port,
462
    token: token,
463
    user: user
464
  } do
465
    source = Path.join(base, "walgit-source")
466
    File.mkdir_p!(source)
467
    sh!(source, "git", ["init", "--initial-branch=main"])
468
    sh!(source, "git", ["config", "user.email", "test@example.com"])
469
    sh!(source, "git", ["config", "user.name", "Upstream"])
470
471
    Enum.each(1..3, fn index ->
472
      File.write!(Path.join(source, "README.md"), "revision #{index}\n")
473
      sh!(source, "git", ["add", "README.md"])
474
      sh!(source, "git", ["commit", "-m", "Revision #{index}"])
475
    end)
476
477
    head = source |> sh!("git", ["rev-parse", "HEAD"]) |> String.trim()
478
    refs = %{"refs/heads/main" => head}
479
480
    {:ok, mirror, _import, :created} =
481
      Repositories.create_user_mirror(
482
        user,
483
        %{
484
          source_repository_id: 909,
485
          source_owner_id: 777_777,
486
          source_full_name: "tobi/walgit",
487
          source_default_branch: "main",
488
          source_ref_digest: ref_digest(source, refs),
489
          source_head_sha: head,
490
          source_refs: refs,
491
          source_uses_lfs: false,
492
          source_public: true,
493
          source_license: "MIT"
494
        },
495
        %{name: "walgit", visibility: "public", default_branch: "main"},
496
        "git-http-mirror-key"
497
      )
498
499
    :ok = OpenAgents.Repositories.Importer.import(mirror, source_url: source)
500
501
    mirror
502
    |> Ecto.Changeset.change(lifecycle_state: "ready", ready_at: DateTime.utc_now())
503
    |> Repo.update!()
504
505
    mirror_url = "http://x:#{token}@127.0.0.1:#{port}/git-http-owner/walgit.git"
506
507
    # A mirror is still a forge repository: EXIT-004 holds for it. The clone
508
    # walks to the root because the copy was taken whole, so no boundary can
509
    # abort the transfer.
510
    clone = Path.join(base, "walgit-clone")
511
    sh!(base, "git", ["clone", mirror_url, clone])
512
    assert String.trim(sh!(clone, "git", ["rev-list", "--count", "HEAD"])) == "3"
513
514
    # `sh!/3` flunks on a nonzero exit, so this asserts fsck succeeded; the
515
    # refutations catch the dangling and missing objects fsck reports while
516
    # still exiting zero.
517
    fsck = sh!(clone, "git", ["fsck", "--full"])
518
    refute fsck =~ "missing"
519
    refute fsck =~ "broken"
520
521
    sh!(clone, "git", ["config", "user.email", "test@example.com"])
522
    sh!(clone, "git", ["config", "user.name", "Forge Test"])
523
    File.write!(Path.join(clone, "local.txt"), "local\n")
524
    sh!(clone, "git", ["add", "local.txt"])
525
    sh!(clone, "git", ["commit", "-m", "Local commit"])
526
527
    {output, status} =
528
      System.cmd("git", ["-c", "credential.helper=", "push", "origin", "HEAD:main"],
529
        cd: clone,
530
        stderr_to_stdout: true
531
      )
532
533
    assert status != 0
534
    assert output =~ "403"
535
536
    # The git client stops at the ref advertisement, so a refusal that lived
537
    # only there would look complete while `POST /git-receive-pack` stayed
538
    # open. Both are asserted, against the same account whose token clones the
539
    # repository fine.
540
    advertisement =
541
      Req.get!(
542
        "http://127.0.0.1:#{port}/git-http-owner/walgit.git/info/refs?service=git-receive-pack",
543
        auth: {:basic, "x:#{token}"},
544
        retry: false
545
      )
546
547
    assert advertisement.status == 403
548
    assert advertisement.body =~ "upstream mirror of https://github.com/tobi/walgit"
549
    assert advertisement.body =~ "accepts no pushes"
550
551
    receive_pack =
552
      Req.post!(
553
        "http://127.0.0.1:#{port}/git-http-owner/walgit.git/git-receive-pack",
554
        auth: {:basic, "x:#{token}"},
555
        headers: [{"content-type", "application/x-git-receive-pack-request"}],
556
        body: "0000",
557
        retry: false
558
      )
559
560
    assert receive_pack.status == 403
561
    assert receive_pack.body =~ "upstream mirror of https://github.com/tobi/walgit"
562
563
    # Nothing moved: the mirror still holds exactly what the upstream had.
564
    assert Repos.refs(mirror.storage_key) == refs
565
  end
566
567
  defp ref_digest(source, refs) do
568
    refs
569
    |> Enum.sort_by(&elem(&1, 0))
570
    |> Enum.map_join("\n", fn {name, sha} ->
571
      object_type = source |> sh!("git", ["cat-file", "-t", sha]) |> String.trim()
572
      Enum.join([name, object_type, sha], "\0")
573
    end)
574
    |> then(&:crypto.hash(:sha256, &1))
575
    |> Base.encode16(case: :lower)
576
  end
577
459 578
  test "the legacy initial repository path remains available", %{
460 579
    base: base,
461 580
    port: port,
test/openagents/repositories/upstream_mirror_test.exs added +318

@@ -0,0 +1,318 @@

1
defmodule OpenAgents.Repositories.UpstreamMirrorTest do
2
  @moduledoc """
3
  An upstream mirror: a repository whose content comes from a public source
4
  this forge does not own.
5
6
  Four properties, and each is proved against the mechanism rather than
7
  against a message. The upstream is recorded; the license travels or its
8
  absence is recorded; the ordinary import path cannot produce a mirror
9
  however its attributes are shaped; and the copy carries whole history, so a
10
  clone of it walks to the root and passes `git fsck --full` (#179, EXIT-004).
11
  """
12
13
  use OpenAgents.DataCase, async: false
14
15
  import OpenAgents.AccountsFixtures
16
17
  alias Ecto.Adapters.SQL
18
  alias OpenAgents.Forge.{Repos, Sync, WAL}
19
  alias OpenAgents.Repositories
20
  alias OpenAgents.Repositories.{Importer, Repository}
21
22
  setup do
23
    root =
24
      Path.join(
25
        System.tmp_dir!(),
26
        "upstream-mirror-#{System.unique_integer([:positive, :monotonic])}"
27
      )
28
29
    previous_data = Application.get_env(:openagents, :forge_data_dir)
30
    previous_wal = Application.get_env(:openagents, :forge_wal_dir)
31
    previous_adapter = Application.get_env(:openagents, :forge_wal_adapter)
32
33
    Application.put_env(:openagents, :forge_data_dir, Path.join(root, "data"))
34
    Application.put_env(:openagents, :forge_wal_dir, Path.join(root, "wal"))
35
    Application.put_env(:openagents, :forge_wal_adapter, OpenAgents.Forge.WAL.Local)
36
37
    on_exit(fn ->
38
      restore_env(:forge_data_dir, previous_data)
39
      restore_env(:forge_wal_dir, previous_wal)
40
      restore_env(:forge_wal_adapter, previous_adapter)
41
      File.rm_rf!(root)
42
    end)
43
44
    %{root: root}
45
  end
46
47
  describe "creating a mirror" do
48
    test "records the upstream and the license it found" do
49
      user = repository_user_fixture("mirror-owner")
50
51
      assert {:ok, repository, _import, :created} =
52
               Repositories.create_user_mirror(
53
                 user,
54
                 foreign_source(license: "MIT"),
55
                 %{name: "walgit", visibility: "public"},
56
                 "mirror-create-key"
57
               )
58
59
      assert repository.upstream_url == "https://github.com/tobi/walgit"
60
      assert repository.upstream_license == "MIT"
61
      assert Repositories.mirror?(repository)
62
    end
63
64
    test "records the absence of a license rather than leaving it blank" do
65
      user = repository_user_fixture("mirror-unlicensed-owner")
66
67
      assert {:ok, repository, _import, :created} =
68
               Repositories.create_user_mirror(
69
                 user,
70
                 foreign_source(license: nil),
71
                 %{name: "unlicensed", visibility: "public"},
72
                 "mirror-unlicensed-key"
73
               )
74
75
      assert repository.upstream_license == "none"
76
    end
77
78
    test "a license GitHub could not identify is an absence, not an identifier" do
79
      user = repository_user_fixture("mirror-noassertion-owner")
80
81
      assert {:ok, repository, _import, :created} =
82
               Repositories.create_user_mirror(
83
                 user,
84
                 foreign_source(license: "NOASSERTION"),
85
                 %{name: "noassertion", visibility: "public"},
86
                 "mirror-noassertion-key"
87
               )
88
89
      assert repository.upstream_license == "none"
90
    end
91
92
    test "a private source cannot become a mirror" do
93
      user = repository_user_fixture("mirror-private-owner")
94
95
      assert {:error, :source_repository_not_public} =
96
               Repositories.create_user_mirror(
97
                 user,
98
                 foreign_source(public: false),
99
                 %{name: "private-source", visibility: "public"},
100
                 "mirror-private-key"
101
               )
102
    end
103
104
    test "the database refuses an upstream recorded without its license" do
105
      user = repository_user_fixture("mirror-halfrecord-owner")
106
107
      assert {:ok, repository, :created} =
108
               Repositories.create_user_repository(user, %{name: "owned"}, "half-record-key")
109
110
      # The license does not travel by convention. The two columns are NULL
111
      # together or set together, so "a mirror whose license nobody recorded"
112
      # is not a state this database can hold, however it is reached.
113
      assert_raise Postgrex.Error, ~r/repositories_upstream_mirror_check/, fn ->
114
        SQL.query!(
115
          OpenAgents.Repo,
116
          "UPDATE repositories SET upstream_url = $1 WHERE id = $2",
117
          ["https://github.com/tobi/walgit", Ecto.UUID.dump!(repository.id)]
118
        )
119
      end
120
    end
121
  end
122
123
  describe "the owner-identity gate" do
124
    test "an import of a source owned by someone else is still refused" do
125
      user = repository_user_fixture("import-foreign-owner")
126
127
      assert {:error, :source_namespace_mismatch} =
128
               Repositories.create_user_import(
129
                 user,
130
                 foreign_source(),
131
                 %{name: "walgit", visibility: "public"},
132
                 "import-foreign-key"
133
               )
134
    end
135
136
    test "the import path cannot produce a mirror, whatever attributes it is given" do
137
      user = repository_user_fixture("import-attrs-owner")
138
139
      # The source owner is this account, so the owner gate admits the import
140
      # and cannot be what refuses the upstream fields. What refuses them is
141
      # that `changeset/2` never casts them: only
142
      # `Repository.mirror_creation_changeset/6` writes an upstream, and only
143
      # `create_user_mirror/4` reaches it.
144
      assert {:ok, repository, _import, :created} =
145
               Repositories.create_user_import(
146
                 user,
147
                 %{foreign_source() | source_owner_id: user.github_id},
148
                 %{
149
                   name: "not-a-mirror",
150
                   visibility: "public",
151
                   upstream_url: "https://github.com/tobi/walgit",
152
                   upstream_license: "MIT"
153
                 },
154
                 "import-attrs-key"
155
               )
156
157
      assert repository.upstream_url == nil
158
      assert repository.upstream_license == nil
159
      refute Repositories.mirror?(repository)
160
    end
161
162
    test "an ordinary repository cannot be given an upstream through the update changeset" do
163
      user = repository_user_fixture("update-attrs-owner")
164
165
      assert {:ok, repository, :created} =
166
               Repositories.create_user_repository(user, %{name: "owned"}, "update-attrs-key")
167
168
      changeset =
169
        Repository.changeset(repository, %{
170
          upstream_url: "https://github.com/tobi/walgit",
171
          upstream_license: "MIT"
172
        })
173
174
      refute Map.has_key?(changeset.changes, :upstream_url)
175
      refute Map.has_key?(changeset.changes, :upstream_license)
176
    end
177
  end
178
179
  describe "copying the history" do
180
    test "a mirror carries whole history, records an empty boundary, and clones clean",
181
         %{root: root} do
182
      source = source_repository!(root, "mirror-source", 3)
183
      user = repository_user_fixture("mirror-history-owner")
184
185
      {:ok, repository, _import, :created} =
186
        Repositories.create_user_mirror(
187
          user,
188
          source_record(source, "tobi/walgit"),
189
          %{name: "walgit", visibility: "public", default_branch: "main"},
190
          "mirror-history-key"
191
        )
192
193
      assert :ok = Importer.import(repository, source_url: source)
194
195
      # #179: a WAL entry that records no boundary is the failure. This one
196
      # records an empty boundary, which is a statement that there is none.
197
      assert {:ok, _generation, index} = WAL.read_index(repository.storage_key)
198
      assert [%{"shallow" => []}] = WAL.entries(index)
199
200
      assert :ok = Sync.ensure_fresh(repository.storage_key, "main")
201
      bare = Repos.bare_path(repository.storage_key)
202
203
      refute File.exists?(Path.join(bare, "shallow"))
204
      assert count_commits(bare) == 3
205
206
      # The population is closed by the walk, not by the ref list: clone over
207
      # a real transport, then fsck the copy. A repository holding every tip
208
      # can still be impossible to clone.
209
      work = Path.join(root, "mirror-clone")
210
      assert {_output, 0} = System.cmd("git", ["clone", "file://" <> bare, work])
211
      assert count_commits(work) == 3
212
      assert {output, 0} = System.cmd("git", ["fsck", "--full"], cd: work)
213
      refute output =~ "missing"
214
      refute output =~ "broken"
215
    end
216
217
    test "an owned import still takes the tip and states the boundary it produced",
218
         %{root: root} do
219
      source = source_repository!(root, "import-source", 3)
220
      user = repository_user_fixture("import-history-owner")
221
222
      {:ok, repository, _import, :created} =
223
        Repositories.create_user_import(
224
          user,
225
          %{
226
            source_record(source, "import-history-owner/source")
227
            | source_owner_id: user.github_id
228
          },
229
          %{name: "shallow-copy", visibility: "public", default_branch: "main"},
230
          "import-history-key"
231
        )
232
233
      assert :ok = Importer.import(repository, source_url: source)
234
235
      assert {:ok, _generation, index} = WAL.read_index(repository.storage_key)
236
      assert [%{"shallow" => [_boundary]}] = WAL.entries(index)
237
238
      assert :ok = Sync.ensure_fresh(repository.storage_key, "main")
239
      bare = Repos.bare_path(repository.storage_key)
240
241
      assert File.exists?(Path.join(bare, "shallow"))
242
      assert count_commits(bare) == 1
243
    end
244
  end
245
246
  defp count_commits(path) do
247
    {output, 0} = System.cmd("git", ["rev-list", "--count", "HEAD"], cd: path)
248
    output |> String.trim() |> String.to_integer()
249
  end
250
251
  defp source_repository!(root, name, commits) do
252
    path = Path.join(root, name)
253
    File.mkdir_p!(path)
254
    git!(path, ["init", "--initial-branch=main"])
255
    git!(path, ["config", "user.email", "test@example.com"])
256
    git!(path, ["config", "user.name", "Mirror test"])
257
258
    Enum.each(1..commits, fn index ->
259
      File.write!(Path.join(path, "README.md"), "revision #{index}\n")
260
      git!(path, ["add", "README.md"])
261
      git!(path, ["commit", "-m", "Revision #{index}"])
262
    end)
263
264
    path
265
  end
266
267
  defp source_record(source, full_name) do
268
    sha = source |> git!(["rev-parse", "HEAD"]) |> String.trim()
269
    refs = %{"refs/heads/main" => sha}
270
271
    %{
272
      source_repository_id: 909,
273
      source_owner_id: 777_777,
274
      source_full_name: full_name,
275
      source_default_branch: "main",
276
      source_ref_digest: ref_digest(source, refs),
277
      source_head_sha: sha,
278
      source_refs: refs,
279
      source_uses_lfs: false,
280
      source_public: true,
281
      source_license: "MIT"
282
    }
283
  end
284
285
  defp foreign_source(options \\ []) do
286
    %{
287
      source_repository_id: 909,
288
      source_owner_id: 777_777,
289
      source_full_name: "tobi/walgit",
290
      source_default_branch: "main",
291
      source_ref_digest: String.duplicate("a", 64),
292
      source_head_sha: String.duplicate("e", 40),
293
      source_refs: %{"refs/heads/main" => String.duplicate("e", 40)},
294
      source_uses_lfs: false,
295
      source_public: Keyword.get(options, :public, true),
296
      source_license: Keyword.get(options, :license, "MIT")
297
    }
298
  end
299
300
  defp git!(directory, args) do
301
    {output, 0} = System.cmd("git", args, cd: directory, stderr_to_stdout: true)
302
    output
303
  end
304
305
  defp ref_digest(source, refs) do
306
    refs
307
    |> Enum.sort_by(&elem(&1, 0))
308
    |> Enum.map_join("\n", fn {name, sha} ->
309
      object_type = source |> git!(["cat-file", "-t", sha]) |> String.trim()
310
      Enum.join([name, object_type, sha], "\0")
311
    end)
312
    |> then(&:crypto.hash(:sha256, &1))
313
    |> Base.encode16(case: :lower)
314
  end
315
316
  defp restore_env(key, nil), do: Application.delete_env(:openagents, key)
317
  defp restore_env(key, value), do: Application.put_env(:openagents, key, value)
318
end
test/openagents_web/controllers/repository_import_controller_test.exs modified +160

@@ -124,6 +124,166 @@ defmodule OpenAgentsWeb.RepositoryImportControllerTest do

124 124
           } = json_response(created, 202)
125 125
  end
126 126
127
  describe "an upstream mirror of a repository this account does not own" do
128
    test "a foreign public source is refused by name, and the refusal names the source", %{
129
      conn: conn
130
    } do
131
      user = github_user("repository-foreign-import", "octavia")
132
      assert {:ok, user} = Accounts.store_github_token(user, "gho_foreign_fixture")
133
134
      expect_foreign_source(String.duplicate("d", 40), false, "MIT")
135
136
      response =
137
        conn
138
        |> authorize(user)
139
        |> put_req_header("idempotency-key", "foreign-import-key")
140
        |> post(~p"/api/v3/user/repos/imports", %{
141
          source: %{provider: "github", repository: "tobi/walgit"}
142
        })
143
144
      body = json_response(response, 403)
145
146
      assert body["code"] == "source_namespace_mismatch"
147
      # The source owner is what failed, so the message says so. The old
148
      # message named the destination namespace, which was this account's own.
149
      assert body["message"] =~ "tobi/walgit"
150
      assert body["message"] =~ "owned by another GitHub account"
151
      assert body["message"] =~ ~s("mirror": true)
152
      assert body["failed"] == %{"source" => "tobi/walgit", "destination" => "eligible"}
153
    end
154
155
    test "the same source is brought in as a mirror, and the response names the upstream", %{
156
      conn: conn
157
    } do
158
      user = github_user("repository-mirror-api", "octavia")
159
      assert {:ok, user} = Accounts.store_github_token(user, "gho_mirror_fixture")
160
      main_sha = String.duplicate("e", 40)
161
162
      expect_foreign_source(main_sha, false, "MIT")
163
164
      response =
165
        conn
166
        |> authorize(user)
167
        |> put_req_header("idempotency-key", "mirror-key")
168
        |> post(~p"/api/v3/user/repos/imports", %{
169
          source: %{provider: "github", repository: "tobi/walgit"},
170
          mirror: true
171
        })
172
173
      assert %{
174
               "name" => "walgit",
175
               "mirror" => true,
176
               "visibility" => "public",
177
               "upstream" => %{
178
                 "url" => "https://github.com/tobi/walgit",
179
                 "license" => "MIT",
180
                 "direction" => "one_way",
181
                 "accepts_pushes" => false
182
               },
183
               "permissions" => permissions
184
             } = json_response(response, 202)
185
186
      # The Git plane refuses every push to a mirror, so no projection of it
187
      # may report a push permission its owner does not have.
188
      assert permissions["push"] == false
189
      assert permissions["admin"] == true
190
    end
191
192
    test "an upstream with no license records the absence rather than omitting it", %{conn: conn} do
193
      user = github_user("repository-unlicensed-mirror", "octavia")
194
      assert {:ok, user} = Accounts.store_github_token(user, "gho_unlicensed_fixture")
195
196
      expect_foreign_source(String.duplicate("f", 40), false, nil)
197
198
      response =
199
        conn
200
        |> authorize(user)
201
        |> put_req_header("idempotency-key", "unlicensed-mirror-key")
202
        |> post(~p"/api/v3/user/repos/imports", %{
203
          source: %{provider: "github", repository: "tobi/walgit"},
204
          mirror: true
205
        })
206
207
      assert %{"upstream" => %{"license" => "none"}} = json_response(response, 202)
208
    end
209
210
    test "a private source cannot be mirrored", %{conn: conn} do
211
      user = github_user("repository-private-mirror", "octavia")
212
      assert {:ok, user} = Accounts.store_github_token(user, "gho_private_mirror_fixture")
213
214
      expect_foreign_source(String.duplicate("1", 40), true, "MIT")
215
216
      response =
217
        conn
218
        |> authorize(user)
219
        |> put_req_header("idempotency-key", "private-mirror-key")
220
        |> post(~p"/api/v3/user/repos/imports", %{
221
          source: %{provider: "github", repository: "tobi/walgit"},
222
          mirror: true
223
        })
224
225
      assert %{"code" => "source_repository_not_public"} = json_response(response, 403)
226
    end
227
228
    test "an owned repository publishes the distinction too", %{conn: conn} do
229
      user = github_user("repository-owned-projection")
230
231
      response =
232
        conn
233
        |> authorize(user)
234
        |> put_req_header("idempotency-key", "owned-projection-key")
235
        |> post(~p"/api/v3/user/repos", %{name: "mine", private: false})
236
237
      assert %{"mirror" => false, "upstream" => nil} = json_response(response, 202)
238
    end
239
  end
240
241
  defp expect_foreign_source(main_sha, private?, license) do
242
    Req.Test.expect(__MODULE__, fn github_conn ->
243
      assert github_conn.request_path == "/repos/tobi/walgit"
244
245
      Req.Test.json(github_conn, %{
246
        "id" => 909,
247
        "node_id" => "R_909",
248
        "name" => "walgit",
249
        "full_name" => "tobi/walgit",
250
        "private" => private?,
251
        "default_branch" => "main",
252
        "license" => if(license, do: %{"spdx_id" => license, "key" => "mit"}),
253
        "owner" => %{
254
          "id" => 777_777,
255
          "node_id" => "U_777777",
256
          "login" => "tobi",
257
          "avatar_url" => "https://avatars.githubusercontent.com/u/777777?v=4",
258
          "type" => "User"
259
        },
260
        "permissions" => %{"pull" => true, "push" => false, "admin" => false}
261
      })
262
    end)
263
264
    Req.Test.expect(__MODULE__, fn github_conn ->
265
      assert github_conn.request_path == "/repos/tobi/walgit/git/matching-refs/heads/"
266
267
      Req.Test.json(github_conn, [
268
        %{"ref" => "refs/heads/main", "object" => %{"type" => "commit", "sha" => main_sha}}
269
      ])
270
    end)
271
272
    Req.Test.expect(__MODULE__, fn github_conn ->
273
      assert github_conn.request_path == "/repos/tobi/walgit/git/matching-refs/tags/"
274
      Req.Test.json(github_conn, [])
275
    end)
276
277
    Req.Test.expect(__MODULE__, fn github_conn ->
278
      assert github_conn.request_path == "/repos/tobi/walgit/git/trees/main"
279
280
      Req.Test.json(github_conn, %{
281
        "truncated" => false,
282
        "tree" => [%{"path" => "README.md", "type" => "blob", "size" => 200}]
283
      })
284
    end)
285
  end
286
127 287
  defp expect_import_source(user, main_sha, tag_sha, private? \\ true) do
128 288
    Req.Test.expect(__MODULE__, fn github_conn ->
129 289
      assert github_conn.request_path == "/repos/octavia/source-project"
test/openagents_web/live/code_live_test.exs modified +73

@@ -383,6 +383,79 @@ defmodule OpenAgentsWeb.CodeLiveTest do

383 383
      refute html =~ "Synced"
384 384
    end
385 385
386
    test "an upstream mirror names its upstream to an anonymous reader", %{conn: conn} do
387
      owner = github_user("mirror-page-owner", "mirror-page-owner")
388
389
      assert {:ok, repository, _import, :created} =
390
               OpenAgents.Repositories.create_user_mirror(
391
                 owner,
392
                 %{
393
                   source_repository_id: 909,
394
                   source_owner_id: 777_777,
395
                   source_full_name: "tobi/walgit",
396
                   source_default_branch: "main",
397
                   source_ref_digest: String.duplicate("a", 64),
398
                   source_head_sha: String.duplicate("c", 40),
399
                   source_refs: %{"refs/heads/main" => String.duplicate("c", 40)},
400
                   source_uses_lfs: false,
401
                   source_public: true,
402
                   source_license: "MIT"
403
                 },
404
                 %{name: "walgit", visibility: "public", default_branch: "main"},
405
                 "mirror-page-key"
406
               )
407
408
      repository
409
      |> Ecto.Changeset.change(lifecycle_state: "ready", ready_at: DateTime.utc_now())
410
      |> Repo.update!()
411
412
      # No session: a stranger reading the page is the reader most likely to
413
      # mistake a mirror for something this account wrote.
414
      {:ok, view, html} = live(conn, "/mirror-page-owner/walgit")
415
416
      assert has_element?(view, ".repo-view__rail #repo-upstream-mirror")
417
      assert html =~ "Upstream mirror"
418
      assert html =~ "https://github.com/tobi/walgit"
419
      assert html =~ "MIT"
420
      assert html =~ "accepts no pushes"
421
422
      # The import block claims OpenAgents owns the snapshot. A mirror makes
423
      # the opposite claim, so the two never appear together.
424
      refute has_element?(view, "#repo-import-provenance")
425
      refute html =~ "Imported from GitHub"
426
    end
427
428
    test "a mirror of an unlicensed upstream says so rather than staying silent", %{conn: conn} do
429
      owner = github_user("unlicensed-mirror-page", "unlicensed-mirror-page")
430
431
      assert {:ok, repository, _import, :created} =
432
               OpenAgents.Repositories.create_user_mirror(
433
                 owner,
434
                 %{
435
                   source_repository_id: 910,
436
                   source_owner_id: 777_777,
437
                   source_full_name: "tobi/unlicensed",
438
                   source_default_branch: "main",
439
                   source_ref_digest: String.duplicate("a", 64),
440
                   source_head_sha: String.duplicate("c", 40),
441
                   source_refs: %{"refs/heads/main" => String.duplicate("c", 40)},
442
                   source_uses_lfs: false,
443
                   source_public: true,
444
                   source_license: nil
445
                 },
446
                 %{name: "unlicensed", visibility: "public", default_branch: "main"},
447
                 "unlicensed-mirror-page-key"
448
               )
449
450
      repository
451
      |> Ecto.Changeset.change(lifecycle_state: "ready", ready_at: DateTime.utc_now())
452
      |> Repo.update!()
453
454
      {:ok, _view, html} = live(conn, "/unlicensed-mirror-page/unlicensed")
455
456
      assert html =~ "No license found upstream"
457
    end
458
386 459
    test "a repository created empty shows no import provenance", %{conn: conn} do
387 460
      owner = github_user("no-provenance-owner", "no-provenance-owner")
388 461

This page updates live while a promote is in flight · changelog