Add scoped staging run cleanup

fba8e4dcf30e · Christopher David · · parent 553c0ee1d8dd

Add scoped staging run cleanup

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified config/config.exs
  • modified config/runtime.exs
  • modified docs/2026-08-20-integration-hardening-and-staging-readiness-recommendations.md
  • modified docs/runtime-configuration.md
  • modified infra/staging/README.md
  • modified lib/openagents/runtime_config.ex
  • added lib/openagents/staging/disposable_resource.ex
  • added lib/openagents/staging_cleanup.ex
  • modified ops/ci/contracts.sh
  • added ops/staging/cleanup-run.sh
  • modified ops/staging/gate-5-profile.sh
  • added priv/repo/migrations/20260820150000_create_staging_disposable_resources.exs
  • modified test/openagents/runtime_config_test.exs
  • added test/openagents/staging_cleanup_test.exs

Diff

14 files changed, +917 -8

config/config.exs modified +1

@@ -33,6 +33,7 @@ config :openagents,

33 33
  generators: [timestamp_type: :utc_datetime],
34 34
  runtime_environment: :development,
35 35
  staging_gate: 0,
36
  staging_cleanup_enabled: false,
36 37
  production_deploy_enabled: false,
37 38
  build_revision: "image",
38 39
  image_digest: nil,
config/runtime.exs modified +2

@@ -96,6 +96,7 @@ if config_env() == :prod do

96 96
97 97
  staging_gate = parse_integer.("OPENAGENTS_STAGING_GATE", 0..16)
98 98
  production_deploy_enabled = parse_boolean.("OPENAGENTS_PRODUCTION_DEPLOY_ENABLED")
99
  staging_cleanup_enabled = parse_boolean.("OPENAGENTS_STAGING_CLEANUP_ENABLED")
99 100
  secure_cookies = parse_boolean.("OPENAGENTS_SECURE_COOKIES")
100 101
  migrate_on_boot = parse_boolean.("OPENAGENTS_MIGRATE_ON_BOOT")
101 102
  host = required_text.("PHX_HOST")

@@ -260,6 +261,7 @@ if config_env() == :prod do

260 261
  config :openagents,
261 262
    runtime_environment: runtime_environment,
262 263
    staging_gate: staging_gate,
264
    staging_cleanup_enabled: staging_cleanup_enabled,
263 265
    production_deploy_enabled: production_deploy_enabled,
264 266
    build_revision: OpenAgents.BuildInfo.revision(),
265 267
    image_digest: optional_text.("OPENAGENTS_IMAGE_DIGEST"),
docs/2026-08-20-integration-hardening-and-staging-readiness-recommendations.md modified +12 -6

@@ -2,7 +2,7 @@

2 2
3 3
Date: 2026-08-20
4 4
5
Status: In progress; Gates 0–11 complete locally, Gate 12 cloud isolation proof pending
5
Status: In progress; Gates 0–11 complete locally, Gate 12 cloud and live cleanup proof pending
6 6
7 7
## Outcome
8 8

@@ -1238,14 +1238,20 @@ Implemented locally on 2026-08-20:

1238 1238
  minimal private deployer BEAM node performs exact instance metadata updates
1239 1239
  and resets under the only identity that holds those permissions; it does not
1240 1240
  start the application, join Ra, open HTTP, or connect to PostgreSQL.
1241
- Added an immutable, manifest-scoped staging cleanup registry and one operator
1242
  command. The harness must register each disposable account, repository,
1243
  recording, and product machine before use. Cleanup refuses canonical or
1244
  administrator resources, online machines, active work or conversations, and
1245
  account-owned resources outside the manifest. It deletes one run in a
1246
  transaction and reports only bounded counts.
1241 1247
1242 1248
The local infrastructure definition and mocked safety tests pass. The cloud
1243
apply, isolation receipt, and one-command disposable-run cleanup proof remain
1244
open; cloud work is blocked until the operator refreshes the expired Google
1245
Cloud CLI and Application Default Credentials. No staging or production cloud
1246
resource changed during this implementation step. See the [isolated staging
1249
apply, isolation receipt, and live disposable-run cleanup proof remain open;
1250
cloud work is blocked until the operator refreshes the expired Google Cloud CLI
1251
and Application Default Credentials. No staging or production cloud resource
1252
changed during this implementation step. See the [isolated staging
1247 1253
infrastructure](../infra/staging/README.md) for the exact bootstrap, plan,
1248
apply, and validation procedure.
1254
apply, validation, and cleanup procedures.
1249 1255
1250 1256
## Gate 13: Deploy to staging reproducibly
1251 1257
docs/runtime-configuration.md modified +1

@@ -53,6 +53,7 @@ URLs, receipts, or checked-in environment files.

53 53
| --- | --- | --- |
54 54
| Release | `OPENAGENTS_ENVIRONMENT` | `staging`; `production` remains separately locked |
55 55
| Release | `OPENAGENTS_STAGING_GATE` | Integer `0` through `16`; feature admission is tied to it |
56
| Release | `OPENAGENTS_STAGING_CLEANUP_ENABLED` | `true` only at staging Gate 12 or later; always `false` elsewhere |
56 57
| Release | `OPENAGENTS_PRODUCTION_DEPLOY_ENABLED` | `false` until a later production decision |
57 58
| Release | `OPENAGENTS_IMAGE_DIGEST` | Exact `sha256:` image digest at staging Gate 12 and later; empty before that gate |
58 59
| Endpoint | `PHX_HOST` | Exactly `stage.openagents.com` in staging |
infra/staging/README.md modified +46 -2

@@ -112,10 +112,54 @@ writes a content-free receipt under

112 112
`.git/openagents/staging-isolation/<full-sha>.json`. Do not commit Terraform
113 113
plans, state, credentials, project inventory, IP addresses, or secret values.
114 114
115
## Clean up a disposable test run
116
117
Give each staging test run a unique lowercase identifier with 8 through 64
118
letters, numbers, and hyphens. Before the test harness uses a disposable
119
resource, register its database ID under that run:
120
121
```elixir
122
OpenAgents.StagingCleanup.register(run_id, :account, user.id)
123
OpenAgents.StagingCleanup.register(run_id, :repository, repository.id)
124
OpenAgents.StagingCleanup.register(run_id, :recording, recording.id)
125
OpenAgents.StagingCleanup.register(run_id, :machine, machine.id)
126
```
127
128
Set `OPENAGENTS_STAGING_CLEANUP_ENABLED=true` only for the staging release at
129
Gate 12 or later. The registration manifest is immutable. A resource can
130
belong to only one run, and the cleanup command does not infer targets from
131
names, timestamps, owners, or labels.
132
133
Preview content-free counts before deletion:
134
135
```sh
136
ops/staging/cleanup-run.sh gate14-20260820-0001 check
137
```
138
139
Apply the same bounded manifest once you confirm the counts:
140
141
```sh
142
ops/staging/cleanup-run.sh gate14-20260820-0001 --apply
143
```
144
145
The command targets a fixed staging fleet node through Identity-Aware Proxy.
146
It refuses the canonical repository, administrator accounts, online machines,
147
machines with queued or running work, active text turns or voice sessions, and
148
accounts that own an unregistered project, machine, or recording. It performs
149
all database deletions in one transaction and removes the manifest only after
150
the transaction succeeds. An interrupted or refused cleanup remains safe to
151
preview and retry.
152
153
The `repository` and `machine` kinds refer to product database records. This
154
command never deletes Compute Engine fleet instances, Cloud SQL, Artifact
155
Registry images, Terraform resources, or production data. Quiesce the test
156
harness before you apply cleanup so it cannot create new run data concurrently.
157
115 158
## Complete Gate 12
116 159
117
Gate 12 remains incomplete until the cloud apply, isolation validator, and
118
manifest-scoped disposable-run cleanup command are proven. Do not populate
160
Gate 12 remains incomplete until the cloud apply, isolation validator, and a
161
live execution of the manifest-scoped disposable-run cleanup command are
162
proven. Do not populate
119 163
secrets, push an image, change DNS for `stage.openagents.com`, or deploy a
120 164
candidate as part of the infrastructure apply. Gate 13 performs those steps on
121 165
one exact, locally gated SHA after a separate review.
lib/openagents/runtime_config.ex modified +17

@@ -67,6 +67,7 @@ defmodule OpenAgents.RuntimeConfig do

67 67
         :ok <- validate_database(settings, environment),
68 68
         :ok <- validate_github(settings, environment),
69 69
         {:ok, features} <- validate_features(settings, environment, staging_gate),
70
         :ok <- validate_staging_cleanup(settings, environment, staging_gate),
70 71
         :ok <- validate_providers(settings, features),
71 72
         :ok <- validate_release_identity(settings, environment, staging_gate),
72 73
         {:ok, allowlist, examples} <- validate_forge(settings, environment, features),

@@ -148,6 +149,22 @@ defmodule OpenAgents.RuntimeConfig do

148 149
149 150
  defp validate_production_lock(_settings, _environment), do: :ok
150 151
152
  defp validate_staging_cleanup(settings, environment, staging_gate) do
153
    case Map.get(settings, :staging_cleanup_enabled) do
154
      false ->
155
        :ok
156
157
      true when environment == :staging and staging_gate >= 12 ->
158
        :ok
159
160
      true ->
161
        error(:staging_cleanup_enabled, "is admitted only in staging at Gate 12 or later")
162
163
      _invalid ->
164
        error(:staging_cleanup_enabled, "must be a boolean")
165
    end
166
  end
167
151 168
  defp validate_release_identity(settings, environment, staging_gate) do
152 169
    required? = environment == :production or (environment == :staging and staging_gate >= 12)
153 170
    revision = Map.get(settings, :build_revision)
lib/openagents/staging/disposable_resource.ex added +37

@@ -0,0 +1,37 @@

1
defmodule OpenAgents.Staging.DisposableResource do
2
  @moduledoc false
3
4
  use Ecto.Schema
5
  import Ecto.Changeset
6
7
  @primary_key {:id, :binary_id, autogenerate: true}
8
  @timestamps_opts [type: :utc_datetime_usec, updated_at: false]
9
  @kinds ~w(account machine recording repository)
10
11
  schema "staging_disposable_resources" do
12
    field :run_id, :string
13
    field :kind, :string
14
    field :resource_id, Ecto.UUID
15
16
    timestamps()
17
  end
18
19
  @type t :: %__MODULE__{
20
          id: Ecto.UUID.t(),
21
          run_id: String.t(),
22
          kind: String.t(),
23
          resource_id: Ecto.UUID.t(),
24
          inserted_at: DateTime.t()
25
        }
26
27
  def create_changeset(resource, attributes) do
28
    resource
29
    |> cast(attributes, [:run_id, :kind, :resource_id])
30
    |> validate_required([:run_id, :kind, :resource_id])
31
    |> validate_format(:run_id, ~r/\A[a-z0-9][a-z0-9-]{7,63}\z/)
32
    |> validate_inclusion(:kind, @kinds)
33
    |> unique_constraint([:kind, :resource_id])
34
    |> check_constraint(:run_id, name: :staging_disposable_run_id_check)
35
    |> check_constraint(:kind, name: :staging_disposable_kind_check)
36
  end
37
end
lib/openagents/staging_cleanup.ex added +402

@@ -0,0 +1,402 @@

1
defmodule OpenAgents.StagingCleanup do
2
  @moduledoc """
3
  Registers and removes resources created by one isolated staging test run.
4
5
  Cleanup can touch only resources that a harness registered under a bounded
6
  run ID. Registration is immutable, and cleanup fails closed for the canonical
7
  repository, administrator accounts, online machines, active work, and account
8
  data that still has an active turn or voice session.
9
  """
10
11
  import Ecto.Query
12
13
  alias OpenAgents.Accounts
14
  alias OpenAgents.Accounts.User
15
  alias OpenAgents.Conversations
16
  alias OpenAgents.Conversations.{Conversation, Visitor}
17
  alias OpenAgents.DataRights
18
  alias OpenAgents.Machines.Machine
19
  alias OpenAgents.ProjectFields.ProjectField
20
  alias OpenAgents.ProjectItems.ProjectItem
21
  alias OpenAgents.Projects.Project
22
  alias OpenAgents.Repo
23
  alias OpenAgents.Repositories.Repository
24
  alias OpenAgents.Staging.DisposableResource
25
  alias OpenAgents.Voice.{Recording, Session}
26
  alias OpenAgents.Work.Job
27
28
  @run_id_pattern ~r/\A[a-z0-9][a-z0-9-]{7,63}\z/
29
  @kinds [:account, :machine, :recording, :repository]
30
  @kind_names Map.new(@kinds, &{&1, Atom.to_string(&1)})
31
  @schemas %{
32
    account: User,
33
    machine: Machine,
34
    recording: Recording,
35
    repository: Repository
36
  }
37
38
  @type kind :: :account | :machine | :recording | :repository
39
40
  @doc "Registers one disposable resource before a staging harness uses it."
41
  @spec register(String.t(), kind(), Ecto.UUID.t()) ::
42
          {:ok, DisposableResource.t()} | {:error, atom() | Ecto.Changeset.t()}
43
  def register(run_id, kind, resource_id)
44
      when kind in @kinds and is_binary(run_id) and is_binary(resource_id) do
45
    with :ok <- ensure_admitted(),
46
         :ok <- validate_run_id(run_id),
47
         {:ok, cast_id} <- cast_resource_id(resource_id),
48
         {:ok, resource} <- fetch_resource(kind, cast_id),
49
         :ok <- validate_resource(kind, resource) do
50
      Repo.transaction(fn ->
51
        lock_run(run_id)
52
53
        %DisposableResource{}
54
        |> DisposableResource.create_changeset(%{
55
          run_id: run_id,
56
          kind: Map.fetch!(@kind_names, kind),
57
          resource_id: cast_id
58
        })
59
        |> Repo.insert()
60
        |> case do
61
          {:ok, registration} -> registration
62
          {:error, changeset} -> Repo.rollback(changeset)
63
        end
64
      end)
65
      |> normalize_transaction()
66
    end
67
  end
68
69
  def register(_run_id, _kind, _resource_id), do: {:error, :invalid_registration}
70
71
  @doc "Returns content-free registration counts for one run."
72
  @spec preview(String.t()) :: {:ok, map()} | {:error, atom()}
73
  def preview(run_id) when is_binary(run_id) do
74
    with :ok <- ensure_admitted(),
75
         :ok <- validate_run_id(run_id) do
76
      {:ok, %{registered: registration_counts(run_id)}}
77
    end
78
  end
79
80
  def preview(_run_id), do: {:error, :invalid_run_id}
81
82
  @doc "Removes every resource registered to one run in a single transaction."
83
  @spec cleanup(String.t()) :: {:ok, map()} | {:error, atom() | tuple()}
84
  def cleanup(run_id) when is_binary(run_id) do
85
    with :ok <- ensure_admitted(),
86
         :ok <- validate_run_id(run_id) do
87
      Repo.transaction(fn ->
88
        lock_run(run_id)
89
90
        registrations =
91
          Repo.all(
92
            from(resource in DisposableResource,
93
              where: resource.run_id == ^run_id,
94
              order_by: [asc: resource.kind, asc: resource.resource_id],
95
              lock: "FOR UPDATE"
96
            )
97
          )
98
99
        registered = counts(registrations)
100
        targets = targets(registrations)
101
102
        validate_cleanup_targets!(targets)
103
104
        deleted = %{
105
          recording: delete_recordings(targets.recording),
106
          repository: delete_repositories(targets.repository),
107
          machine: delete_machines(targets.machine),
108
          account: delete_accounts(targets.account)
109
        }
110
111
        {_registration_count, nil} =
112
          Repo.delete_all(from(resource in DisposableResource, where: resource.run_id == ^run_id))
113
114
        %{registered: registered, deleted: stringify_counts(deleted)}
115
      end)
116
      |> normalize_transaction()
117
    end
118
  end
119
120
  def cleanup(_run_id), do: {:error, :invalid_run_id}
121
122
  @doc "Returns one bounded JSON result for the staging operator command."
123
  @spec command!(String.t(), String.t()) :: String.t()
124
  def command!(run_id, "check") do
125
    case preview(run_id) do
126
      {:ok, result} -> encode_result(run_id, "checked", result)
127
      {:error, reason} -> raise "staging cleanup refused: #{bounded_reason(reason)}"
128
    end
129
  end
130
131
  def command!(run_id, "apply") do
132
    case cleanup(run_id) do
133
      {:ok, result} -> encode_result(run_id, "cleaned", result)
134
      {:error, reason} -> raise "staging cleanup refused: #{bounded_reason(reason)}"
135
    end
136
  end
137
138
  def command!(_run_id, _mode), do: raise("staging cleanup mode is invalid")
139
140
  defp ensure_admitted do
141
    environment = Application.get_env(:openagents, :runtime_environment)
142
    enabled? = Application.get_env(:openagents, :staging_cleanup_enabled, false) == true
143
    staging_gate = Application.get_env(:openagents, :staging_gate, 0)
144
145
    if enabled? and
146
         (environment == :test or (environment == :staging and staging_gate >= 12)) do
147
      :ok
148
    else
149
      {:error, :staging_cleanup_not_admitted}
150
    end
151
  end
152
153
  defp validate_run_id(run_id) do
154
    if Regex.match?(@run_id_pattern, run_id), do: :ok, else: {:error, :invalid_run_id}
155
  end
156
157
  defp cast_resource_id(resource_id) do
158
    case Ecto.UUID.cast(resource_id) do
159
      {:ok, cast_id} -> {:ok, cast_id}
160
      :error -> {:error, :invalid_resource_id}
161
    end
162
  end
163
164
  defp fetch_resource(kind, resource_id) do
165
    case Repo.get(Map.fetch!(@schemas, kind), resource_id) do
166
      nil -> {:error, :resource_not_found}
167
      resource -> {:ok, resource}
168
    end
169
  end
170
171
  defp validate_resource(:repository, %Repository{
172
         owner_key: "openagentsinc",
173
         name_key: "openagents.com"
174
       }),
175
       do: {:error, :canonical_repository_forbidden}
176
177
  defp validate_resource(:account, %User{} = user) do
178
    if Accounts.admin?(user), do: {:error, :administrator_account_forbidden}, else: :ok
179
  end
180
181
  defp validate_resource(_kind, _resource), do: :ok
182
183
  defp lock_run(run_id) do
184
    _result = Repo.query!("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [run_id])
185
    :ok
186
  end
187
188
  defp registration_counts(run_id) do
189
    Repo.all(
190
      from(resource in DisposableResource,
191
        where: resource.run_id == ^run_id,
192
        group_by: resource.kind,
193
        select: {resource.kind, count(resource.id)}
194
      )
195
    )
196
    |> Map.new()
197
    |> complete_counts()
198
  end
199
200
  defp counts(registrations) do
201
    registrations
202
    |> Enum.frequencies_by(& &1.kind)
203
    |> complete_counts()
204
  end
205
206
  defp complete_counts(counts) do
207
    Map.new(Map.values(@kind_names), &{&1, Map.get(counts, &1, 0)})
208
  end
209
210
  defp stringify_counts(counts) do
211
    Map.new(@kinds, fn kind -> {Map.fetch!(@kind_names, kind), Map.fetch!(counts, kind)} end)
212
  end
213
214
  defp targets(registrations) do
215
    by_kind = Enum.group_by(registrations, & &1.kind, & &1.resource_id)
216
217
    %{
218
      account: Map.get(by_kind, "account", []),
219
      machine: Map.get(by_kind, "machine", []),
220
      recording: Map.get(by_kind, "recording", []),
221
      repository: Map.get(by_kind, "repository", [])
222
    }
223
  end
224
225
  defp validate_cleanup_targets!(targets) do
226
    validate_repositories!(targets.repository)
227
228
    validate_accounts!(
229
      targets.account,
230
      targets.repository,
231
      targets.machine,
232
      targets.recording
233
    )
234
235
    validate_machines!(targets.machine)
236
  end
237
238
  defp validate_repositories!(repository_ids) do
239
    canonical? =
240
      Repo.exists?(
241
        from(repository in Repository,
242
          where:
243
            repository.id in ^repository_ids and repository.owner_key == "openagentsinc" and
244
              repository.name_key == "openagents.com"
245
        )
246
      )
247
248
    if canonical?, do: Repo.rollback(:canonical_repository_forbidden)
249
  end
250
251
  defp validate_accounts!(account_ids, repository_ids, machine_ids, recording_ids) do
252
    users = Repo.all(from(user in User, where: user.id in ^account_ids, lock: "FOR UPDATE"))
253
254
    if Enum.any?(users, &Accounts.admin?/1),
255
      do: Repo.rollback(:administrator_account_forbidden)
256
257
    outside_project? =
258
      Repo.exists?(
259
        from(project in Project,
260
          where:
261
            project.owner_user_id in ^account_ids and
262
              project.repository_id not in ^repository_ids
263
        )
264
      )
265
266
    if outside_project?, do: Repo.rollback(:account_owns_unregistered_project)
267
268
    outside_machine? =
269
      Repo.exists?(
270
        from(machine in Machine,
271
          where: machine.user_id in ^account_ids and machine.id not in ^machine_ids
272
        )
273
      )
274
275
    if outside_machine?, do: Repo.rollback(:account_owns_unregistered_machine)
276
277
    outside_recording? =
278
      Repo.exists?(
279
        from(recording in Recording,
280
          join: session in Session,
281
          on: session.id == recording.voice_session_id,
282
          join: conversation in Conversation,
283
          on: conversation.id == session.conversation_id,
284
          join: visitor in Visitor,
285
          on: visitor.id == conversation.visitor_id,
286
          where: visitor.user_id in ^account_ids and recording.id not in ^recording_ids
287
        )
288
      )
289
290
    if outside_recording?, do: Repo.rollback(:account_owns_unregistered_recording)
291
  end
292
293
  defp validate_machines!(machine_ids) do
294
    if Enum.any?(machine_ids, &OpenAgents.Computer.online?/1),
295
      do: Repo.rollback(:machine_online)
296
297
    active_work? =
298
      Repo.exists?(
299
        from(job in Job,
300
          where: job.machine_id in ^machine_ids and job.status in ["queued", "running"]
301
        )
302
      )
303
304
    if active_work?, do: Repo.rollback(:machine_has_active_work)
305
  end
306
307
  defp delete_recordings([]), do: 0
308
309
  defp delete_recordings(recording_ids) do
310
    {count, nil} =
311
      Repo.delete_all(from(recording in Recording, where: recording.id in ^recording_ids))
312
313
    count
314
  end
315
316
  defp delete_repositories([]), do: 0
317
318
  defp delete_repositories(repository_ids) do
319
    project_ids =
320
      from(project in Project,
321
        where: project.repository_id in ^repository_ids,
322
        select: project.id
323
      )
324
325
    {_field_count, nil} =
326
      Repo.delete_all(
327
        from(field in ProjectField, where: field.project_id in subquery(project_ids))
328
      )
329
330
    {_item_count, nil} =
331
      Repo.delete_all(from(item in ProjectItem, where: item.repository_id in ^repository_ids))
332
333
    {count, nil} =
334
      Repo.delete_all(from(repository in Repository, where: repository.id in ^repository_ids))
335
336
    count
337
  end
338
339
  defp delete_machines([]), do: 0
340
341
  defp delete_machines(machine_ids) do
342
    {_job_count, nil} = Repo.delete_all(from(job in Job, where: job.machine_id in ^machine_ids))
343
    {count, nil} = Repo.delete_all(from(machine in Machine, where: machine.id in ^machine_ids))
344
    count
345
  end
346
347
  defp delete_accounts([]), do: 0
348
349
  defp delete_accounts(account_ids) do
350
    account_ids
351
    |> Enum.sort()
352
    |> Enum.count(fn account_id ->
353
      case Repo.get(User, account_id) do
354
        nil ->
355
          false
356
357
        %User{} = user ->
358
          delete_account_data!(user)
359
360
          case Repo.delete(user) do
361
            {:ok, _deleted} -> true
362
            {:error, _changeset} -> Repo.rollback(:account_delete_failed)
363
          end
364
      end
365
    end)
366
  end
367
368
  defp delete_account_data!(user) do
369
    case Conversations.get_conversation_for_user(user) do
370
      nil ->
371
        :ok
372
373
      conversation ->
374
        owner = Conversations.get_conversation_owner!(conversation)
375
376
        case DataRights.delete(user, owner, conversation) do
377
          {:ok, :deleted} -> :ok
378
          {:error, reason} -> Repo.rollback({:account_data_cleanup_failed, reason})
379
        end
380
    end
381
  end
382
383
  defp normalize_transaction({:ok, result}), do: {:ok, result}
384
  defp normalize_transaction({:error, reason}), do: {:error, reason}
385
386
  defp encode_result(run_id, status, result) do
387
    Jason.encode!(%{
388
      "schema" => "openagents.staging_cleanup.v1",
389
      "run_id" => run_id,
390
      "status" => status,
391
      "registered" => result.registered,
392
      "deleted" => Map.get(result, :deleted)
393
    })
394
  end
395
396
  defp bounded_reason({:account_data_cleanup_failed, reason}) when is_atom(reason),
397
    do: "account_data_#{reason}"
398
399
  defp bounded_reason(reason) when is_atom(reason), do: Atom.to_string(reason)
400
  defp bounded_reason(%Ecto.Changeset{}), do: "registration_conflict"
401
  defp bounded_reason(_reason), do: "cleanup_failed"
402
end
ops/ci/contracts.sh modified +2

@@ -7,10 +7,12 @@ repo_root=$(CDPATH= cd -- "$script_dir/../.." && pwd)

7 7
cd "$repo_root"
8 8
9 9
ops/ci/reference-check.sh
10
sh -n ops/staging/cleanup-run.sh
10 11
elixir ops/ci/docs-check.exs
11 12
MIX_ENV=test mix test --warnings-as-errors \
12 13
  test/openagents/log_safety_test.exs \
13 14
  test/openagents/runtime_config_test.exs \
15
  test/openagents/staging_cleanup_test.exs \
14 16
  test/openagents_web/icon_affordances_test.exs \
15 17
  test/openagents_web/icons_test.exs \
16 18
  test/openagents_web/ui_test.exs
ops/staging/cleanup-run.sh added +61

@@ -0,0 +1,61 @@

1
#!/bin/sh
2
set -eu
3
4
run_id=${1:-}
5
mode=${2:-check}
6
staging_project=${OPENAGENTS_STAGING_PROJECT_ID:-}
7
production_project=${OPENAGENTS_PRODUCTION_PROJECT_ID:-}
8
zone=${OPENAGENTS_STAGING_ZONE:-us-central1-a}
9
10
: "${staging_project:?OPENAGENTS_STAGING_PROJECT_ID is required}"
11
: "${production_project:?OPENAGENTS_PRODUCTION_PROJECT_ID is required}"
12
13
case "$staging_project" in
14
  *stag*) ;;
15
  *) echo "staging project ID must contain 'stag'" >&2; exit 1 ;;
16
esac
17
18
if [ "$staging_project" = "$production_project" ]; then
19
  echo "staging and production project IDs must differ" >&2
20
  exit 1
21
fi
22
23
case "$run_id" in
24
  *[!a-z0-9-]* | "") echo "run ID must contain only lowercase letters, numbers, and hyphens" >&2; exit 64 ;;
25
esac
26
27
run_id_length=${#run_id}
28
if [ "$run_id_length" -lt 8 ] || [ "$run_id_length" -gt 64 ]; then
29
  echo "run ID must contain 8 through 64 characters" >&2
30
  exit 64
31
fi
32
33
case "$run_id" in
34
  [a-z0-9]*) ;;
35
  *) echo "run ID must start with a lowercase letter or number" >&2; exit 64 ;;
36
esac
37
38
case "$mode" in
39
  check) action=check ;;
40
  --apply) action=apply ;;
41
  *) echo "usage: ops/staging/cleanup-run.sh RUN_ID [check|--apply]" >&2; exit 64 ;;
42
esac
43
44
for command_name in gcloud; do
45
  if ! command -v "$command_name" >/dev/null 2>&1; then
46
    echo "$command_name is required" >&2
47
    exit 1
48
  fi
49
done
50
51
gcloud auth print-access-token >/dev/null
52
53
remote_expression="IO.puts(OpenAgents.StagingCleanup.command!(\"$run_id\", \"$action\"))"
54
remote_command="sudo docker exec openagents /app/bin/openagents eval '$remote_expression'"
55
56
gcloud compute ssh openagents-fleet-1 \
57
  --project="$staging_project" \
58
  --zone="$zone" \
59
  --tunnel-through-iap \
60
  --quiet \
61
  --command="$remote_command"
ops/staging/gate-5-profile.sh modified +1

@@ -79,6 +79,7 @@ export OPENAGENTS_RA_DATA_DIR="/var/lib/openagents/ra"

79 79
export OPENAGENTS_RA_EXPECTED_SIZE="3"
80 80
export OPENAGENTS_SECURE_COOKIES="true"
81 81
export OPENAGENTS_STAGING_GATE="5"
82
export OPENAGENTS_STAGING_CLEANUP_ENABLED="false"
82 83
export PHX_HOST="stage.openagents.com"
83 84
export POOL_SIZE="${POOL_SIZE:-10}"
84 85
export VOICE_RECORDING_ENCRYPTION_KEY=""
priv/repo/migrations/20260820150000_create_staging_disposable_resources.exs added +48

@@ -0,0 +1,48 @@

1
defmodule OpenAgents.Repo.Migrations.CreateStagingDisposableResources do
2
  use Ecto.Migration
3
4
  def up do
5
    create table(:staging_disposable_resources, primary_key: false) do
6
      add :id, :binary_id, primary_key: true, default: fragment("gen_random_uuid()")
7
      add :run_id, :string, null: false
8
      add :kind, :string, null: false
9
      add :resource_id, :binary_id, null: false
10
11
      add :inserted_at, :utc_datetime_usec,
12
        null: false,
13
        default: fragment("timezone('utc', now())")
14
    end
15
16
    create index(:staging_disposable_resources, [:run_id])
17
    create unique_index(:staging_disposable_resources, [:kind, :resource_id])
18
19
    create constraint(:staging_disposable_resources, :staging_disposable_run_id_check,
20
             check: "run_id ~ '^[a-z0-9][a-z0-9-]{7,63}$'"
21
           )
22
23
    create constraint(:staging_disposable_resources, :staging_disposable_kind_check,
24
             check: "kind IN ('account', 'machine', 'recording', 'repository')"
25
           )
26
27
    execute("""
28
    CREATE FUNCTION prevent_staging_disposable_resource_update()
29
    RETURNS trigger AS $$
30
    BEGIN
31
      RAISE EXCEPTION 'staging disposable resource registrations are immutable';
32
    END;
33
    $$ LANGUAGE plpgsql;
34
    """)
35
36
    execute("""
37
    CREATE TRIGGER staging_disposable_resources_prevent_update
38
    BEFORE UPDATE ON staging_disposable_resources
39
    FOR EACH ROW
40
    EXECUTE FUNCTION prevent_staging_disposable_resource_update();
41
    """)
42
  end
43
44
  def down do
45
    drop table(:staging_disposable_resources)
46
    execute("DROP FUNCTION prevent_staging_disposable_resource_update()")
47
  end
48
end
test/openagents/runtime_config_test.exs modified +24

@@ -65,6 +65,30 @@ defmodule OpenAgents.RuntimeConfigTest do

65 65
             |> RuntimeConfig.validate()
66 66
  end
67 67
68
  test "staging cleanup is admitted only at Gate 12 or later" do
69
    exact_identity = %{
70
      build_revision: String.duplicate("a", 40),
71
      image_digest: "sha256:" <> String.duplicate("b", 64)
72
    }
73
74
    assert {:ok, _config} =
75
             staging_settings()
76
             |> Map.merge(exact_identity)
77
             |> Map.merge(%{staging_gate: 12, staging_cleanup_enabled: true})
78
             |> RuntimeConfig.validate()
79
80
    assert {:error, %{setting: :staging_cleanup_enabled}} =
81
             staging_settings()
82
             |> Map.merge(exact_identity)
83
             |> Map.merge(%{staging_gate: 11, staging_cleanup_enabled: true})
84
             |> RuntimeConfig.validate()
85
86
    assert {:error, %{setting: :staging_cleanup_enabled}} =
87
             staging_settings()
88
             |> Map.put(:staging_cleanup_enabled, "true")
89
             |> RuntimeConfig.validate()
90
  end
91
68 92
  test "fleet deployment requires the isolated GCP rolling provider" do
69 93
    settings =
70 94
      staging_settings()
test/openagents/staging_cleanup_test.exs added +263

@@ -0,0 +1,263 @@

1
defmodule OpenAgents.StagingCleanupTest do
2
  use OpenAgents.DataCase, async: false
3
4
  import OpenAgents.AccountsFixtures
5
6
  alias OpenAgents.Conversations
7
  alias OpenAgents.Machines.Machine
8
  alias OpenAgents.ProjectFields.ProjectField
9
  alias OpenAgents.Projects
10
  alias OpenAgents.Repo
11
  alias OpenAgents.Repositories
12
  alias OpenAgents.Repositories.Repository
13
  alias OpenAgents.Staging.DisposableResource
14
  alias OpenAgents.StagingCleanup
15
  alias OpenAgents.Voice
16
  alias OpenAgents.Voice.{Config, Recording, Recordings}
17
  alias OpenAgents.Work.Job
18
19
  @run_id "gate14-cleanup-0001"
20
21
  setup do
22
    original_enabled = Application.get_env(:openagents, :staging_cleanup_enabled)
23
    original_environment = Application.get_env(:openagents, :runtime_environment)
24
    original_gate = Application.get_env(:openagents, :staging_gate)
25
26
    Application.put_env(:openagents, :staging_cleanup_enabled, true)
27
    Application.put_env(:openagents, :runtime_environment, :test)
28
    Application.put_env(:openagents, :staging_gate, 0)
29
30
    on_exit(fn ->
31
      restore_env(:staging_cleanup_enabled, original_enabled)
32
      restore_env(:runtime_environment, original_environment)
33
      restore_env(:staging_gate, original_gate)
34
    end)
35
36
    :ok
37
  end
38
39
  test "one run removes only its registered accounts, repositories, recordings, and machines" do
40
    user = repository_user_fixture("cleanup-target")
41
    unrelated_user = repository_user_fixture("cleanup-unrelated")
42
    repository = repository_fixture(user)
43
    project = project_fixture(repository, user)
44
    field = project_field_fixture(project)
45
    machine = machine_fixture(user)
46
    recording = recording_fixture(user)
47
48
    for {kind, resource_id} <- [
49
          {:account, user.id},
50
          {:repository, repository.id},
51
          {:machine, machine.id},
52
          {:recording, recording.id}
53
        ] do
54
      assert {:ok, %DisposableResource{}} = StagingCleanup.register(@run_id, kind, resource_id)
55
    end
56
57
    assert {:ok,
58
            %{
59
              registered: %{
60
                "account" => 1,
61
                "machine" => 1,
62
                "recording" => 1,
63
                "repository" => 1
64
              }
65
            }} = StagingCleanup.preview(@run_id)
66
67
    checked = @run_id |> StagingCleanup.command!("check") |> Jason.decode!()
68
    assert checked["schema"] == "openagents.staging_cleanup.v1"
69
    assert checked["status"] == "checked"
70
    assert checked["deleted"] == nil
71
72
    assert {:ok, result} = StagingCleanup.cleanup(@run_id)
73
    assert result.registered == result.deleted
74
75
    assert Repo.get(OpenAgents.Accounts.User, user.id) == nil
76
    assert Repo.get(Repository, repository.id) == nil
77
    assert Repo.get(Machine, machine.id) == nil
78
    assert Repo.get(Recording, recording.id) == nil
79
    assert Repo.get(ProjectField, field.id) == nil
80
    assert Repo.get(OpenAgents.Accounts.User, unrelated_user.id)
81
    assert Repositories.initial_repository!()
82
    assert Repo.aggregate(DisposableResource, :count) == 0
83
84
    assert {:ok, %{registered: empty}} = StagingCleanup.preview(@run_id)
85
    assert Enum.all?(empty, fn {_kind, count} -> count == 0 end)
86
  end
87
88
  test "registration refuses the canonical repository and administrator accounts" do
89
    assert {:error, :canonical_repository_forbidden} =
90
             StagingCleanup.register(@run_id, :repository, Repositories.initial_repository!().id)
91
92
    user = repository_user_fixture("cleanup-admin")
93
    original_ids = Application.get_env(:openagents, :admin_github_ids)
94
    Application.put_env(:openagents, :admin_github_ids, [user.github_id])
95
    on_exit(fn -> restore_env(:admin_github_ids, original_ids) end)
96
97
    assert {:error, :administrator_account_forbidden} =
98
             StagingCleanup.register(@run_id, :account, user.id)
99
  end
100
101
  test "cleanup refuses an account that owns a project outside the registered repositories" do
102
    user = repository_user_fixture("cleanup-project-owner")
103
    project = project_fixture(Repositories.initial_repository!(), user)
104
    assert {:ok, _registration} = StagingCleanup.register(@run_id, :account, user.id)
105
106
    assert {:error, :account_owns_unregistered_project} = StagingCleanup.cleanup(@run_id)
107
    assert Repo.get(OpenAgents.Accounts.User, user.id)
108
    assert Repo.get(OpenAgents.Projects.Project, project.id)
109
    assert Repo.get_by(DisposableResource, run_id: @run_id)
110
  end
111
112
  test "cleanup refuses an account that owns an unregistered machine" do
113
    user = repository_user_fixture("cleanup-machine-owner")
114
    machine = machine_fixture(user)
115
    assert {:ok, _registration} = StagingCleanup.register(@run_id, :account, user.id)
116
117
    assert {:error, :account_owns_unregistered_machine} = StagingCleanup.cleanup(@run_id)
118
    assert Repo.get(OpenAgents.Accounts.User, user.id)
119
    assert Repo.get(Machine, machine.id)
120
  end
121
122
  test "cleanup refuses an account that owns an unregistered recording" do
123
    user = repository_user_fixture("cleanup-recording-owner")
124
    recording = recording_fixture(user)
125
    assert {:ok, _registration} = StagingCleanup.register(@run_id, :account, user.id)
126
127
    assert {:error, :account_owns_unregistered_recording} = StagingCleanup.cleanup(@run_id)
128
    assert Repo.get(OpenAgents.Accounts.User, user.id)
129
    assert Repo.get(Recording, recording.id)
130
  end
131
132
  test "cleanup refuses an online machine" do
133
    user = repository_user_fixture("cleanup-online-machine")
134
    machine = machine_fixture(user)
135
    assert {:ok, _registration} = StagingCleanup.register(@run_id, :machine, machine.id)
136
    assert {:ok, _pid} = OpenAgents.Computer.register(machine.id)
137
    on_exit(fn -> OpenAgents.Computer.unregister(machine.id) end)
138
139
    assert {:error, :machine_online} = StagingCleanup.cleanup(@run_id)
140
    assert Repo.get(Machine, machine.id)
141
  end
142
143
  test "cleanup refuses a machine with queued work" do
144
    user = repository_user_fixture("cleanup-active-work")
145
    machine = machine_fixture(user)
146
    {:ok, conversation} = Conversations.ensure_conversation(user)
147
    owner = Conversations.get_conversation_owner!(conversation)
148
149
    job =
150
      %Job{}
151
      |> Job.create_changeset(%{
152
        conversation_id: conversation.id,
153
        owner_visitor_id: owner.id,
154
        machine_id: machine.id,
155
        surface: "text",
156
        goal: "Keep this staging machine busy"
157
      })
158
      |> Repo.insert!()
159
160
    assert {:ok, _registration} = StagingCleanup.register(@run_id, :machine, machine.id)
161
    assert {:error, :machine_has_active_work} = StagingCleanup.cleanup(@run_id)
162
    assert Repo.get(Machine, machine.id)
163
    assert Repo.get(Job, job.id)
164
  end
165
166
  test "cleanup refuses account data with an active voice session" do
167
    user = repository_user_fixture("cleanup-active-voice")
168
    {:ok, conversation} = Conversations.ensure_conversation(user)
169
    {:ok, _session} = Voice.admit_session(conversation, enabled_voice_config())
170
    assert {:ok, _registration} = StagingCleanup.register(@run_id, :account, user.id)
171
172
    assert {:error, {:account_data_cleanup_failed, :voice_session_in_progress}} =
173
             StagingCleanup.cleanup(@run_id)
174
175
    assert Repo.get(OpenAgents.Accounts.User, user.id)
176
  end
177
178
  test "cleanup is unavailable unless the staging-only feature is admitted" do
179
    user = repository_user_fixture("cleanup-disabled")
180
    Application.put_env(:openagents, :staging_cleanup_enabled, false)
181
182
    assert {:error, :staging_cleanup_not_admitted} =
183
             StagingCleanup.register(@run_id, :account, user.id)
184
185
    Application.put_env(:openagents, :staging_cleanup_enabled, true)
186
    Application.put_env(:openagents, :runtime_environment, :production)
187
188
    assert {:error, :staging_cleanup_not_admitted} = StagingCleanup.preview(@run_id)
189
  end
190
191
  defp repository_fixture(user) do
192
    {:ok, repository} =
193
      Repositories.create_repository(%{
194
        owner: "OpenAgentsStaging",
195
        name: "cleanup-#{System.unique_integer([:positive])}",
196
        visibility: "private",
197
        default_branch: "main"
198
      })
199
200
    {:ok, _membership} = Repositories.add_member(repository, user, "owner")
201
    repository
202
  end
203
204
  defp project_fixture(repository, user) do
205
    {:ok, project} =
206
      Projects.create_project(
207
        repository,
208
        %{title: "Disposable cleanup project", owner: user.github_login},
209
        user
210
      )
211
212
    project
213
  end
214
215
  defp project_field_fixture(project) do
216
    {:ok, field} =
217
      Projects.create_project_field(%{
218
        name: "Status",
219
        data_type: "single_select",
220
        options: %{},
221
        project_id: project.id
222
      })
223
224
    field
225
  end
226
227
  defp machine_fixture(user) do
228
    %Machine{user_id: user.id}
229
    |> Machine.create_changeset(%{name: "Disposable machine", tier: "probe", roots: []})
230
    |> Ecto.Changeset.change(
231
      token_digest: :crypto.hash(:sha256, "disposable-machine-token"),
232
      token_expires_at: DateTime.add(DateTime.utc_now(), 3_600, :second)
233
    )
234
    |> Repo.insert!()
235
  end
236
237
  defp recording_fixture(user) do
238
    {:ok, conversation} = Conversations.ensure_conversation(user)
239
    {:ok, session} = Voice.admit_session(conversation, enabled_voice_config())
240
241
    {:ok, _recording} =
242
      Recordings.append_chunk(session, session.generation, 1, "audio", "audio/webm")
243
244
    {:ok, recording} = Recordings.finalize(session, session.generation, "complete", 500)
245
    {:ok, _session} = Voice.end_session(session, session.generation, "user_ended")
246
    recording
247
  end
248
249
  defp enabled_voice_config do
250
    Config.build!(
251
      enabled: true,
252
      architecture: :openai_realtime,
253
      provider: "openai",
254
      model: "gpt-realtime-2.1",
255
      voice: "marin",
256
      reasoning_effort: "low",
257
      maximum_session_seconds: 3_000
258
    )
259
  end
260
261
  defp restore_env(key, nil), do: Application.delete_env(:openagents, key)
262
  defp restore_env(key, value), do: Application.put_env(:openagents, key, value)
263
end

This page updates live while a promote is in flight · changelog