OpenAgents

Commit

Build isolated staging infrastructure

553c0ee1d8dd · Christopher David · · parent bfa752191887 

Build isolated staging infrastructure

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • modified .gitignore
  • modified Dockerfile
  • modified README.md
  • modified config/config.exs
  • modified config/runtime.exs
  • modified docs/2026-08-20-integration-hardening-and-staging-readiness-recommendations.md
  • modified docs/decisions/0006-isolate-web-and-distributed-fleet-staging.md
  • modified docs/runtime-configuration.md
  • modified docs/security/secrets-and-log-handling.md
  • added infra/staging/.terraform.lock.hcl
  • added infra/staging/README.md
  • added infra/staging/main.tf
  • added infra/staging/outputs.tf
  • added infra/staging/templates/deployer-startup.sh.tftpl
  • added infra/staging/templates/fleet-startup.sh.tftpl
  • added infra/staging/tests/safety.tftest.hcl
  • added infra/staging/variables.tf
  • added infra/staging/versions.tf
  • modified lib/openagents/build_info.ex
  • modified lib/openagents/cluster.ex
  • added lib/openagents/cluster/admission.ex
  • modified lib/openagents/forge/gate_receipt.ex
  • added lib/openagents/forge/rolling_node_probe.ex
  • modified lib/openagents/forge/rolling_provider.ex
  • added lib/openagents/forge/rolling_provider/gcp.ex
  • added lib/openagents/forge/rolling_provider/gcp/compute.ex
  • added lib/openagents/forge/rolling_provider/gcp/deployer.ex
  • modified lib/openagents/forge/rolling_replacement.ex
  • modified lib/openagents/runtime_config.ex
  • modified lib/openagents_web/controllers/health_controller.ex
  • modified ops/ci/gate.sh
  • added ops/ci/staging-infra.sh
  • modified ops/deploy/build-image.sh
  • added ops/staging/bootstrap-project.sh
  • modified ops/staging/gate-5-profile.sh
  • added ops/staging/terraform.sh
  • added ops/staging/validate-isolation.sh
  • modified test/openagents/cluster_test.exs
  • modified test/openagents/forge/gate_receipt_test.exs
  • added test/openagents/forge/rolling_provider/gcp/compute_test.exs
  • added test/openagents/forge/rolling_provider/gcp_test.exs
  • modified test/openagents/runtime_config_test.exs
  • modified test/openagents_web/controllers/health_controller_test.exs
  • added test/support/openagents/test/rolling_gcp_driver.ex

Diff

44 files changed, +2845 -18

.gitignore modified +9 -1

@@ -35,6 +35,14 @@ openagents-*.tar

35 35 
npm-debug.log
36 36 
/assets/node_modules/
37 37
38 
# Terraform working data, plans, variable values, and state are local only.
39 
**/.terraform/
40 
*.tfplan
41 
*.tfstate
42 
*.tfstate.*
43 
*.tfvars
44 
*.tfvars.json
45 
!.terraform.lock.hcl
46
38 47 
# Local environment variables
39 48 
/.env
40
Dockerfile modified +3

@@ -14,6 +14,9 @@ ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}"

14 14
15 15 
FROM ${BUILDER_IMAGE} AS builder
16 16
17 
ARG OPENAGENTS_BUILD_REVISION="image"
18 
ENV OPENAGENTS_BUILD_REVISION=${OPENAGENTS_BUILD_REVISION}
19
17 20 
# install build dependencies
18 21 
RUN apt-get update \
19 22 
  && apt-get install -y --no-install-recommends build-essential git \
README.md modified +9

@@ -100,6 +100,15 @@ Before pushing a release candidate, provision a disposable database and run

100 100 
[the release deployment fallback runbook](docs/operations/release-deployment-fallbacks.md).
101 101 
This repository deliberately has no hosted CI configuration.
102 102
103 
Validate the isolated staging infrastructure without changing cloud state:
104
105 
```sh
106 
ops/staging/terraform.sh validate
107 
```
108
109 
Read the [isolated staging infrastructure runbook](infra/staging/README.md)
110 
before you bootstrap, plan, or apply any staging resource.
111
103 112 
## Contributing and source control
104 113
105 114 
Read `AGENTS.md` before changing the application. GitHub is temporarily the
config/config.exs modified +3

@@ -34,6 +34,8 @@ config :openagents,

34 34 
  runtime_environment: :development,
35 35 
  staging_gate: 0,
36 36 
  production_deploy_enabled: false,
37 
  build_revision: "image",
38 
  image_digest: nil,
37 39 
  migrate_on_boot: false,
38 40 
  secure_cookies: false,
39 41 
  https_aliases: [],

@@ -209,6 +211,7 @@ config :openagents,

209 211 
  forge_wal_dir: nil,
210 212 
  forge_wal_bucket: nil,
211 213 
  forge_gcs_token_provider: nil,
214 
  forge_rolling_provider: nil,
212 215 
  # Hot-load allowlist: MODULE names, not repo paths. An entry ending in `.`
213 216 
  # is a prefix; any other entry is an exact module name (see
214 217 
  # `OpenAgents.Forge.HotLoader.allowlisted?/2`). The narrow list was never
config/runtime.exs modified +35

@@ -219,6 +219,28 @@ if config_env() == :prod do

219 219 
      _invalid -> raise "environment variable OPENAGENTS_FORGE_BUILD_EXECUTOR is not admitted"
220 220 
    end
221 221
222 
  forge_rolling_provider =
223 
    case optional_text.("OPENAGENTS_FORGE_ROLLING_PROVIDER") do
224 
      nil -> nil
225 
      "gcp" -> OpenAgents.Forge.RollingProvider.Gcp
226 
      _invalid -> raise "environment variable OPENAGENTS_FORGE_ROLLING_PROVIDER is not admitted"
227 
    end
228
229 
  rolling_instances =
230 
    case optional_text.("OPENAGENTS_GCP_ROLLING_INSTANCES_JSON") do
231 
      nil ->
232 
        %{}
233
234 
      encoded ->
235 
        case Jason.decode(encoded) do
236 
          {:ok, instances} when is_map(instances) ->
237 
            instances
238
239 
          _invalid ->
240 
            raise "environment variable OPENAGENTS_GCP_ROLLING_INSTANCES_JSON must be a JSON object"
241 
        end
242 
    end
243
222 244 
  distribution_enabled = System.get_env("RELEASE_DISTRIBUTION") in ["name", "longnames"]
223 245 
  release_node = optional_text.("RELEASE_NODE")
224 246 
  release_cookie = optional_text.("RELEASE_COOKIE")

@@ -239,6 +261,8 @@ if config_env() == :prod do

239 261 
    runtime_environment: runtime_environment,
240 262 
    staging_gate: staging_gate,
241 263 
    production_deploy_enabled: production_deploy_enabled,
264 
    build_revision: OpenAgents.BuildInfo.revision(),
265 
    image_digest: optional_text.("OPENAGENTS_IMAGE_DIGEST"),
242 266 
    secure_cookies: secure_cookies,
243 267 
    https_aliases: https_aliases,
244 268 
    migrate_on_boot: migrate_on_boot,

@@ -270,6 +294,7 @@ if config_env() == :prod do

270 294 
    forge_enabled: forge_enabled,
271 295 
    forge_deploy_lane_enabled: forge_deploy_enabled,
272 296 
    forge_boot_converge_enabled: boot_convergence_enabled,
297 
    forge_rolling_provider: forge_rolling_provider,
273 298 
    forge_repos: forge_repos,
274 299 
    forge_repo_owners: Map.new(forge_repos, &{&1, forge_owner}),
275 300 
    forge_public_visibility: Map.new(forge_repos, &{&1, :l3}),

@@ -302,6 +327,16 @@ if config_env() == :prod do

302 327 
    dns_cluster_query: optional_text.("DNS_CLUSTER_QUERY"),
303 328 
    distribution: distribution
304 329
330 
  config :openagents, OpenAgents.Forge.RollingProvider.Gcp,
331 
    project_id: optional_text.("OPENAGENTS_GCP_ROLLING_PROJECT_ID"),
332 
    production_project_id: optional_text.("OPENAGENTS_PRODUCTION_PROJECT_ID"),
333 
    zone: optional_text.("OPENAGENTS_GCP_ROLLING_ZONE"),
334 
    instances: rolling_instances,
335 
    image_repository: optional_text.("OPENAGENTS_GCP_IMAGE_REPOSITORY"),
336 
    deployer_node: :"openagents-deployer@openagents-deployer.staging.internal",
337 
    rpc_timeout_ms: parse_integer.("OPENAGENTS_GCP_ROLLING_RPC_TIMEOUT_MS", 1_000..120_000),
338 
    compute_timeout_ms: parse_integer.("OPENAGENTS_GCP_COMPUTE_TIMEOUT_MS", 30_000..600_000)
339
305 340 
  config :openagents, OpenAgents.Repo, repo_config
306 341
307 342 
  config :openagents, OpenAgentsWeb.Endpoint,
docs/2026-08-20-integration-hardening-and-staging-readiness-recommendations.md modified +41 -2

@@ -2,7 +2,7 @@

2 2
3 3 
Date: 2026-08-20
4 4
5 
Status: In progress; Gates 0–5 and 7–10 complete, Gate 6 application controls locally verified
5 
Status: In progress; Gates 0–11 complete locally, Gate 12 cloud isolation proof pending
6 6
7 7 
## Outcome
8 8

@@ -1208,6 +1208,45 @@ as evidence for an OTP relup.

1208 1208 
distributed deployment mechanism without touching production state or sharing
1209 1209 
production's database capacity and failure domain.
1210 1210
1211 
### Gate 12 implementation status
1212
1213 
Implemented locally on 2026-08-20:
1214
1215 
- Added a Terraform root for a dedicated staging project. It defines a private
1216 
  VPC and DNS zone, a separate private-IP Cloud SQL instance, three stable fleet
1217 
  nodes without public IP addresses, durable node-local state disks, a private
1218 
  deployer controller, Cloud NAT, staging-only buckets, Secret Manager
1219 
  placeholders, Artifact Registry, and split workload identities.
1220 
- Added Terraform safety tests that require a project marked as staging, refuse
1221 
  the production project, preserve Cloud SQL deletion protection, and prove
1222 
  that the fleet and deployer have no public access configuration.
1223 
- Added Terraform formatting, initialization, validation, and safety tests as a
1224 
  required stage of every exact-SHA release gate receipt.
1225 
- Added exact-SHA plan and apply wrappers. The wrappers require a clean
1226 
  worktree, a protected remote state bucket, distinct staging and production
1227 
  project IDs, valid Application Default Credentials, and an explicit apply
1228 
  confirmation.
1229 
- Added a content-free isolation validator. It compares project numbers,
1230 
  verifies the private database, its separate application role, and the fleet,
1231 
  checks staging-only storage, secrets, identities, DNS, and networking, and
1232 
  rejects production service accounts in staging IAM. The database password is
1233 
  accepted only as a Terraform ephemeral write-only value and never enters the
1234 
  plan or state.
1235 
- Added exact packaged source and image identity, node-local admission fencing,
1236 
  a bounded rolling node probe, and a Google Cloud rolling provider. The
1237 
  provider uses private Erlang distribution for drain and health checks. A
1238 
  minimal private deployer BEAM node performs exact instance metadata updates
1239 
  and resets under the only identity that holds those permissions; it does not
1240 
  start the application, join Ra, open HTTP, or connect to PostgreSQL.
1241
1242 
The local infrastructure definition and mocked safety tests pass. The cloud
1243 
apply, isolation receipt, and one-command disposable-run cleanup proof remain
1244 
open; cloud work is blocked until the operator refreshes the expired Google
1245 
Cloud CLI and Application Default Credentials. No staging or production cloud
1246 
resource changed during this implementation step. See the [isolated staging
1247 
infrastructure](../infra/staging/README.md) for the exact bootstrap, plan,
1248 
apply, and validation procedure.
1249
1211 1250 
## Gate 13: Deploy to staging reproducibly
1212 1251
1213 1252 
Use this sequence for every staging candidate:

@@ -1516,7 +1555,7 @@ each handoff.

1516 1555 
- [x] Fleet deployment is transactional and rolls back every affected node.
1517 1556 
- [x] Boot convergence controls readiness.
1518 1557 
- [ ] Relup and rolling replacement pass their staging drills.
1519 
- [ ] Owned local gates produce exact-SHA receipts.
1558 
- [x] Owned local gates produce exact-SHA receipts.
1520 1559 
- [ ] Web and distributed staging are isolated from production.
1521 1560 
- [ ] Staging has a separate database instance and failure domain.
1522 1561 
- [ ] The migration lineage is mapped and rehearsed for every nonempty target.
docs/decisions/0006-isolate-web-and-distributed-fleet-staging.md modified +1 -1

@@ -2,7 +2,7 @@

2 2
3 3 
Date: 2026-08-20
4 4
5 
Status: Accepted; implementation pending
5 
Status: Accepted; reproducible infrastructure implemented; cloud apply pending
6 6
7 7 
## Context
8 8
docs/runtime-configuration.md modified +9

@@ -54,6 +54,7 @@ URLs, receipts, or checked-in environment files.

54 54 
| Release | `OPENAGENTS_ENVIRONMENT` | `staging`; `production` remains separately locked |
55 55 
| Release | `OPENAGENTS_STAGING_GATE` | Integer `0` through `16`; feature admission is tied to it |
56 56 
| Release | `OPENAGENTS_PRODUCTION_DEPLOY_ENABLED` | `false` until a later production decision |
57 
| Release | `OPENAGENTS_IMAGE_DIGEST` | Exact `sha256:` image digest at staging Gate 12 and later; empty before that gate |
57 58 
| Endpoint | `PHX_HOST` | Exactly `stage.openagents.com` in staging |
58 59 
| Endpoint | `OPENAGENTS_ALLOWED_ORIGINS` | Comma-separated exact HTTPS origins including `https://stage.openagents.com` |
59 60 
| Endpoint | `OPENAGENTS_HTTPS_ALIASES` | Comma-separated hostnames; empty means no aliases |

@@ -140,6 +141,14 @@ discovery, node identity, cookie, and bounded distribution ports.

140 141 
| `OPENAGENTS_FORGE_DEPLOY_TOKEN_TTL_MS` | `120000`; admitted range 30 seconds to 30 minutes and at least eight deployment timeouts |
141 142 
| `OPENAGENTS_FORGE_BOOT_RETRY_MIN_MS` | `1000`; admitted range 100 milliseconds to one minute |
142 143 
| `OPENAGENTS_FORGE_BOOT_RETRY_MAX_MS` | `30000`; admitted range one to 300 seconds and not less than the minimum |
144 
| `OPENAGENTS_FORGE_ROLLING_PROVIDER` | `gcp` when fleet deployment is enabled; empty while disabled |
145 
| `OPENAGENTS_GCP_ROLLING_PROJECT_ID` | Isolated staging project; must differ from `OPENAGENTS_PRODUCTION_PROJECT_ID` |
146 
| `OPENAGENTS_PRODUCTION_PROJECT_ID` | Production project used only as a fail-closed comparison value |
147 
| `OPENAGENTS_GCP_ROLLING_ZONE` | Zone that contains the three stable staging instances |
148 
| `OPENAGENTS_GCP_ROLLING_INSTANCES_JSON` | Exact JSON map from three BEAM node names to three staging instance names |
149 
| `OPENAGENTS_GCP_IMAGE_REPOSITORY` | Staging Artifact Registry repository and image path without a tag or digest |
150 
| `OPENAGENTS_GCP_ROLLING_RPC_TIMEOUT_MS` | Bounded private node-probe timeout from one to 120 seconds |
151 
| `OPENAGENTS_GCP_COMPUTE_TIMEOUT_MS` | Bounded private deployer call timeout from 30 seconds to 10 minutes |
143 152 
| `OPENAGENTS_CODING_JOBS_DIR` | Absolute durable path outside `/tmp` when work or computers are enabled |
144 153 
| `OPENAGENTS_RA_DATA_DIR` | Absolute durable path outside `/tmp` when Ra is enabled |
145 154 
| `OPENAGENTS_RA_EXPECTED_SIZE` | At least `3` when Ra is enabled |
docs/security/secrets-and-log-handling.md modified +11 -9

@@ -12,6 +12,8 @@ more than one role on a node:

12 12 
| Runtime identity | Purpose | May read |
13 13 
| --- | --- | --- |
14 14 
| `openagents-staging-web` | Phoenix release and forge Git endpoint | Web, provider, OAuth, vault, recording, forge verification, database, and cluster secrets listed below |
15 
| `openagents-staging-fleet` | Three-node distributed application lane | The application secrets required by the same candidate plus the cluster cookie; no Compute mutation permission |
16 
| `openagents-staging-deployer` | Minimal private rolling-replacement controller | Cluster cookie only; Compute mutation comes from its bounded workload identity |
15 17 
| `openagents-staging-migrator` | Release migration and token rewrap job | Database URL, active GitHub vault key, prior GitHub vault keyring |
16 18 
| `openagents-staging-builder` | Isolated BEAM build sidecar | Forge operator token only; use an askpass helper, never a URL or argv value |
17 19 
| `openagents-staging-operator` | Human-triggered staging operations | No application secrets by default; short-lived platform access to invoke jobs and read redacted logs |

@@ -28,15 +30,15 @@ use distinct names and values and remains locked.

28 30
29 31 
| Environment input | Staging secret name | Readers | Rotation trigger |
30 32 
| --- | --- | --- | --- |
31 
| `DATABASE_URL` | `openagents-staging-database-url` | web, migrator | Database credential rotation or suspected log/process exposure |
32 
| `SECRET_KEY_BASE` | `openagents-staging-secret-key-base` | web | Suspected exposure; rotation invalidates browser sessions |
33 
| `GITHUB_CLIENT_SECRET` | `openagents-staging-github-client-secret` | web | OAuth app rotation or suspected exposure |
34 
| `GITHUB_TOKEN_ENCRYPTION_KEY` | `openagents-staging-github-vault-active` | web, migrator | Scheduled vault rotation or suspected exposure |
35 
| `GITHUB_TOKEN_DECRYPTION_KEYS_JSON` | `openagents-staging-github-vault-previous` | web, migrator, only during rewrap | Delete after every row uses the active key ID |
36 
| `OPENAI_API_KEY` | `openagents-staging-openai-api-key` | web | Provider rotation or suspected prompt/log exposure |
37 
| `VOICE_RECORDING_ENCRYPTION_KEY` | `openagents-staging-voice-recording-key` | web when recording is admitted | Scheduled recording-key procedure or suspected exposure |
38 
| `OPENAGENTS_FORGE_OPERATOR_TOKEN` | `openagents-staging-forge-operator-token` | web, builder | Scheduled rotation, builder replacement, or suspected URL/argv/log exposure |
39 
| `RELEASE_COOKIE` | `openagents-staging-release-cookie` | web fleet nodes | Fleet-wide coordinated rotation or suspected exposure |
33 
| `DATABASE_URL` | `openagents-staging-database-url` | web, fleet, migrator | Database credential rotation or suspected log/process exposure |
34 
| `SECRET_KEY_BASE` | `openagents-staging-secret-key-base` | web, fleet | Suspected exposure; rotation invalidates browser sessions |
35 
| `GITHUB_CLIENT_SECRET` | `openagents-staging-github-client-secret` | web, fleet | OAuth app rotation |
36 
| `GITHUB_TOKEN_ENCRYPTION_KEY` | `openagents-staging-github-vault-active` | web, fleet, migrator | Scheduled vault rotation or suspected exposure |
37 
| `GITHUB_TOKEN_DECRYPTION_KEYS_JSON` | `openagents-staging-github-vault-previous` | web, fleet, migrator, only during rewrap | Delete after every row uses the active key ID |
38 
| `OPENAI_API_KEY` | `openagents-staging-openai-api-key` | web, fleet | Provider rotation or suspected prompt/log exposure |
39 
| `VOICE_RECORDING_ENCRYPTION_KEY` | `openagents-staging-voice-recording-key` | web and fleet when recording is admitted | Scheduled recording-key procedure or suspected exposure |
40 
| `OPENAGENTS_FORGE_OPERATOR_TOKEN` | `openagents-staging-forge-operator-token` | web, fleet, builder | Scheduled rotation, builder replacement, or suspected URL/argv/log exposure |
41 
| `RELEASE_COOKIE` | `openagents-staging-release-cookie` | web, fleet, deployer | Fleet-wide coordinated rotation or suspected exposure |
40 42
41 43 
`GITHUB_CLIENT_ID` and `GITHUB_TOKEN_ENCRYPTION_KEY_ID` are identifiers, not
42 44 
secrets. `DB_PASSWORD` is not used by the admitted staging profile because it
infra/staging/.terraform.lock.hcl added +22

@@ -0,0 +1,22 @@

1 
# This file is maintained automatically by "terraform init".
2 
# Manual edits may be lost in future updates.
3
4 
provider "registry.terraform.io/hashicorp/google" {
5 
  version     = "7.45.0"
6 
  constraints = "~> 7.41"
7 
  hashes = [
8 
    "h1:5bwzwKa/bvJmUkVMkrF18v9AfFeJ/wjR230oY+4LHrc=",
9 
    "zh:0f33485d068e39b1661d4ad789dfac353134b99cb5746e463e7719c053d70c06",
10 
    "zh:162335c448b15125924c3451dc0a411635138c00e36784d9586b9df61bc8a3d7",
11 
    "zh:172c9e4902e9a01b1111bbbb9063a47804ec5f4207b2221df9c398239ff3b350",
12 
    "zh:39acce0806f1aeca106ea529053778681e332f3f895c21c6a6a9fb267bed1058",
13 
    "zh:58dadd7b96b7b706e2995a04f356862be7cfd6b55f6257464c8b10986abf8efa",
14 
    "zh:7230a5e49abed243317b9de0ee7fd365b4a2775b532a5d66006b39c4703978ac",
15 
    "zh:a1b60814ee9a0726c0cd43d36da2cee3db97ec296ee66b111a7e603997de9122",
16 
    "zh:a454c09162ccd342e101704dca4b0eedc2d11939ac48ebd7b7d505a96c1933e6",
17 
    "zh:ad6fdf73e072f510a7d8e2b70f9e1021d08b9076eae146b420f333905708c982",
18 
    "zh:de1de65b909c99430f8bef092718b75dc5c777e98daf474a2215621f35095bb8",
19 
    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
20 
    "zh:fb1b9d1ea7bc79b7409f02aa7c19ba39afa22dbead69e83ae7eb2691ac5c2426",
21 
  ]
22 
}
infra/staging/README.md added +125

@@ -0,0 +1,125 @@

1 
# Isolated staging infrastructure
2
3 
This Terraform root creates the infrastructure boundary required by Gate 12.
4 
It targets a dedicated Google Cloud project and refuses a project ID that
5 
matches production. It does not create, read, or modify production resources.
6
7 
## Topology
8
9 
The configuration creates these staging-only resources:
10
11 
- A custom VPC, private subnet, private DNS zone, Cloud NAT, and logged firewall
12 
  rules.
13 
- One private-IP Cloud SQL for PostgreSQL instance with its own connection
14 
  budget, backups, and deletion protection.
15 
- Three stable Compute Engine instances without public IP addresses. Each node
16 
  has a fixed private address and a separate durable state disk for Ra, forge,
17 
  build-queue, artifact-cache, and job data.
18 
- Separate web, fleet, and deployer service accounts. A dedicated private
19 
  deployer VM runs a minimal BEAM node and receives only the permissions to
20 
  read, reset, and update metadata on staging instances. It does not start the
21 
  application, join Ra, open HTTP, or connect to PostgreSQL, and it has no
22 
  production project authority.
23 
- Separate non-secret configuration placeholders for the web, fleet, and
24 
  builder lanes, plus one Secret Manager resource per credential in the
25 
  staging secret inventory. The deployer can read only the release cookie.
26 
  Terraform never creates a secret version or stores a credential in state.
27 
- Separate buckets for forge artifacts, forge WAL, recordings, and evidence.
28 
- One Artifact Registry repository for digest-addressed application and builder
29 
  images.
30
31 
The instances start fenced. They do not run an application until an operator
32 
assigns exact application and builder image digests and creates the required
33 
staging-only secret versions during Gate 13.
34
35 
## Prerequisites
36
37 
You need these local tools:
38
39 
- Google Cloud CLI with an active operator login and Application Default
40 
  Credentials.
41 
- Terraform 1.11 or later.
42 
- `jq`.
43
44 
Select a globally unique staging project ID that contains `stag`. Export the
45 
production project ID only as a comparison value. The scripts never select it
46 
as an action target.
47
48 
```sh
49 
export OPENAGENTS_STAGING_PROJECT_ID='openagents-staging-UNIQUE'
50 
export OPENAGENTS_PRODUCTION_PROJECT_ID='PRODUCTION_PROJECT_ID'
51 
export OPENAGENTS_STAGING_BILLING_ACCOUNT='000000-000000-000000'
52 
export OPENAGENTS_STAGING_TF_STATE_BUCKET="$OPENAGENTS_STAGING_PROJECT_ID-openagents-tfstate"
53 
read -r -s TF_VAR_database_password
54 
export TF_VAR_database_password
55 
printf '\n'
56 
```
57
58 
The database password is a Terraform ephemeral, write-only input and is not
59 
stored in the plan or state. Generate it with an approved password manager,
60 
keep it out of shell history, and use the same value when Gate 13 creates the
61 
staging runtime secret. Never reuse a production credential.
62
63 
Authenticate both the Cloud CLI and Terraform provider:
64
65 
```sh
66 
gcloud auth login
67 
gcloud auth application-default login
68 
```
69
70 
## Provision the boundary
71
72 
1. Review the project bootstrap targets without changing cloud state:
73
74 
   ```sh
75 
   ops/staging/bootstrap-project.sh check
76 
   ```
77
78 
2. Create the staging project, attach billing, and create the protected state
79 
   bucket:
80
81 
   ```sh
82 
   ops/staging/bootstrap-project.sh --apply
83 
   ```
84
85 
3. Validate the Terraform configuration:
86
87 
   ```sh
88 
   ops/staging/terraform.sh validate
89 
   ```
90
91 
4. Commit the exact candidate and create a plan tied to that clean Git SHA:
92
93 
   ```sh
94 
   ops/staging/terraform.sh plan
95 
   ```
96
97 
5. Review the saved plan. Apply that exact plan only when every target belongs
98 
   to the staging project:
99
100 
   ```sh
101 
   ops/staging/terraform.sh apply --apply
102 
   ```
103
104 
6. Validate the resulting boundary against both project identities:
105
106 
   ```sh
107 
   ops/staging/validate-isolation.sh
108 
   ```
109
110 
The validator verifies the separate `openagents_staging` database role and
111 
writes a content-free receipt under
112 
`.git/openagents/staging-isolation/<full-sha>.json`. Do not commit Terraform
113 
plans, state, credentials, project inventory, IP addresses, or secret values.
114
115 
## Complete Gate 12
116
117 
Gate 12 remains incomplete until the cloud apply, isolation validator, and
118 
manifest-scoped disposable-run cleanup command are proven. Do not populate
119 
secrets, push an image, change DNS for `stage.openagents.com`, or deploy a
120 
candidate as part of the infrastructure apply. Gate 13 performs those steps on
121 
one exact, locally gated SHA after a separate review.
122
123 
Do not run `terraform destroy`. Cloud SQL and the fleet instances have deletion
124 
protection. Use a separate, reviewed decommission plan after staging evidence
125 
is no longer required.
infra/staging/main.tf added +710

@@ -0,0 +1,710 @@

1 
locals {
2 
  labels = merge(var.labels, {
3 
    application = "openagents"
4 
    environment = "staging"
5 
    managed_by  = "terraform"
6 
  })
7
8 
  nodes = {
9 
    "openagents-fleet-1" = "10.42.0.11"
10 
    "openagents-fleet-2" = "10.42.0.12"
11 
    "openagents-fleet-3" = "10.42.0.13"
12 
  }
13
14 
  required_services = toset([
15 
    "artifactregistry.googleapis.com",
16 
    "compute.googleapis.com",
17 
    "dns.googleapis.com",
18 
    "iam.googleapis.com",
19 
    "logging.googleapis.com",
20 
    "monitoring.googleapis.com",
21 
    "secretmanager.googleapis.com",
22 
    "servicenetworking.googleapis.com",
23 
    "sqladmin.googleapis.com",
24 
    "storage.googleapis.com"
25 
  ])
26
27 
  runtime_secrets = toset([
28 
    "openagents-staging-web-config",
29 
    "openagents-staging-fleet-config",
30 
    "openagents-staging-builder-config",
31 
    "openagents-staging-database-url",
32 
    "openagents-staging-secret-key-base",
33 
    "openagents-staging-github-client-secret",
34 
    "openagents-staging-github-vault-active",
35 
    "openagents-staging-github-vault-previous",
36 
    "openagents-staging-openai-api-key",
37 
    "openagents-staging-voice-recording-key",
38 
    "openagents-staging-forge-operator-token",
39 
    "openagents-staging-release-cookie"
40 
  ])
41
42 
  application_secrets = toset([
43 
    "openagents-staging-database-url",
44 
    "openagents-staging-secret-key-base",
45 
    "openagents-staging-github-client-secret",
46 
    "openagents-staging-github-vault-active",
47 
    "openagents-staging-github-vault-previous",
48 
    "openagents-staging-openai-api-key",
49 
    "openagents-staging-voice-recording-key",
50 
    "openagents-staging-forge-operator-token",
51 
    "openagents-staging-release-cookie"
52 
  ])
53 
}
54
55 
resource "google_project_service" "required" {
56 
  for_each = local.required_services
57
58 
  project            = var.staging_project_id
59 
  service            = each.value
60 
  disable_on_destroy = false
61 
}
62
63 
resource "google_compute_network" "staging" {
64 
  name                    = "openagents-staging"
65 
  auto_create_subnetworks = false
66 
  routing_mode            = "REGIONAL"
67
68 
  depends_on = [google_project_service.required]
69 
}
70
71 
resource "google_compute_subnetwork" "staging" {
72 
  name                     = "openagents-staging-${var.region}"
73 
  ip_cidr_range            = var.network_cidr
74 
  region                   = var.region
75 
  network                  = google_compute_network.staging.id
76 
  private_ip_google_access = true
77
78 
  log_config {
79 
    aggregation_interval = "INTERVAL_5_SEC"
80 
    flow_sampling        = 0.5
81 
    metadata             = "INCLUDE_ALL_METADATA"
82 
  }
83 
}
84
85 
resource "google_compute_global_address" "private_services" {
86 
  name          = "openagents-staging-private-services"
87 
  purpose       = "VPC_PEERING"
88 
  address_type  = "INTERNAL"
89 
  prefix_length = 20
90 
  network       = google_compute_network.staging.id
91 
}
92
93 
resource "google_service_networking_connection" "private_services" {
94 
  network                 = google_compute_network.staging.id
95 
  service                 = "servicenetworking.googleapis.com"
96 
  reserved_peering_ranges = [google_compute_global_address.private_services.name]
97
98 
  depends_on = [google_project_service.required]
99 
}
100
101 
resource "google_compute_router" "staging" {
102 
  name    = "openagents-staging"
103 
  region  = var.region
104 
  network = google_compute_network.staging.id
105 
}
106
107 
resource "google_compute_router_nat" "staging" {
108 
  name                               = "openagents-staging"
109 
  router                             = google_compute_router.staging.name
110 
  region                             = var.region
111 
  nat_ip_allocate_option             = "AUTO_ONLY"
112 
  source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
113
114 
  subnetwork {
115 
    name                    = google_compute_subnetwork.staging.id
116 
    source_ip_ranges_to_nat = ["ALL_IP_RANGES"]
117 
  }
118
119 
  log_config {
120 
    enable = true
121 
    filter = "ERRORS_ONLY"
122 
  }
123 
}
124
125 
resource "google_dns_managed_zone" "staging_private" {
126 
  name        = "openagents-staging-internal"
127 
  dns_name    = "staging.internal."
128 
  description = "Private service discovery for isolated OpenAgents staging."
129 
  visibility  = "private"
130
131 
  private_visibility_config {
132 
    networks {
133 
      network_url = google_compute_network.staging.id
134 
    }
135 
  }
136
137 
  depends_on = [google_project_service.required]
138 
}
139
140 
resource "google_compute_address" "fleet" {
141 
  for_each = local.nodes
142
143 
  name         = each.key
144 
  region       = var.region
145 
  address_type = "INTERNAL"
146 
  subnetwork   = google_compute_subnetwork.staging.id
147 
  address      = each.value
148 
}
149
150 
resource "google_dns_record_set" "fleet_nodes" {
151 
  for_each = local.nodes
152
153 
  name         = "${each.key}.staging.internal."
154 
  type         = "A"
155 
  ttl          = 30
156 
  managed_zone = google_dns_managed_zone.staging_private.name
157 
  rrdatas      = [google_compute_address.fleet[each.key].address]
158 
}
159
160 
resource "google_dns_record_set" "fleet_discovery" {
161 
  name         = "openagents-fleet.staging.internal."
162 
  type         = "A"
163 
  ttl          = 30
164 
  managed_zone = google_dns_managed_zone.staging_private.name
165 
  rrdatas      = [for node in sort(keys(local.nodes)) : google_compute_address.fleet[node].address]
166 
}
167
168 
resource "google_dns_record_set" "deployer" {
169 
  name         = "openagents-deployer.staging.internal."
170 
  type         = "A"
171 
  ttl          = 30
172 
  managed_zone = google_dns_managed_zone.staging_private.name
173 
  rrdatas      = [google_compute_address.deployer.address]
174 
}
175
176 
resource "google_compute_firewall" "fleet_internal" {
177 
  name      = "openagents-staging-fleet-internal"
178 
  network   = google_compute_network.staging.name
179 
  direction = "INGRESS"
180 
  priority  = 900
181
182 
  source_ranges = [var.network_cidr]
183 
  target_tags   = ["openagents-staging-fleet", "openagents-staging-controller"]
184
185 
  allow {
186 
    protocol = "tcp"
187 
    ports    = ["4000", "4369", "9100-9115"]
188 
  }
189
190 
  log_config {
191 
    metadata = "INCLUDE_ALL_METADATA"
192 
  }
193 
}
194
195 
resource "google_compute_firewall" "iap_ssh" {
196 
  name      = "openagents-staging-iap-ssh"
197 
  network   = google_compute_network.staging.name
198 
  direction = "INGRESS"
199 
  priority  = 900
200
201 
  source_ranges = ["35.235.240.0/20"]
202 
  target_tags   = ["openagents-staging-fleet"]
203
204 
  allow {
205 
    protocol = "tcp"
206 
    ports    = ["22"]
207 
  }
208
209 
  log_config {
210 
    metadata = "INCLUDE_ALL_METADATA"
211 
  }
212 
}
213
214 
resource "google_service_account" "web" {
215 
  account_id   = "openagents-staging-web"
216 
  display_name = "OpenAgents staging web"
217 
  description  = "Runs only the staging web acceptance lane."
218 
}
219
220 
resource "google_service_account" "fleet" {
221 
  account_id   = "openagents-staging-fleet"
222 
  display_name = "OpenAgents staging fleet"
223 
  description  = "Runs only the three staging BEAM nodes."
224 
}
225
226 
resource "google_service_account" "deployer" {
227 
  account_id   = "openagents-staging-deployer"
228 
  display_name = "OpenAgents staging deployer"
229 
  description  = "Performs one-node-at-a-time replacement in staging."
230 
}
231
232 
resource "google_project_iam_member" "web" {
233 
  for_each = toset([
234 
    "roles/cloudsql.client",
235 
    "roles/logging.logWriter",
236 
    "roles/monitoring.metricWriter"
237 
  ])
238
239 
  project = var.staging_project_id
240 
  role    = each.value
241 
  member  = google_service_account.web.member
242 
}
243
244 
resource "google_project_iam_member" "fleet" {
245 
  for_each = toset([
246 
    "roles/artifactregistry.reader",
247 
    "roles/cloudsql.client",
248 
    "roles/logging.logWriter",
249 
    "roles/monitoring.metricWriter"
250 
  ])
251
252 
  project = var.staging_project_id
253 
  role    = each.value
254 
  member  = google_service_account.fleet.member
255 
}
256
257 
resource "google_project_iam_custom_role" "deployer" {
258 
  role_id     = "openagentsStagingDeployer"
259 
  title       = "OpenAgents staging deployer"
260 
  description = "Resets one staging fleet instance after updating its exact image metadata."
261 
  stage       = "GA"
262
263 
  permissions = [
264 
    "compute.instances.get",
265 
    "compute.instances.reset",
266 
    "compute.instances.setMetadata",
267 
    "compute.zoneOperations.get"
268 
  ]
269 
}
270
271 
resource "google_project_iam_member" "deployer" {
272 
  project = var.staging_project_id
273 
  role    = google_project_iam_custom_role.deployer.id
274 
  member  = google_service_account.deployer.member
275 
}
276
277 
resource "google_project_iam_member" "deployer_runtime" {
278 
  for_each = toset([
279 
    "roles/artifactregistry.reader",
280 
    "roles/logging.logWriter",
281 
    "roles/monitoring.metricWriter"
282 
  ])
283
284 
  project = var.staging_project_id
285 
  role    = each.value
286 
  member  = google_service_account.deployer.member
287 
}
288
289 
resource "google_project_iam_member" "iap_tunnel" {
290 
  for_each = var.iap_ssh_members
291
292 
  project = var.staging_project_id
293 
  role    = "roles/iap.tunnelResourceAccessor"
294 
  member  = each.value
295 
}
296
297 
resource "google_project_iam_member" "os_login" {
298 
  for_each = var.iap_ssh_members
299
300 
  project = var.staging_project_id
301 
  role    = "roles/compute.osLogin"
302 
  member  = each.value
303 
}
304
305 
resource "google_secret_manager_secret" "runtime" {
306 
  for_each = local.runtime_secrets
307
308 
  secret_id = each.value
309 
  labels    = local.labels
310
311 
  replication {
312 
    auto {}
313 
  }
314
315 
  depends_on = [google_project_service.required]
316 
}
317
318 
resource "google_secret_manager_secret_iam_member" "web_env" {
319 
  secret_id = google_secret_manager_secret.runtime["openagents-staging-web-config"].id
320 
  role      = "roles/secretmanager.secretAccessor"
321 
  member    = google_service_account.web.member
322 
}
323
324 
resource "google_secret_manager_secret_iam_member" "fleet_env" {
325 
  secret_id = google_secret_manager_secret.runtime["openagents-staging-fleet-config"].id
326 
  role      = "roles/secretmanager.secretAccessor"
327 
  member    = google_service_account.fleet.member
328 
}
329
330 
resource "google_secret_manager_secret_iam_member" "builder_env" {
331 
  secret_id = google_secret_manager_secret.runtime["openagents-staging-builder-config"].id
332 
  role      = "roles/secretmanager.secretAccessor"
333 
  member    = google_service_account.fleet.member
334 
}
335
336 
resource "google_secret_manager_secret_iam_member" "web_secrets" {
337 
  for_each = local.application_secrets
338
339 
  secret_id = google_secret_manager_secret.runtime[each.value].id
340 
  role      = "roles/secretmanager.secretAccessor"
341 
  member    = google_service_account.web.member
342 
}
343
344 
resource "google_secret_manager_secret_iam_member" "fleet_secrets" {
345 
  for_each = local.application_secrets
346
347 
  secret_id = google_secret_manager_secret.runtime[each.value].id
348 
  role      = "roles/secretmanager.secretAccessor"
349 
  member    = google_service_account.fleet.member
350 
}
351
352 
resource "google_secret_manager_secret_iam_member" "deployer_cookie" {
353 
  secret_id = google_secret_manager_secret.runtime["openagents-staging-release-cookie"].id
354 
  role      = "roles/secretmanager.secretAccessor"
355 
  member    = google_service_account.deployer.member
356 
}
357
358 
resource "google_artifact_registry_repository" "openagents" {
359 
  location      = var.region
360 
  repository_id = "openagents-staging"
361 
  description   = "Immutable OpenAgents staging release and builder images."
362 
  format        = "DOCKER"
363 
  labels        = local.labels
364
365 
  cleanup_policy_dry_run = true
366
367 
  depends_on = [google_project_service.required]
368 
}
369
370 
resource "google_storage_bucket" "artifacts" {
371 
  name                        = "${var.staging_project_id}-openagents-artifacts"
372 
  location                    = upper(var.region)
373 
  uniform_bucket_level_access = true
374 
  public_access_prevention    = "enforced"
375 
  force_destroy               = false
376 
  labels                      = local.labels
377
378 
  versioning {
379 
    enabled = true
380 
  }
381
382 
  soft_delete_policy {
383 
    retention_duration_seconds = 604800
384 
  }
385
386 
  depends_on = [google_project_service.required]
387 
}
388
389 
resource "google_storage_bucket" "wal" {
390 
  name                        = "${var.staging_project_id}-openagents-forge-wal"
391 
  location                    = upper(var.region)
392 
  uniform_bucket_level_access = true
393 
  public_access_prevention    = "enforced"
394 
  force_destroy               = false
395 
  labels                      = local.labels
396
397 
  versioning {
398 
    enabled = true
399 
  }
400
401 
  soft_delete_policy {
402 
    retention_duration_seconds = 604800
403 
  }
404
405 
  depends_on = [google_project_service.required]
406 
}
407
408 
resource "google_storage_bucket" "recordings" {
409 
  name                        = "${var.staging_project_id}-openagents-recordings"
410 
  location                    = upper(var.region)
411 
  uniform_bucket_level_access = true
412 
  public_access_prevention    = "enforced"
413 
  force_destroy               = false
414 
  labels                      = local.labels
415
416 
  lifecycle_rule {
417 
    condition {
418 
      age = 30
419 
    }
420 
    action {
421 
      type = "Delete"
422 
    }
423 
  }
424
425 
  soft_delete_policy {
426 
    retention_duration_seconds = 604800
427 
  }
428
429 
  depends_on = [google_project_service.required]
430 
}
431
432 
resource "google_storage_bucket" "evidence" {
433 
  name                        = "${var.staging_project_id}-openagents-evidence"
434 
  location                    = upper(var.region)
435 
  uniform_bucket_level_access = true
436 
  public_access_prevention    = "enforced"
437 
  force_destroy               = false
438 
  labels                      = local.labels
439
440 
  versioning {
441 
    enabled = true
442 
  }
443
444 
  retention_policy {
445 
    retention_period = 2592000
446 
    is_locked        = false
447 
  }
448
449 
  soft_delete_policy {
450 
    retention_duration_seconds = 604800
451 
  }
452
453 
  depends_on = [google_project_service.required]
454 
}
455
456 
resource "google_storage_bucket_iam_member" "fleet_artifacts" {
457 
  bucket = google_storage_bucket.artifacts.name
458 
  role   = "roles/storage.objectAdmin"
459 
  member = google_service_account.fleet.member
460 
}
461
462 
resource "google_storage_bucket_iam_member" "fleet_wal" {
463 
  bucket = google_storage_bucket.wal.name
464 
  role   = "roles/storage.objectAdmin"
465 
  member = google_service_account.fleet.member
466 
}
467
468 
resource "google_storage_bucket_iam_member" "web_recordings" {
469 
  bucket = google_storage_bucket.recordings.name
470 
  role   = "roles/storage.objectAdmin"
471 
  member = google_service_account.web.member
472 
}
473
474 
resource "google_storage_bucket_iam_member" "fleet_recordings" {
475 
  bucket = google_storage_bucket.recordings.name
476 
  role   = "roles/storage.objectAdmin"
477 
  member = google_service_account.fleet.member
478 
}
479
480 
resource "google_storage_bucket_iam_member" "deployer_evidence" {
481 
  bucket = google_storage_bucket.evidence.name
482 
  role   = "roles/storage.objectCreator"
483 
  member = google_service_account.deployer.member
484 
}
485
486 
resource "google_sql_database_instance" "staging" {
487 
  name                = "openagents-staging-postgres"
488 
  region              = var.region
489 
  database_version    = "POSTGRES_17"
490 
  deletion_protection = true
491
492 
  settings {
493 
    tier                        = var.database_tier
494 
    availability_type           = "ZONAL"
495 
    disk_type                   = "PD_SSD"
496 
    disk_size                   = 20
497 
    disk_autoresize             = true
498 
    deletion_protection_enabled = true
499 
    user_labels                 = local.labels
500
501 
    backup_configuration {
502 
      enabled                        = true
503 
      point_in_time_recovery_enabled = true
504 
      start_time                     = "05:00"
505 
      transaction_log_retention_days = 7
506
507 
      backup_retention_settings {
508 
        retained_backups = 7
509 
        retention_unit   = "COUNT"
510 
      }
511 
    }
512
513 
    ip_configuration {
514 
      ipv4_enabled    = false
515 
      private_network = google_compute_network.staging.id
516 
      ssl_mode        = "ENCRYPTED_ONLY"
517 
    }
518
519 
    insights_config {
520 
      query_insights_enabled  = true
521 
      query_string_length     = 1024
522 
      record_application_tags = true
523 
    }
524
525 
    maintenance_window {
526 
      day          = 7
527 
      hour         = 6
528 
      update_track = "stable"
529 
    }
530 
  }
531
532 
  depends_on = [
533 
    google_project_service.required,
534 
    google_service_networking_connection.private_services
535 
  ]
536 
}
537
538 
resource "google_sql_database" "openagents" {
539 
  name            = "openagents_staging"
540 
  instance        = google_sql_database_instance.staging.name
541 
  deletion_policy = "ABANDON"
542 
}
543
544 
resource "google_sql_user" "openagents" {
545 
  name                = "openagents_staging"
546 
  instance            = google_sql_database_instance.staging.name
547 
  password_wo         = var.database_password
548 
  password_wo_version = var.database_password_version
549 
  deletion_policy     = "ABANDON"
550 
}
551
552 
data "google_compute_image" "cos" {
553 
  family  = "cos-stable"
554 
  project = "cos-cloud"
555 
}
556
557 
resource "google_compute_disk" "fleet_state" {
558 
  for_each = local.nodes
559
560 
  name   = "${each.key}-state"
561 
  type   = "pd-balanced"
562 
  zone   = var.zone
563 
  size   = var.fleet_state_disk_gib
564 
  labels = local.labels
565 
}
566
567 
resource "google_compute_instance" "fleet" {
568 
  for_each = local.nodes
569
570 
  name                      = each.key
571 
  zone                      = var.zone
572 
  machine_type              = var.fleet_machine_type
573 
  allow_stopping_for_update = true
574 
  can_ip_forward            = false
575 
  deletion_protection       = true
576 
  tags                      = ["openagents-staging-fleet"]
577 
  labels                    = merge(local.labels, { lane = "distributed" })
578
579 
  boot_disk {
580 
    auto_delete = true
581
582 
    initialize_params {
583 
      image = data.google_compute_image.cos.self_link
584 
      size  = 20
585 
      type  = "pd-balanced"
586 
    }
587 
  }
588
589 
  attached_disk {
590 
    source      = google_compute_disk.fleet_state[each.key].id
591 
    device_name = "openagents-state"
592 
    mode        = "READ_WRITE"
593 
  }
594
595 
  network_interface {
596 
    subnetwork = google_compute_subnetwork.staging.id
597 
    network_ip = google_compute_address.fleet[each.key].address
598 
  }
599
600 
  metadata = {
601 
    block-project-ssh-keys    = "TRUE"
602 
    enable-oslogin            = "TRUE"
603 
    openagents-environment    = "staging"
604 
    openagents-image          = ""
605 
    openagents-image-digest   = ""
606 
    openagents-builder-image  = ""
607 
    openagents-builder-digest = ""
608 
    openagents-sha            = ""
609 
    openagents-runtime-secret = google_secret_manager_secret.runtime["openagents-staging-fleet-config"].secret_id
610 
    openagents-builder-secret = google_secret_manager_secret.runtime["openagents-staging-builder-config"].secret_id
611 
    startup-script = templatefile("${path.module}/templates/fleet-startup.sh.tftpl", {
612 
      project_id = var.staging_project_id
613 
      region     = var.region
614 
    })
615 
  }
616
617 
  service_account {
618 
    email  = google_service_account.fleet.email
619 
    scopes = ["cloud-platform"]
620 
  }
621
622 
  scheduling {
623 
    automatic_restart   = true
624 
    on_host_maintenance = "MIGRATE"
625 
    provisioning_model  = "STANDARD"
626 
  }
627
628 
  shielded_instance_config {
629 
    enable_secure_boot          = true
630 
    enable_vtpm                 = true
631 
    enable_integrity_monitoring = true
632 
  }
633
634 
  depends_on = [
635 
    google_project_service.required,
636 
    google_compute_router_nat.staging,
637 
    google_secret_manager_secret_iam_member.fleet_env
638 
  ]
639 
}
640
641 
resource "google_compute_address" "deployer" {
642 
  name         = "openagents-staging-deployer"
643 
  region       = var.region
644 
  address_type = "INTERNAL"
645 
  subnetwork   = google_compute_subnetwork.staging.id
646 
  address      = "10.42.0.20"
647 
}
648
649 
resource "google_compute_instance" "deployer" {
650 
  name                      = "openagents-staging-deployer"
651 
  zone                      = var.zone
652 
  machine_type              = "e2-small"
653 
  allow_stopping_for_update = true
654 
  can_ip_forward            = false
655 
  deletion_protection       = true
656 
  tags                      = ["openagents-staging-controller"]
657 
  labels                    = merge(local.labels, { lane = "deployer" })
658
659 
  boot_disk {
660 
    auto_delete = true
661
662 
    initialize_params {
663 
      image = data.google_compute_image.cos.self_link
664 
      size  = 20
665 
      type  = "pd-balanced"
666 
    }
667 
  }
668
669 
  network_interface {
670 
    subnetwork = google_compute_subnetwork.staging.id
671 
    network_ip = google_compute_address.deployer.address
672 
  }
673
674 
  metadata = {
675 
    block-project-ssh-keys      = "TRUE"
676 
    enable-oslogin              = "TRUE"
677 
    openagents-environment      = "staging"
678 
    openagents-controller-image = ""
679 
    openagents-controller-sha   = ""
680 
    openagents-cookie-secret    = google_secret_manager_secret.runtime["openagents-staging-release-cookie"].secret_id
681 
    startup-script = templatefile("${path.module}/templates/deployer-startup.sh.tftpl", {
682 
      project_id = var.staging_project_id
683 
      region     = var.region
684 
    })
685 
  }
686
687 
  service_account {
688 
    email  = google_service_account.deployer.email
689 
    scopes = ["cloud-platform"]
690 
  }
691
692 
  scheduling {
693 
    automatic_restart   = true
694 
    on_host_maintenance = "MIGRATE"
695 
    provisioning_model  = "STANDARD"
696 
  }
697
698 
  shielded_instance_config {
699 
    enable_secure_boot          = true
700 
    enable_vtpm                 = true
701 
    enable_integrity_monitoring = true
702 
  }
703
704 
  depends_on = [
705 
    google_project_service.required,
706 
    google_compute_router_nat.staging,
707 
    google_project_iam_member.deployer,
708 
    google_secret_manager_secret_iam_member.deployer_cookie
709 
  ]
710 
}
infra/staging/outputs.tf added +56

@@ -0,0 +1,56 @@

1 
output "staging_project_id" {
2 
  description = "Dedicated staging project."
3 
  value       = var.staging_project_id
4 
}
5
6 
output "network" {
7 
  description = "Private staging VPC self-link."
8 
  value       = google_compute_network.staging.self_link
9 
}
10
11 
output "database_connection_name" {
12 
  description = "Staging-only Cloud SQL connection name."
13 
  value       = google_sql_database_instance.staging.connection_name
14 
}
15
16 
output "database_private_ip" {
17 
  description = "Private Cloud SQL address for the distributed lane."
18 
  value       = google_sql_database_instance.staging.private_ip_address
19 
}
20
21 
output "image_repository" {
22 
  description = "Digest-only image repository for staging candidates."
23 
  value       = "${var.region}-docker.pkg.dev/${var.staging_project_id}/${google_artifact_registry_repository.openagents.repository_id}/openagents"
24 
}
25
26 
output "builder_image_repository" {
27 
  description = "Digest-only image repository for isolated staging builders."
28 
  value       = "${var.region}-docker.pkg.dev/${var.staging_project_id}/${google_artifact_registry_repository.openagents.repository_id}/openagents-builder"
29 
}
30
31 
output "fleet_nodes" {
32 
  description = "Exact BEAM node-to-instance map for runtime configuration."
33 
  value = {
34 
    for instance_name in sort(keys(local.nodes)) :
35 
    "openagents@${instance_name}.staging.internal" => instance_name
36 
  }
37 
}
38
39 
output "buckets" {
40 
  description = "Staging-only durable storage buckets."
41 
  value = {
42 
    artifacts  = google_storage_bucket.artifacts.name
43 
    evidence   = google_storage_bucket.evidence.name
44 
    recordings = google_storage_bucket.recordings.name
45 
    wal        = google_storage_bucket.wal.name
46 
  }
47 
}
48
49 
output "service_accounts" {
50 
  description = "Staging-only runtime identities."
51 
  value = {
52 
    deployer = google_service_account.deployer.email
53 
    fleet    = google_service_account.fleet.email
54 
    web      = google_service_account.web.email
55 
  }
56 
}
infra/staging/templates/deployer-startup.sh.tftpl added +90

@@ -0,0 +1,90 @@

1 
#!/bin/bash
2 
set -euo pipefail
3
4 
exec >>/var/log/openagents-deployer-startup.log 2>&1
5
6 
metadata() {
7 
  curl --fail --silent --show-error \
8 
    --header 'Metadata-Flavor: Google' \
9 
    "http://metadata.google.internal/computeMetadata/v1/$1"
10 
}
11
12 
metadata_attribute() {
13 
  metadata "instance/attributes/$1" 2>/dev/null || true
14 
}
15
16 
image=$(metadata_attribute openagents-controller-image)
17 
source_sha=$(metadata_attribute openagents-controller-sha)
18 
cookie_secret=$(metadata_attribute openagents-cookie-secret)
19
20 
if [ -z "$image" ] || [ -z "$source_sha" ]; then
21 
  echo "No staging controller candidate is assigned."
22 
  exit 0
23 
fi
24
25 
controller_image_prefix="${region}-docker.pkg.dev/${project_id}/openagents-staging/openagents@"
26
27 
case "$image" in
28 
  "$controller_image_prefix"sha256:????????????????????????????????????????????????????????????????) ;;
29 
  *) echo "Assigned controller image is not digest-addressed" >&2; exit 1 ;;
30 
esac
31
32 
case "$source_sha" in
33 
  ????????????????????????????????????????) ;;
34 
  *) echo "Assigned controller source revision is malformed" >&2; exit 1 ;;
35 
esac
36
37 
case "$source_sha" in
38 
  *[!0-9a-f]*) echo "Assigned controller source revision is malformed" >&2; exit 1 ;;
39 
esac
40
41 
access_token=$(metadata instance/service-accounts/default/token | jq -r '.access_token')
42 
mkdir -p /run/openagents
43
44 
release_cookie=$(
45 
  curl --fail --silent --show-error \
46 
    --header "Authorization: Bearer $access_token" \
47 
    "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$cookie_secret/versions/latest:access" \
48 
    | jq -r '.payload.data' \
49 
    | tr '_-' '/+' \
50 
    | base64 -d
51 
)
52
53 
if [ -z "$release_cookie" ] || [ "$(printf %s "$release_cookie" | wc -l)" -ne 0 ]; then
54 
  echo "Release cookie must be a nonempty single-line value" >&2
55 
  exit 1
56 
fi
57
58 
printf 'RELEASE_COOKIE=%s\n' "$release_cookie" >/run/openagents/deployer.env
59 
unset release_cookie
60
61 
chmod 0600 /run/openagents/deployer.env
62 
docker-credential-gcr configure-docker --registries=${region}-docker.pkg.dev
63 
docker pull "$image"
64 
docker rm --force openagents-deployer 2>/dev/null || true
65 
docker run --detach \
66 
  --name openagents-deployer \
67 
  --network host \
68 
  --restart always \
69 
  --env-file /run/openagents/deployer.env \
70 
  --env "OPENAGENTS_CONTROLLER_SHA=$source_sha" \
71 
  --entrypoint /bin/sh \
72 
  "$image" -c '
73 
    set -eu
74 
    : "$${RELEASE_COOKIE:?RELEASE_COOKIE is required}"
75 
    release_version=$(awk '\''{print $2}'\'' /app/releases/start_erl.data)
76 
    cookie_root=/tmp/openagents-deployer-home
77 
    mkdir -p "$cookie_root"
78 
    umask 077
79 
    printf %s "$RELEASE_COOKIE" >"$cookie_root/.erlang.cookie"
80 
    unset RELEASE_COOKIE
81 
    export HOME="$cookie_root"
82 
    exec /app/erts-*/bin/erl \
83 
      -boot_var RELEASE_LIB /app/lib \
84 
      -boot "/app/releases/$release_version/start_clean" \
85 
      -name openagents-deployer@openagents-deployer.staging.internal \
86 
      -kernel inet_dist_listen_min 9100 inet_dist_listen_max 9115 \
87 
      -noshell \
88 
      -pa /app/lib/openagents-*/ebin \
89 
      -eval "'\''Elixir.OpenAgents.Forge.RollingProvider.Gcp.Deployer'\'':start()."
90 
  '
infra/staging/templates/fleet-startup.sh.tftpl added +190

@@ -0,0 +1,190 @@

1 
#!/bin/bash
2 
set -euo pipefail
3
4 
exec >>/var/log/openagents-startup.log 2>&1
5
6 
metadata() {
7 
  curl --fail --silent --show-error \
8 
    --header 'Metadata-Flavor: Google' \
9 
    "http://metadata.google.internal/computeMetadata/v1/$1"
10 
}
11
12 
metadata_attribute() {
13 
  metadata "instance/attributes/$1" 2>/dev/null || true
14 
}
15
16 
state_device=/dev/disk/by-id/google-openagents-state
17 
state_root=/var/lib/openagents
18
19 
if ! blkid "$state_device" >/dev/null 2>&1; then
20 
  mkfs.ext4 -F "$state_device"
21 
fi
22
23 
mkdir -p "$state_root"
24 
mountpoint -q "$state_root" || mount "$state_device" "$state_root"
25 
mkdir -p \
26 
  "$state_root/artifacts" \
27 
  "$state_root/coding-jobs" \
28 
  "$state_root/forge" \
29 
  "$state_root/forge-wal" \
30 
  "$state_root/ra" \
31 
  "$state_root/workspace/build" \
32 
  "$state_root/workspace/build-queue"
33
34 
image=$(metadata_attribute openagents-image)
35 
image_digest=$(metadata_attribute openagents-image-digest)
36 
builder_image=$(metadata_attribute openagents-builder-image)
37 
builder_digest=$(metadata_attribute openagents-builder-digest)
38 
source_sha=$(metadata_attribute openagents-sha)
39 
runtime_secret=$(metadata_attribute openagents-runtime-secret)
40 
builder_secret=$(metadata_attribute openagents-builder-secret)
41 
instance_name=$(metadata instance/name)
42
43 
if [ -z "$image" ] || [ -z "$image_digest" ] || [ -z "$source_sha" ]; then
44 
  echo "No staging candidate is assigned; leaving the node fenced."
45 
  exit 0
46 
fi
47
48 
application_image_prefix="${region}-docker.pkg.dev/${project_id}/openagents-staging/openagents@"
49 
builder_image_prefix="${region}-docker.pkg.dev/${project_id}/openagents-staging/openagents-builder@"
50
51 
case "$image" in
52 
  "$application_image_prefix"sha256:????????????????????????????????????????????????????????????????) ;;
53 
  *) echo "Assigned application image is not digest-addressed" >&2; exit 1 ;;
54 
esac
55
56 
case "$source_sha" in
57 
  ????????????????????????????????????????) ;;
58 
  *) echo "Assigned source revision is malformed" >&2; exit 1 ;;
59 
esac
60
61 
case "$source_sha" in
62 
  *[!0-9a-f]*) echo "Assigned source revision is malformed" >&2; exit 1 ;;
63 
esac
64
65 
case "$image_digest" in
66 
  sha256:????????????????????????????????????????????????????????????????) ;;
67 
  *) echo "Assigned application digest is malformed" >&2; exit 1 ;;
68 
esac
69
70 
image_digest_hex=$${image_digest#sha256:}
71 
case "$image_digest_hex" in
72 
  *[!0-9a-f]*) echo "Assigned application digest is malformed" >&2; exit 1 ;;
73 
esac
74
75 
case "$image" in
76 
  *"@$image_digest") ;;
77 
  *) echo "Assigned application image and digest disagree" >&2; exit 1 ;;
78 
esac
79
80 
access_token=$(metadata instance/service-accounts/default/token | jq -r '.access_token')
81
82 
fetch_secret() {
83 
  secret_name=$1
84 
  destination=$2
85
86 
  curl --fail --silent --show-error \
87 
    --header "Authorization: Bearer $access_token" \
88 
    "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$secret_name/versions/latest:access" \
89 
    | jq -r '.payload.data' \
90 
    | tr '_-' '/+' \
91 
    | base64 -d >"$destination"
92
93 
  chmod 0600 "$destination"
94 
}
95
96 
append_secret() {
97 
  environment_name=$1
98 
  secret_name=$2
99 
  destination=$3
100 
  value=$(
101 
    curl --fail --silent --show-error \
102 
      --header "Authorization: Bearer $access_token" \
103 
      "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$secret_name/versions/latest:access" \
104 
      | jq -r '.payload.data' \
105 
      | tr '_-' '/+' \
106 
      | base64 -d
107 
  )
108
109 
  if [ -z "$value" ] || [ "$(printf %s "$value" | wc -l)" -ne 0 ]; then
110 
    echo "Runtime secrets must be nonempty single-line values" >&2
111 
    exit 1
112 
  fi
113
114 
  printf '%s=%s\n' "$environment_name" "$value" >>"$destination"
115 
}
116
117 
mkdir -p /run/openagents
118 
fetch_secret "$runtime_secret" /run/openagents/runtime.env
119
120 
if grep -Eq '^(DATABASE_URL|SECRET_KEY_BASE|GITHUB_CLIENT_SECRET|GITHUB_TOKEN_ENCRYPTION_KEY|GITHUB_TOKEN_DECRYPTION_KEYS_JSON|OPENAI_API_KEY|VOICE_RECORDING_ENCRYPTION_KEY|OPENAGENTS_FORGE_OPERATOR_TOKEN|RELEASE_COOKIE)=' /run/openagents/runtime.env; then
121 
  echo "Fleet configuration must not duplicate named secrets" >&2
122 
  exit 1
123 
fi
124
125 
append_secret DATABASE_URL openagents-staging-database-url /run/openagents/runtime.env
126 
append_secret SECRET_KEY_BASE openagents-staging-secret-key-base /run/openagents/runtime.env
127 
append_secret GITHUB_CLIENT_SECRET openagents-staging-github-client-secret /run/openagents/runtime.env
128 
append_secret GITHUB_TOKEN_ENCRYPTION_KEY openagents-staging-github-vault-active /run/openagents/runtime.env
129 
append_secret GITHUB_TOKEN_DECRYPTION_KEYS_JSON openagents-staging-github-vault-previous /run/openagents/runtime.env
130 
append_secret OPENAI_API_KEY openagents-staging-openai-api-key /run/openagents/runtime.env
131 
append_secret VOICE_RECORDING_ENCRYPTION_KEY openagents-staging-voice-recording-key /run/openagents/runtime.env
132 
append_secret OPENAGENTS_FORGE_OPERATOR_TOKEN openagents-staging-forge-operator-token /run/openagents/runtime.env
133 
append_secret RELEASE_COOKIE openagents-staging-release-cookie /run/openagents/runtime.env
134
135 
cat >>/run/openagents/runtime.env <<EOF
136 
DNS_CLUSTER_QUERY=openagents-fleet.staging.internal
137 
OPENAGENTS_IMAGE_DIGEST=$image_digest
138 
RELEASE_DISTRIBUTION=name
139 
RELEASE_NODE=openagents@$instance_name.staging.internal
140 
EOF
141
142 
docker-credential-gcr configure-docker --registries=${region}-docker.pkg.dev
143 
docker pull "$image"
144 
docker rm --force openagents 2>/dev/null || true
145 
docker run --detach \
146 
  --name openagents \
147 
  --network host \
148 
  --restart always \
149 
  --env-file /run/openagents/runtime.env \
150 
  --volume "$state_root:$state_root" \
151 
  "$image"
152
153 
if [ -n "$builder_image" ] || [ -n "$builder_digest" ]; then
154 
  case "$builder_image" in
155 
    "$builder_image_prefix"sha256:????????????????????????????????????????????????????????????????) ;;
156 
    *) echo "Assigned builder image is not digest-addressed" >&2; exit 1 ;;
157 
  esac
158
159
160 
  case "$builder_digest" in
161 
    sha256:????????????????????????????????????????????????????????????????) ;;
162 
    *) echo "Assigned builder digest is malformed" >&2; exit 1 ;;
163 
  esac
164
165 
  builder_digest_hex=$${builder_digest#sha256:}
166 
  case "$builder_digest_hex" in
167 
    *[!0-9a-f]*) echo "Assigned builder digest is malformed" >&2; exit 1 ;;
168 
  esac
169
170 
  case "$builder_image" in
171 
    *"@$builder_digest") ;;
172 
    *) echo "Assigned builder image and digest disagree" >&2; exit 1 ;;
173 
  esac
174
175 
  fetch_secret "$builder_secret" /run/openagents/builder.env
176 
  if grep -Eq '^OPENAGENTS_FORGE_OPERATOR_TOKEN=' /run/openagents/builder.env; then
177 
    echo "Builder configuration must not duplicate its named secret" >&2
178 
    exit 1
179 
  fi
180 
  append_secret OPENAGENTS_FORGE_OPERATOR_TOKEN openagents-staging-forge-operator-token /run/openagents/builder.env
181 
  docker pull "$builder_image"
182 
  docker rm --force openagents-builder 2>/dev/null || true
183 
  docker run --detach \
184 
    --name openagents-builder \
185 
    --network host \
186 
    --restart always \
187 
    --env-file /run/openagents/builder.env \
188 
    --volume "$state_root/workspace:$state_root/workspace" \
189 
    "$builder_image"
190 
fi
infra/staging/tests/safety.tftest.hcl added +88

@@ -0,0 +1,88 @@

1 
mock_provider "google" {}
2
3 
run "isolated_topology" {
4 
  command = plan
5
6 
  variables {
7 
    staging_project_id    = "openagents-staging-test"
8 
    production_project_id = "openagents-production"
9 
    database_password     = "test-only-database-password-000000000000"
10 
  }
11
12 
  assert {
13 
    condition     = length(google_compute_instance.fleet) == 3
14 
    error_message = "The distributed staging lane must contain exactly three nodes."
15 
  }
16
17 
  assert {
18 
    condition = alltrue([
19 
      for instance in values(google_compute_instance.fleet) :
20 
      length(instance.network_interface[0].access_config) == 0
21 
    ])
22 
    error_message = "Staging fleet nodes must not have public access configurations."
23 
  }
24
25 
  assert {
26 
    condition     = length(google_compute_instance.deployer.network_interface[0].access_config) == 0
27 
    error_message = "The staging deployer must not have a public access configuration."
28 
  }
29
30 
  assert {
31 
    condition     = google_sql_database_instance.staging.deletion_protection
32 
    error_message = "The staging database must keep Terraform deletion protection enabled."
33 
  }
34
35 
  assert {
36 
    condition     = google_sql_database_instance.staging.settings[0].ip_configuration[0].ipv4_enabled == false
37 
    error_message = "The staging database must not expose a public IPv4 address."
38 
  }
39
40 
  assert {
41 
    condition     = google_sql_user.openagents.name == "openagents_staging"
42 
    error_message = "Staging must have a separate application database role."
43 
  }
44
45 
  assert {
46 
    condition     = length(google_secret_manager_secret.runtime) == 12
47 
    error_message = "Every named staging credential and lane configuration needs its own secret resource."
48 
  }
49
50 
  assert {
51 
    condition = toset(google_project_iam_custom_role.deployer.permissions) == toset([
52 
      "compute.instances.get",
53 
      "compute.instances.reset",
54 
      "compute.instances.setMetadata",
55 
      "compute.zoneOperations.get"
56 
    ])
57 
    error_message = "The deployer role must retain its bounded Compute permission set."
58 
  }
59
60 
  assert {
61 
    condition     = google_secret_manager_secret_iam_member.deployer_cookie.role == "roles/secretmanager.secretAccessor"
62 
    error_message = "The deployer identity may read only the cluster cookie secret."
63 
  }
64 
}
65
66 
run "rejects_production_project" {
67 
  command = plan
68
69 
  variables {
70 
    staging_project_id    = "openagents-staging-test"
71 
    production_project_id = "openagents-staging-test"
72 
    database_password     = "test-only-database-password-000000000000"
73 
  }
74
75 
  expect_failures = [var.production_project_id]
76 
}
77
78 
run "rejects_unmarked_project" {
79 
  command = plan
80
81 
  variables {
82 
    staging_project_id    = "openagents-testing"
83 
    production_project_id = "openagents-production"
84 
    database_password     = "test-only-database-password-000000000000"
85 
  }
86
87 
  expect_failures = [var.staging_project_id]
88 
}
infra/staging/variables.tf added +114

@@ -0,0 +1,114 @@

1 
variable "staging_project_id" {
2 
  description = "Dedicated Google Cloud project for OpenAgents staging."
3 
  type        = string
4
5 
  validation {
6 
    condition = (
7 
      can(regex("^[a-z][a-z0-9-]{4,28}[a-z0-9]$", var.staging_project_id)) &&
8 
      can(regex("stag", var.staging_project_id))
9 
    )
10 
    error_message = "The staging project ID must be valid and contain 'stag'."
11 
  }
12 
}
13
14 
variable "production_project_id" {
15 
  description = "Production project ID used only by isolation safety checks."
16 
  type        = string
17
18 
  validation {
19 
    condition = (
20 
      can(regex("^[a-z][a-z0-9-]{4,28}[a-z0-9]$", var.production_project_id)) &&
21 
      var.production_project_id != var.staging_project_id
22 
    )
23 
    error_message = "The production project ID must be valid and differ from staging."
24 
  }
25 
}
26
27 
variable "region" {
28 
  description = "Region for all staging resources."
29 
  type        = string
30 
  default     = "us-central1"
31 
}
32
33 
variable "zone" {
34 
  description = "Zone for stable staging fleet instances."
35 
  type        = string
36 
  default     = "us-central1-a"
37
38 
  validation {
39 
    condition     = startswith(var.zone, "${var.region}-")
40 
    error_message = "The fleet zone must belong to the configured region."
41 
  }
42 
}
43
44 
variable "network_cidr" {
45 
  description = "Private subnet for the web and distributed staging lanes."
46 
  type        = string
47 
  default     = "10.42.0.0/24"
48 
}
49
50 
variable "database_tier" {
51 
  description = "Cloud SQL machine tier with a staging-only connection budget."
52 
  type        = string
53 
  default     = "db-custom-1-3840"
54 
}
55
56 
variable "database_password" {
57 
  description = "Write-only password for the staging application database role."
58 
  type        = string
59 
  sensitive   = true
60 
  ephemeral   = true
61
62 
  validation {
63 
    condition     = length(var.database_password) >= 32 && length(var.database_password) <= 256
64 
    error_message = "The staging database password must contain 32 through 256 characters."
65 
  }
66 
}
67
68 
variable "database_password_version" {
69 
  description = "Monotonic version that triggers write-only database password rotation."
70 
  type        = number
71 
  default     = 1
72
73 
  validation {
74 
    condition     = var.database_password_version >= 1 && floor(var.database_password_version) == var.database_password_version
75 
    error_message = "The staging database password version must be a positive integer."
76 
  }
77 
}
78
79 
variable "fleet_machine_type" {
80 
  description = "Machine type for each distributed staging node."
81 
  type        = string
82 
  default     = "e2-standard-2"
83 
}
84
85 
variable "fleet_state_disk_gib" {
86 
  description = "Durable state disk size for each distributed staging node."
87 
  type        = number
88 
  default     = 100
89
90 
  validation {
91 
    condition     = var.fleet_state_disk_gib >= 50 && var.fleet_state_disk_gib <= 1024
92 
    error_message = "Fleet state disks must be between 50 and 1024 GiB."
93 
  }
94 
}
95
96 
variable "iap_ssh_members" {
97 
  description = "Operator principals allowed to use IAP and OS Login for staging."
98 
  type        = set(string)
99 
  default     = []
100
101 
  validation {
102 
    condition = alltrue([
103 
      for member in var.iap_ssh_members :
104 
      can(regex("^(user|group):[^[:space:]]+$", member))
105 
    ])
106 
    error_message = "IAP members must use a user: or group: principal."
107 
  }
108 
}
109
110 
variable "labels" {
111 
  description = "Additional non-sensitive labels for staging resources."
112 
  type        = map(string)
113 
  default     = {}
114 
}
infra/staging/versions.tf added +20

@@ -0,0 +1,20 @@

1 
terraform {
2 
  required_version = ">= 1.11, < 2.0"
3
4 
  required_providers {
5 
    google = {
6 
      source  = "hashicorp/google"
7 
      version = "~> 7.41"
8 
    }
9 
  }
10
11 
  backend "gcs" {
12 
    prefix = "openagents/staging"
13 
  }
14 
}
15
16 
provider "google" {
17 
  project = var.staging_project_id
18 
  region  = var.region
19 
  zone    = var.zone
20 
}
lib/openagents/build_info.ex modified +4 -1

@@ -6,11 +6,14 @@ defmodule OpenAgents.BuildInfo do

6 6 
  `@revision` and status shows it fleet-wide in seconds.
7 7 
  """
8 8
9 
  @revision "image"
9 
  @revision System.get_env("OPENAGENTS_BUILD_REVISION", "image")
10 10
11 11 
  @doc "The compiled-in revision string of this module."
12 12 
  def revision, do: @revision
13 13
14 14 
  @doc "When this module was hot-loaded, or nil for the boot image version."
15 15 
  def loaded_at, do: nil
16
17 
  @doc "The immutable runtime image digest, or nil outside a packaged image."
18 
  def image_digest, do: Application.get_env(:openagents, :image_digest)
16 19 
end
lib/openagents/cluster.ex modified +4 -1

@@ -66,17 +66,20 @@ defmodule OpenAgents.Cluster do

66 66 
    boot = OpenAgents.Forge.BootConverge.state()
67 67 
    boot_ready? = OpenAgents.Forge.BootConverge.ready?()
68 68 
    deployment = OpenAgents.Forge.DeploymentNode.health()
69 
    admission_ready? = OpenAgents.Cluster.Admission.ready?()
69 70
70 71 
    %{
71 72 
      "schema" => "openagents.cluster_health.v1",
72 73 
      "node" => to_string(Node.self()),
73 74 
      "version" => to_string(Application.spec(:openagents, :vsn) || "unknown"),
74 75 
      "revision" => deployment["revision"] || boot["sha"] || OpenAgents.BuildInfo.revision(),
76 
      "image_digest" => OpenAgents.BuildInfo.image_digest(),
75 77 
      "boot_converged" => boot_ready?,
76 78 
      "deployment_ready" => deployment["participant_ready"],
79 
      "admission_ready" => admission_ready?,
77 80 
      "uptime_ms" => uptime_ms(),
78 81 
      "live" => true,
79 
      "ready" => boot_ready? and deployment["ready"] == true
82 
      "ready" => boot_ready? and deployment["ready"] == true and admission_ready?
80 83 
    }
81 84 
  end
82 85
lib/openagents/cluster/admission.ex added +26

@@ -0,0 +1,26 @@

1 
defmodule OpenAgents.Cluster.Admission do
2 
  @moduledoc """
3 
  Controls whether the local node can enter external readiness.
4
5 
  A rolling provider removes admission before it drains or replaces a node.
6 
  The state is node-local and content-free. A fresh VM starts admitted, but the
7 
  endpoint remains unavailable until boot convergence and database checks pass.
8 
  """
9
10 
  @state_key {__MODULE__, :ready}
11
12 
  @doc "Remove the local node from readiness."
13 
  def remove do
14 
    :persistent_term.put(@state_key, false)
15 
    :ok
16 
  end
17
18 
  @doc "Allow the local node to become ready after every other check passes."
19 
  def restore do
20 
    :persistent_term.put(@state_key, true)
21 
    :ok
22 
  end
23
24 
  @doc "Return whether the local node is admitted to readiness."
25 
  def ready?, do: :persistent_term.get(@state_key, true)
26 
end
lib/openagents/forge/gate_receipt.ex modified +1

@@ -24,6 +24,7 @@ defmodule OpenAgents.Forge.GateReceipt do

24 24 
    interrupted_install
25 25 
    rolling_replacement
26 26 
    contracts
27 
    staging_infra
27 28 
    release_smoke
28 29 
  )
29 30
lib/openagents/forge/rolling_node_probe.ex added +32

@@ -0,0 +1,32 @@

1 
defmodule OpenAgents.Forge.RollingNodeProbe do
2 
  @moduledoc """
3 
  Returns the bounded node-local facts required by rolling replacement.
4
5 
  The infrastructure provider calls this module over Erlang distribution. The
6 
  response contains identity and readiness state only. It never includes a
7 
  database address, credential, request, or application content.
8 
  """
9
10 
  @doc "Return node health and Ra quorum for an expected fleet size."
11 
  def status(expected_fleet_size)
12 
      when is_integer(expected_fleet_size) and expected_fleet_size > 0 do
13 
    report = OpenAgents.Cluster.local_report()
14 
    ra_members = OpenAgents.Cluster.Ra.members()
15
16 
    %{
17 
      member: true,
18 
      ready: report["ready"] == true,
19 
      boot_converged: report["boot_converged"] == true,
20 
      database_ready: database_ready?(),
21 
      sha: report["revision"],
22 
      image_digest: report["image_digest"],
23 
      ra_quorum: length(ra_members) * 2 > expected_fleet_size
24 
    }
25 
  end
26
27 
  defp database_ready? do
28 
    match?({:ok, _result}, OpenAgents.Repo.query("SELECT 1"))
29 
  rescue
30 
    _error -> false
31 
  end
32 
end
lib/openagents/forge/rolling_provider.ex modified +2

@@ -9,7 +9,9 @@ defmodule OpenAgents.Forge.RollingProvider do

9 9
10 10 
  @type context :: %{
11 11 
          required(:sha) => binary(),
12 
          required(:previous_sha) => binary(),
12 13 
          required(:image_digest) => binary(),
14 
          required(:previous_image_digest) => binary(),
13 15 
          required(:expected_nodes) => [node()]
14 16 
        }
15 17
lib/openagents/forge/rolling_provider/gcp.ex added +211

@@ -0,0 +1,211 @@

1 
defmodule OpenAgents.Forge.RollingProvider.Gcp do
2 
  @moduledoc """
3 
  Implements rolling replacement for the isolated Google Cloud staging fleet.
4
5 
  Readiness, drain, and health checks use private Erlang distribution. Image
6 
  replacement uses the Compute Engine API through a narrowly authorized
7 
  staging deployer identity. The provider accepts only an exact node-to-instance
8 
  map and refuses a project that matches the configured production project.
9 
  """
10
11 
  @behaviour OpenAgents.Forge.RollingProvider
12
13 
  alias OpenAgents.Cluster.Admission
14 
  alias OpenAgents.Cluster.Drain
15 
  alias OpenAgents.Forge.RollingNodeProbe
16 
  alias OpenAgents.Forge.RollingProvider.Gcp.Compute
17
18 
  @impl true
19 
  def remove_readiness(node, _context) do
20 
    with {:ok, config} <- config(),
21 
         :ok <- rpc(config, node, Admission, :remove, []) do
22 
      :ok
23 
    end
24 
  end
25
26 
  @impl true
27 
  def restore_readiness(node, _context) do
28 
    with {:ok, config} <- config(),
29 
         :ok <- rpc(config, node, Admission, :restore, []) do
30 
      :ok
31 
    end
32 
  end
33
34 
  @impl true
35 
  def drain(node, _context) do
36 
    with {:ok, config} <- config(),
37 
         {:ok, count} <- rpc(config, node, Drain, :drain, []) do
38 
      {:ok, count}
39 
    end
40 
  end
41
42 
  @impl true
43 
  def capacity(nodes, context) do
44 
    with {:ok, config} <- config(),
45 
         {:ok, probes} <- probes(config, nodes, length(context.expected_nodes)) do
46 
      ready = Enum.count(probes, & &1.ready)
47 
      majority = div(length(context.expected_nodes), 2) + 1
48 
      ra_quorum? = Enum.any?(probes, & &1.ra_quorum)
49 
      {:ok, %{ready: ready, quorum: ready >= majority and ra_quorum?}}
50 
    end
51 
  end
52
53 
  @impl true
54 
  def replace(node, digest, context) do
55 
    with {:ok, config} <- config(),
56 
         {:ok, instance} <- instance(config, node) do
57 
      compute(config, instance, context.sha, digest)
58 
    end
59 
  end
60
61 
  @impl true
62 
  def status(node, context) do
63 
    with {:ok, config} <- config(),
64 
         {:ok, probe} <- probe(config, node, length(context.expected_nodes)) do
65 
      {:ok,
66 
       Map.take(probe, [:member, :ready, :boot_converged, :database_ready, :sha, :image_digest])}
67 
    end
68 
  end
69
70 
  @impl true
71 
  def rollback(node, digest, context) do
72 
    with {:ok, config} <- config(),
73 
         {:ok, instance} <- instance(config, node) do
74 
      compute(config, instance, context.previous_sha, digest)
75 
    end
76 
  end
77
78 
  @doc "Validate the content-free staging provider configuration."
79 
  def validate_config(config) when is_list(config) do
80 
    project_id = Keyword.get(config, :project_id)
81 
    production_project_id = Keyword.get(config, :production_project_id)
82 
    zone = Keyword.get(config, :zone)
83 
    instances = Keyword.get(config, :instances)
84 
    image_repository = Keyword.get(config, :image_repository)
85 
    deployer_node = Keyword.get(config, :deployer_node)
86
87 
    cond do
88 
      not bounded_identifier?(project_id) -> {:error, :invalid_staging_project}
89 
      project_id == production_project_id -> {:error, :staging_project_matches_production}
90 
      not bounded_identifier?(production_project_id) -> {:error, :invalid_production_project}
91 
      not bounded_identifier?(zone) -> {:error, :invalid_staging_zone}
92 
      not valid_instances?(instances) -> {:error, :invalid_instance_map}
93 
      not image_repository?(image_repository) -> {:error, :invalid_image_repository}
94 
      not deployer_node?(deployer_node) -> {:error, :invalid_deployer_node}
95 
      true -> :ok
96 
    end
97 
  end
98
99 
  def validate_config(_config), do: {:error, :invalid_provider_config}
100
101 
  defp probes(config, nodes, expected_fleet_size) do
102 
    results =
103 
      Task.async_stream(
104 
        nodes,
105 
        &probe(config, &1, expected_fleet_size),
106 
        ordered: false,
107 
        timeout: timeout(config)
108 
      )
109 
      |> Enum.to_list()
110
111 
    case Enum.reduce_while(results, [], fn
112 
           {:ok, {:ok, probe}}, acc -> {:cont, [probe | acc]}
113 
           {:ok, {:error, reason}}, _acc -> {:halt, {:error, reason}}
114 
           {:exit, reason}, _acc -> {:halt, {:error, {:probe_exit, reason}}}
115 
         end) do
116 
      {:error, reason} -> {:error, reason}
117 
      probes -> {:ok, probes}
118 
    end
119 
  end
120
121 
  defp probe(config, node, expected_fleet_size) do
122 
    case rpc(config, node, RollingNodeProbe, :status, [expected_fleet_size]) do
123 
      %{member: true} = result -> {:ok, result}
124 
      {:error, reason} -> {:error, reason}
125 
      other -> {:error, {:invalid_node_probe, other}}
126 
    end
127 
  end
128
129 
  defp rpc(config, node, module, function, arguments, call_timeout \\ nil) do
130 
    rpc = Keyword.get(config, :rpc, &:erpc.call/5)
131
132 
    case rpc.(node, module, function, arguments, call_timeout || timeout(config)) do
133 
      {:error, _reason} = error -> error
134 
      result -> result
135 
    end
136 
  catch
137 
    :exit, reason -> {:error, {:rpc_exit, reason}}
138 
  end
139
140 
  defp instance(config, node) do
141 
    case get_in(config, [:instances, to_string(node)]) do
142 
      instance when is_binary(instance) and instance != "" -> {:ok, instance}
143 
      _missing -> {:error, :node_instance_not_configured}
144 
    end
145 
  end
146
147 
  defp driver(config), do: Keyword.get(config, :driver, Compute)
148 
  defp timeout(config), do: Keyword.get(config, :rpc_timeout_ms, 5_000)
149 
  defp compute_timeout(config), do: Keyword.get(config, :compute_timeout_ms, 300_000)
150
151 
  defp compute(config, instance, sha, digest) do
152 
    arguments = [instance, sha, digest, compute_config(config)]
153
154 
    rpc(
155 
      config,
156 
      Keyword.fetch!(config, :deployer_node),
157 
      driver(config),
158 
      :replace,
159 
      arguments,
160 
      compute_timeout(config)
161 
    )
162 
  end
163
164 
  defp compute_config(config) do
165 
    Keyword.take(config, [
166 
      :project_id,
167 
      :zone,
168 
      :image_repository,
169 
      :operation_attempts,
170 
      :operation_interval_ms,
171 
      :api_url
172 
    ])
173 
  end
174
175 
  defp config do
176 
    config = Application.get_env(:openagents, __MODULE__, [])
177
178 
    case validate_config(config) do
179 
      :ok -> {:ok, config}
180 
      {:error, _reason} = error -> error
181 
    end
182 
  end
183
184 
  defp valid_instances?(instances) when is_map(instances) and map_size(instances) == 3 do
185 
    Enum.all?(instances, fn {node_name, instance_name} ->
186 
      is_binary(node_name) and String.starts_with?(node_name, "openagents@") and
187 
        bounded_identifier?(instance_name)
188 
    end)
189 
  end
190
191 
  defp valid_instances?(_instances), do: false
192
193 
  defp bounded_identifier?(value) when is_binary(value) do
194 
    byte_size(value) in 1..63 and Regex.match?(~r/\A[a-z][a-z0-9-]*[a-z0-9]\z/, value)
195 
  end
196
197 
  defp bounded_identifier?(_value), do: false
198
199 
  defp image_repository?(value) when is_binary(value) do
200 
    byte_size(value) in 1..512 and
201 
      Regex.match?(
202 
        ~r/\A[a-z0-9.-]+-docker\.pkg\.dev\/[a-z0-9-]+\/[a-z0-9._-]+\/[a-z0-9._-]+\z/,
203 
        value
204 
      )
205 
  end
206
207 
  defp image_repository?(_value), do: false
208
209 
  defp deployer_node?(:"openagents-deployer@openagents-deployer.staging.internal"), do: true
210 
  defp deployer_node?(_node), do: false
211 
end
lib/openagents/forge/rolling_provider/gcp/compute.ex added +215

@@ -0,0 +1,215 @@

1 
defmodule OpenAgents.Forge.RollingProvider.Gcp.Compute do
2 
  @moduledoc """
3 
  Updates one staging instance's immutable image metadata and resets it.
4
5 
  Requests go directly to the Compute Engine API with a metadata-server token
6 
  or an injected test token. Errors expose only bounded operation and status
7 
  codes. Response bodies and access tokens never enter logs or receipts.
8 
  """
9
10 
  @api_url "https://compute.googleapis.com/compute/v1"
11 
  @metadata_token_url "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"
12 
  @sha_pattern ~r/\A[0-9a-f]{40}\z/
13 
  @digest_pattern ~r/\Asha256:[0-9a-f]{64}\z/
14 
  @instance_pattern ~r/\A[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?\z/
15
16 
  @doc "Replace one instance's application container with an exact image."
17 
  def replace(instance, sha, digest, config) do
18 
    with :ok <- validate(instance, sha, digest),
19 
         {:ok, token} <- token(config),
20 
         {:ok, metadata} <- instance_metadata(instance, token, config),
21 
         {:ok, operation} <- set_identity(instance, sha, digest, metadata, token, config),
22 
         :ok <- wait_for_operation(operation, token, config),
23 
         {:ok, operation} <- reset(instance, token, config),
24 
         :ok <- wait_for_operation(operation, token, config) do
25 
      :ok
26 
    end
27 
  end
28
29 
  defp instance_metadata(instance, token, config) do
30 
    case request(:get, instance_url(config, instance), nil, token, config) do
31 
      {:ok, %Req.Response{status: 200, body: %{"metadata" => metadata}}}
32 
      when is_map(metadata) ->
33 
        {:ok, metadata}
34
35 
      {:ok, %Req.Response{status: status}} ->
36 
        {:error, {:compute_api_error, :get_instance, status}}
37
38 
      {:error, reason} ->
39 
        {:error, {:compute_transport_error, :get_instance, safe_reason(reason)}}
40 
    end
41 
  end
42
43 
  defp set_identity(instance, sha, digest, metadata, token, config) do
44 
    with fingerprint when is_binary(fingerprint) <- Map.get(metadata, "fingerprint"),
45 
         items when is_list(items) <- Map.get(metadata, "items", []) do
46 
      identity = %{
47 
        "openagents-image" => Keyword.fetch!(config, :image_repository) <> "@" <> digest,
48 
        "openagents-image-digest" => digest,
49 
        "openagents-sha" => sha
50 
      }
51
52 
      body = %{
53 
        "fingerprint" => fingerprint,
54 
        "items" => merge_metadata(items, identity)
55 
      }
56
57 
      operation_request(
58 
        :post,
59 
        instance_url(config, instance) <> "/setMetadata",
60 
        body,
61 
        token,
62 
        config,
63 
        :set_metadata
64 
      )
65 
    else
66 
      _invalid -> {:error, :invalid_instance_metadata}
67 
    end
68 
  end
69
70 
  defp reset(instance, token, config) do
71 
    operation_request(
72 
      :post,
73 
      instance_url(config, instance) <> "/reset",
74 
      %{},
75 
      token,
76 
      config,
77 
      :reset
78 
    )
79 
  end
80
81 
  defp operation_request(method, url, body, token, config, action) do
82 
    case request(method, url, body, token, config) do
83 
      {:ok, %Req.Response{status: status, body: %{"name" => name} = operation}}
84 
      when status in 200..299 and is_binary(name) ->
85 
        {:ok, operation}
86
87 
      {:ok, %Req.Response{status: status}} ->
88 
        {:error, {:compute_api_error, action, status}}
89
90 
      {:error, reason} ->
91 
        {:error, {:compute_transport_error, action, safe_reason(reason)}}
92 
    end
93 
  end
94
95 
  defp wait_for_operation(%{"status" => "DONE", "error" => error}, _token, _config)
96 
       when not is_nil(error),
97 
       do: {:error, :compute_operation_failed}
98
99 
  defp wait_for_operation(%{"status" => "DONE"}, _token, _config), do: :ok
100
101 
  defp wait_for_operation(%{"name" => name}, token, config) do
102 
    attempts = Keyword.get(config, :operation_attempts, 120)
103 
    do_wait_for_operation(name, token, config, attempts)
104 
  end
105
106 
  defp wait_for_operation(_operation, _token, _config), do: {:error, :invalid_compute_operation}
107
108 
  defp do_wait_for_operation(_name, _token, _config, 0),
109 
    do: {:error, :compute_operation_timeout}
110
111 
  defp do_wait_for_operation(name, token, config, attempts) do
112 
    url =
113 
      api_url(config) <>
114 
        "/projects/#{Keyword.fetch!(config, :project_id)}/zones/" <>
115 
        "#{Keyword.fetch!(config, :zone)}/operations/#{name}"
116
117 
    case request(:get, url, nil, token, config) do
118 
      {:ok, %Req.Response{status: 200, body: %{"status" => "DONE", "error" => error}}}
119 
      when not is_nil(error) ->
120 
        {:error, :compute_operation_failed}
121
122 
      {:ok, %Req.Response{status: 200, body: %{"status" => "DONE"}}} ->
123 
        :ok
124
125 
      {:ok, %Req.Response{status: 200}} ->
126 
        wait(config)
127 
        do_wait_for_operation(name, token, config, attempts - 1)
128
129 
      {:ok, %Req.Response{status: status}} ->
130 
        {:error, {:compute_api_error, :get_operation, status}}
131
132 
      {:error, reason} ->
133 
        {:error, {:compute_transport_error, :get_operation, safe_reason(reason)}}
134 
    end
135 
  end
136
137 
  defp request(method, url, body, token, config) do
138 
    options = [
139 
      method: method,
140 
      url: url,
141 
      headers: [{"authorization", "Bearer " <> token}],
142 
      connect_options: [timeout: 2_000],
143 
      receive_timeout: 10_000,
144 
      retry: false
145 
    ]
146
147 
    options = if is_nil(body), do: options, else: Keyword.put(options, :json, body)
148 
    Req.request(Keyword.merge(options, Keyword.get(config, :request_options, [])))
149 
  end
150
151 
  defp token(config) do
152 
    case Keyword.get(config, :token_provider) do
153 
      provider when is_function(provider, 0) -> normalize_token(provider.())
154 
      nil -> metadata_token(config)
155 
      _invalid -> {:error, :invalid_token_provider}
156 
    end
157 
  end
158
159 
  defp metadata_token(config) do
160 
    options = [
161 
      url: @metadata_token_url,
162 
      headers: [{"metadata-flavor", "Google"}],
163 
      connect_options: [timeout: 2_000],
164 
      receive_timeout: 5_000,
165 
      retry: false
166 
    ]
167
168 
    case Req.get(Keyword.merge(options, Keyword.get(config, :request_options, []))) do
169 
      {:ok, %Req.Response{status: 200, body: %{"access_token" => token}}} ->
170 
        normalize_token(token)
171
172 
      {:ok, %Req.Response{status: status}} ->
173 
        {:error, {:metadata_token_error, status}}
174
175 
      {:error, reason} ->
176 
        {:error, {:metadata_token_error, safe_reason(reason)}}
177 
    end
178 
  end
179
180 
  defp normalize_token({:ok, token}), do: normalize_token(token)
181 
  defp normalize_token(token) when is_binary(token) and token != "", do: {:ok, token}
182 
  defp normalize_token(_invalid), do: {:error, :invalid_access_token}
183
184 
  defp merge_metadata(items, identity) do
185 
    retained = Enum.reject(items, &Map.has_key?(identity, Map.get(&1, "key")))
186 
    additions = Enum.map(identity, fn {key, value} -> %{"key" => key, "value" => value} end)
187 
    Enum.sort_by(retained ++ additions, &Map.get(&1, "key", ""))
188 
  end
189
190 
  defp instance_url(config, instance) do
191 
    api_url(config) <>
192 
      "/projects/#{Keyword.fetch!(config, :project_id)}/zones/" <>
193 
      "#{Keyword.fetch!(config, :zone)}/instances/#{instance}"
194 
  end
195
196 
  defp api_url(config), do: Keyword.get(config, :api_url, @api_url)
197
198 
  defp validate(instance, sha, digest) do
199 
    if Regex.match?(@instance_pattern, instance) and Regex.match?(@sha_pattern, sha) and
200 
         Regex.match?(@digest_pattern, digest),
201 
       do: :ok,
202 
       else: {:error, :invalid_replacement_identity}
203 
  end
204
205 
  defp wait(config) do
206 
    receive do
207 
    after
208 
      Keyword.get(config, :operation_interval_ms, 1_000) -> :ok
209 
    end
210 
  end
211
212 
  defp safe_reason(%{reason: reason}) when is_atom(reason), do: reason
213 
  defp safe_reason(reason) when is_atom(reason), do: reason
214 
  defp safe_reason(_reason), do: :request_failed
215 
end
lib/openagents/forge/rolling_provider/gcp/deployer.ex added +25

@@ -0,0 +1,25 @@

1 
defmodule OpenAgents.Forge.RollingProvider.Gcp.Deployer do
2 
  @moduledoc """
3 
  Starts the minimal private BEAM node used for Google Cloud replacement calls.
4
5 
  The deployer container starts this module with the release's clean boot file.
6 
  It does not start the OpenAgents application, join Ra, open an HTTP listener,
7 
  or connect to PostgreSQL. It starts only the dependencies needed by `Req` and
8 
  then waits for bounded `erpc` calls from the three-node staging fleet.
9 
  """
10
11 
  @sha_pattern ~r/\A[0-9a-f]{40}\z/
12
13 
  @doc false
14 
  def start do
15 
    expected_revision = System.fetch_env!("OPENAGENTS_CONTROLLER_SHA")
16
17 
    unless Regex.match?(@sha_pattern, expected_revision) and
18 
             expected_revision == OpenAgents.BuildInfo.revision() do
19 
      raise "deployer image revision does not match its assigned Git SHA"
20 
    end
21
22 
    {:ok, _applications} = Application.ensure_all_started(:req)
23 
    Process.sleep(:infinity)
24 
  end
25 
end
lib/openagents/forge/rolling_replacement.ex modified +2

@@ -308,7 +308,9 @@ defmodule OpenAgents.Forge.RollingReplacement do

308 308 
  defp context(request) do
309 309 
    %{
310 310 
      sha: request.sha,
311 
      previous_sha: request.previous_sha,
311 312 
      image_digest: request.image_digest,
313 
      previous_image_digest: request.previous_image_digest,
312 314 
      expected_nodes: request.expected_nodes
313 315 
    }
314 316 
  end
lib/openagents/runtime_config.ex modified +47

@@ -9,6 +9,7 @@ defmodule OpenAgents.RuntimeConfig do

9 9 
  """
10 10
11 11 
  alias OpenAgents.Forge.HotLoader
12 
  alias OpenAgents.Forge.RollingProvider.Gcp
12 13 
  alias OpenAgents.Tools.Snapshot
13 14
14 15 
  @persistent_key {__MODULE__, :current}

@@ -67,6 +68,7 @@ defmodule OpenAgents.RuntimeConfig do

67 68 
         :ok <- validate_github(settings, environment),
68 69 
         {:ok, features} <- validate_features(settings, environment, staging_gate),
69 70 
         :ok <- validate_providers(settings, features),
71 
         :ok <- validate_release_identity(settings, environment, staging_gate),
70 72 
         {:ok, allowlist, examples} <- validate_forge(settings, environment, features),
71 73 
         :ok <- validate_cluster(settings, environment, features) do
72 74 
      {:ok,

@@ -146,6 +148,26 @@ defmodule OpenAgents.RuntimeConfig do

146 148
147 149 
  defp validate_production_lock(_settings, _environment), do: :ok
148 150
151 
  defp validate_release_identity(settings, environment, staging_gate) do
152 
    required? = environment == :production or (environment == :staging and staging_gate >= 12)
153 
    revision = Map.get(settings, :build_revision)
154 
    image_digest = Map.get(settings, :image_digest)
155
156 
    cond do
157 
      not required? ->
158 
        :ok
159
160 
      not exact_sha?(revision) ->
161 
        error(:build_revision, "must identify the exact packaged Git commit")
162
163 
      not image_digest?(image_digest) ->
164 
        error(:image_digest, "must identify the immutable packaged image")
165
166 
      true ->
167 
        :ok
168 
    end
169 
  end
170
149 171 
  defp verify_compiled_settings!(%__MODULE__{environment: environment} = config)
150 172 
       when environment in [:staging, :production] do
151 173 
    if keyword_value(OpenAgentsWeb.Endpoint.session_options(), :secure) == true do

@@ -269,6 +291,16 @@ defmodule OpenAgents.RuntimeConfig do

269 291
270 292 
  defp decryption_keyring?(_keys, _active_key_id, _environment), do: false
271 293
294 
  defp exact_sha?(value) when is_binary(value),
295 
    do: Regex.match?(~r/\A[0-9a-f]{40}\z/, value)
296
297 
  defp exact_sha?(_value), do: false
298
299 
  defp image_digest?(value) when is_binary(value),
300 
    do: Regex.match?(~r/\Asha256:[0-9a-f]{64}\z/, value)
301
302 
  defp image_digest?(_value), do: false
303
272 304 
  defp validate_features(settings, environment, staging_gate) do
273 305 
    with {:ok, tools?} <- required_boolean(settings, :tools_enabled),
274 306 
         {:ok, voice?} <- nested_boolean(settings, :voice, :enabled),

@@ -455,6 +487,8 @@ defmodule OpenAgents.RuntimeConfig do

455 487 
    boot_retry_max_ms = Map.get(settings, :forge_boot_retry_max_ms)
456 488 
    operator_token = Map.get(settings, :forge_operator_token)
457 489 
    mirror_urls = Map.get(settings, :forge_mirror_urls)
490 
    rolling_provider = Map.get(settings, :forge_rolling_provider)
491 
    rolling_provider_config = Map.get(settings, Gcp, [])
458 492 
    durable_required? = environment in [:staging, :production] and features.forge
459 493
460 494 
    with :ok <-

@@ -534,6 +568,7 @@ defmodule OpenAgents.RuntimeConfig do

534 568 
             :forge_expected_fleet_size,
535 569 
             "must include a canary and peer for deployment"
536 570 
           ),
571 
         :ok <- validate_rolling_provider(rolling_provider, rolling_provider_config, features),
537 572 
         :ok <- validate_forge_secrets(operator_token, durable_required? or features.forge_deploy),
538 573 
         :ok <- validate_forge_paths(settings, durable_required? or features.forge_deploy),
539 574 
         :ok <- validate_wal(settings, durable_required? or features.forge_deploy) do

@@ -541,6 +576,18 @@ defmodule OpenAgents.RuntimeConfig do

541 576 
    end
542 577 
  end
543 578
579 
  defp validate_rolling_provider(_provider, _config, %{forge_deploy: false}), do: :ok
580
581 
  defp validate_rolling_provider(Gcp, config, %{forge_deploy: true}) do
582 
    case Gcp.validate_config(config) do
583 
      :ok -> :ok
584 
      {:error, _reason} -> error(:forge_rolling_provider, "must use an isolated staging project")
585 
    end
586 
  end
587
588 
  defp validate_rolling_provider(_provider, _config, %{forge_deploy: true}),
589 
    do: error(:forge_rolling_provider, "must be the admitted infrastructure provider")
590
544 591 
  defp validate_forge_secrets(operator_token, true) do
545 592 
    if present?(operator_token),
546 593 
      do: :ok,
lib/openagents_web/controllers/health_controller.ex modified +2 -1

@@ -13,7 +13,8 @@ defmodule OpenAgentsWeb.HealthController do

13 13 
          status: "unavailable",
14 14 
          reason: "runtime_not_ready",
15 15 
          boot_converged: report["boot_converged"],
16 
          deployment_ready: report["deployment_ready"]
16 
          deployment_ready: report["deployment_ready"],
17 
          admission_ready: report["admission_ready"]
17 18 
        })
18 19
19 20 
      {{:error, _reason}, _report} ->
ops/ci/gate.sh modified +3 -1

@@ -4,7 +4,7 @@ set -eu

4 4 
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
5 5 
repo_root=$(CDPATH= cd -- "$script_dir/../.." && pwd)
6 6 
receipt_root="$repo_root/.git/openagents/release-gate-receipts"
7 
required_stages='compile production_compile precommit cluster javascript direct_transaction relup version_chain interrupted_install rolling_replacement contracts release_smoke'
7 
required_stages='compile production_compile precommit cluster javascript direct_transaction relup version_chain interrupted_install rolling_replacement contracts staging_infra release_smoke'
8 8 
mode=${1:-run}
9 9
10 10 
if [ ! -d "$repo_root/.git" ]; then

@@ -132,6 +132,7 @@ run_stage interrupted_install env \

132 132 
run_stage rolling_replacement env MIX_ENV=test mix test --warnings-as-errors \
133 133 
  test/openagents/forge/rolling_replacement_test.exs
134 134 
run_stage contracts ops/ci/contracts.sh
135 
run_stage staging_infra ops/ci/staging-infra.sh
135 136 
run_stage release_smoke ops/ci/release-smoke.sh
136 137
137 138 
if [ "$(git rev-parse --verify HEAD)" != "$git_sha" ]; then

@@ -173,6 +174,7 @@ cat >"$receipt_temp" <<EOF

173 174 
    "interrupted_install": {"status": "passed", "duration_seconds": $interrupted_install_duration_seconds},
174 175 
    "rolling_replacement": {"status": "passed", "duration_seconds": $rolling_replacement_duration_seconds},
175 176 
    "contracts": {"status": "passed", "duration_seconds": $contracts_duration_seconds},
177 
    "staging_infra": {"status": "passed", "duration_seconds": $staging_infra_duration_seconds},
176 178 
    "release_smoke": {"status": "passed", "duration_seconds": $release_smoke_duration_seconds}
177 179 
  }
178 180 
}
ops/ci/staging-infra.sh added +16

@@ -0,0 +1,16 @@

1 
#!/bin/sh
2 
set -eu
3
4 
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
5 
repo_root=$(CDPATH= cd -- "$script_dir/../.." && pwd)
6 
terraform_root="$repo_root/infra/staging"
7
8 
if ! command -v terraform >/dev/null 2>&1; then
9 
  echo "terraform is required for the staging infrastructure gate" >&2
10 
  exit 1
11 
fi
12
13 
terraform -chdir="$terraform_root" fmt -check -recursive
14 
terraform -chdir="$terraform_root" init -backend=false -input=false -no-color
15 
terraform -chdir="$terraform_root" validate -no-color
16 
terraform -chdir="$terraform_root" test -no-color
ops/deploy/build-image.sh modified +14

@@ -24,6 +24,7 @@ trap cleanup EXIT INT TERM

24 24 
"$repo_root/ops/ci/gate.sh" --verify
25 25
26 26 
docker build \
27 
  --build-arg "OPENAGENTS_BUILD_REVISION=$git_sha" \
27 28 
  --iidfile "$iid_file" \
28 29 
  --label "org.opencontainers.image.revision=$git_sha" \
29 30 
  --tag "$tag" \

@@ -40,6 +41,19 @@ case "$image_digest" in

40 41 
    ;;
41 42 
esac
42 43
44 
embedded_revision=$(
45 
  docker run --rm \
46 
    --entrypoint /bin/sh \
47 
    "$image_digest" \
48 
    -c 'release_version=$(awk '\''{print $2}'\'' /app/releases/start_erl.data); /app/erts-*/bin/erl -boot_var RELEASE_LIB /app/lib -boot "/app/releases/$release_version/start_clean" -noshell -pa /app/lib/openagents-*/ebin -eval "io:put_chars('\''Elixir.OpenAgents.BuildInfo'\'':revision()), halt()."' \
49 
    | tail -n 1
50 
)
51
52 
if [ "$embedded_revision" != "$git_sha" ]; then
53 
  echo "packaged BuildInfo revision does not match the exact Git SHA" >&2
54 
  exit 1
55 
fi
56
43 57 
mkdir -p "$image_root"
44 58 
umask 077
45 59
ops/staging/bootstrap-project.sh added +75

@@ -0,0 +1,75 @@

1 
#!/bin/sh
2 
set -eu
3
4 
mode=${1:-check}
5 
staging_project=${OPENAGENTS_STAGING_PROJECT_ID:-}
6 
production_project=${OPENAGENTS_PRODUCTION_PROJECT_ID:-}
7 
billing_account=${OPENAGENTS_STAGING_BILLING_ACCOUNT:-}
8 
state_bucket=${OPENAGENTS_STAGING_TF_STATE_BUCKET:-${staging_project}-openagents-tfstate}
9 
location=${OPENAGENTS_STAGING_STATE_LOCATION:-US-CENTRAL1}
10
11 
require_inputs() {
12 
  : "${staging_project:?OPENAGENTS_STAGING_PROJECT_ID is required}"
13 
  : "${production_project:?OPENAGENTS_PRODUCTION_PROJECT_ID is required}"
14 
  : "${billing_account:?OPENAGENTS_STAGING_BILLING_ACCOUNT is required}"
15
16 
  case "$staging_project" in
17 
    *stag*) ;;
18 
    *) echo "staging project ID must contain 'stag'" >&2; exit 1 ;;
19 
  esac
20
21 
  if [ "$staging_project" = "$production_project" ]; then
22 
    echo "staging and production project IDs must differ" >&2
23 
    exit 1
24 
  fi
25 
}
26
27 
describe_actions() {
28 
  echo "Staging project: $staging_project"
29 
  echo "Production comparison project: $production_project"
30 
  echo "Billing account: $billing_account"
31 
  echo "Terraform state bucket: gs://$state_bucket"
32 
  echo "Mode: $mode"
33 
}
34
35 
apply_bootstrap() {
36 
  gcloud auth print-access-token >/dev/null
37
38 
  if ! gcloud projects describe "$staging_project" >/dev/null 2>&1; then
39 
    gcloud projects create "$staging_project" --name="OpenAgents staging"
40 
  fi
41
42 
  gcloud billing projects link "$staging_project" --billing-account="$billing_account"
43 
  gcloud services enable serviceusage.googleapis.com storage.googleapis.com \
44 
    --project="$staging_project"
45
46 
  if ! gcloud storage buckets describe "gs://$state_bucket" \
47 
    --project="$staging_project" >/dev/null 2>&1; then
48 
    gcloud storage buckets create "gs://$state_bucket" \
49 
      --project="$staging_project" \
50 
      --location="$location" \
51 
      --uniform-bucket-level-access \
52 
      --public-access-prevention \
53 
      --soft-delete-duration=7d
54 
  fi
55
56 
  gcloud storage buckets update "gs://$state_bucket" --versioning
57 
}
58
59 
require_inputs
60
61 
case "$mode" in
62 
  check)
63 
    describe_actions
64 
    ;;
65
66 
  --apply)
67 
    describe_actions
68 
    apply_bootstrap
69 
    ;;
70
71 
  *)
72 
    echo "usage: ops/staging/bootstrap-project.sh [check|--apply]" >&2
73 
    exit 64
74 
    ;;
75 
esac
ops/staging/gate-5-profile.sh modified +9

@@ -62,9 +62,18 @@ export OPENAGENTS_FORGE_WAL_ADAPTER="local"

62 62 
export OPENAGENTS_FORGE_WAL_BUCKET=""
63 63 
export OPENAGENTS_FORGE_WAL_DIR="/var/lib/openagents/forge-wal"
64 64 
export OPENAGENTS_HTTPS_ALIASES=""
65 
export OPENAGENTS_IMAGE_DIGEST=""
65 66 
export OPENAGENTS_INFERENCE_PROXY_URL=""
66 67 
export OPENAGENTS_MACHINE_TOKEN_TTL_SECONDS="2592000"
67 68 
export OPENAGENTS_MIGRATE_ON_BOOT="true"
69 
export OPENAGENTS_FORGE_ROLLING_PROVIDER=""
70 
export OPENAGENTS_GCP_IMAGE_REPOSITORY=""
71 
export OPENAGENTS_GCP_COMPUTE_TIMEOUT_MS="300000"
72 
export OPENAGENTS_GCP_ROLLING_INSTANCES_JSON=""
73 
export OPENAGENTS_GCP_ROLLING_PROJECT_ID=""
74 
export OPENAGENTS_GCP_ROLLING_RPC_TIMEOUT_MS="5000"
75 
export OPENAGENTS_GCP_ROLLING_ZONE=""
76 
export OPENAGENTS_PRODUCTION_PROJECT_ID=""
68 77 
export OPENAGENTS_PRODUCTION_DEPLOY_ENABLED="false"
69 78 
export OPENAGENTS_RA_DATA_DIR="/var/lib/openagents/ra"
70 79 
export OPENAGENTS_RA_EXPECTED_SIZE="3"
ops/staging/terraform.sh added +97

@@ -0,0 +1,97 @@

1 
#!/bin/sh
2 
set -eu
3
4 
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
5 
repo_root=$(CDPATH= cd -- "$script_dir/../.." && pwd)
6 
terraform_dir="$repo_root/infra/staging"
7 
command_name=${1:-validate}
8 
confirmation=${2:-}
9 
staging_project=${OPENAGENTS_STAGING_PROJECT_ID:-}
10 
production_project=${OPENAGENTS_PRODUCTION_PROJECT_ID:-}
11 
state_bucket=${OPENAGENTS_STAGING_TF_STATE_BUCKET:-}
12 
git_sha=$(git -C "$repo_root" rev-parse --verify HEAD)
13 
plan_root="$repo_root/.git/openagents/staging-plans"
14 
plan_path="$plan_root/$git_sha.tfplan"
15
16 
require_boundary() {
17 
  : "${staging_project:?OPENAGENTS_STAGING_PROJECT_ID is required}"
18 
  : "${production_project:?OPENAGENTS_PRODUCTION_PROJECT_ID is required}"
19 
  : "${state_bucket:?OPENAGENTS_STAGING_TF_STATE_BUCKET is required}"
20 
  : "${TF_VAR_database_password:?TF_VAR_database_password is required and remains write-only}"
21
22 
  case "$staging_project" in
23 
    *stag*) ;;
24 
    *) echo "staging project ID must contain 'stag'" >&2; exit 1 ;;
25 
  esac
26
27 
  if [ "$staging_project" = "$production_project" ]; then
28 
    echo "staging and production project IDs must differ" >&2
29 
    exit 1
30 
  fi
31
32 
  gcloud auth application-default print-access-token >/dev/null
33 
  export TF_VAR_staging_project_id="$staging_project"
34 
  export TF_VAR_production_project_id="$production_project"
35 
}
36
37 
initialize_backend() {
38 
  terraform -chdir="$terraform_dir" init \
39 
    -input=false \
40 
    -reconfigure \
41 
    -backend-config="bucket=$state_bucket"
42 
}
43
44 
case "$command_name" in
45 
  validate)
46 
    terraform -chdir="$terraform_dir" fmt -check -recursive
47 
    terraform -chdir="$terraform_dir" init -backend=false -input=false
48 
    terraform -chdir="$terraform_dir" validate
49 
    ;;
50
51 
  plan)
52 
    require_boundary
53
54 
    if [ -n "$(git -C "$repo_root" status --porcelain --untracked-files=all)" ]; then
55 
      echo "staging plan requires a clean worktree" >&2
56 
      exit 1
57 
    fi
58
59 
    initialize_backend
60 
    mkdir -p "$plan_root"
61 
    terraform -chdir="$terraform_dir" plan -input=false -out="$plan_path"
62 
    echo "Saved exact-SHA staging plan: .git/openagents/staging-plans/$git_sha.tfplan"
63 
    ;;
64
65 
  apply)
66 
    require_boundary
67
68 
    if [ "$confirmation" != "--apply" ]; then
69 
      echo "staging apply requires the explicit --apply argument" >&2
70 
      exit 64
71 
    fi
72
73 
    if [ -n "$(git -C "$repo_root" status --porcelain --untracked-files=all)" ]; then
74 
      echo "staging apply requires a clean worktree" >&2
75 
      exit 1
76 
    fi
77
78 
    if [ ! -f "$plan_path" ]; then
79 
      echo "no Terraform plan exists for exact SHA $git_sha" >&2
80 
      exit 1
81 
    fi
82
83 
    initialize_backend
84 
    terraform -chdir="$terraform_dir" apply -input=false "$plan_path"
85 
    ;;
86
87 
  output)
88 
    require_boundary
89 
    initialize_backend
90 
    terraform -chdir="$terraform_dir" output -json
91 
    ;;
92
93 
  *)
94 
    echo "usage: ops/staging/terraform.sh [validate|plan|apply --apply|output]" >&2
95 
    exit 64
96 
    ;;
97 
esac
ops/staging/validate-isolation.sh added +172

@@ -0,0 +1,172 @@

1 
#!/bin/sh
2 
set -eu
3
4 
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
5 
repo_root=$(CDPATH= cd -- "$script_dir/../.." && pwd)
6 
staging_project=${OPENAGENTS_STAGING_PROJECT_ID:-}
7 
production_project=${OPENAGENTS_PRODUCTION_PROJECT_ID:-}
8 
git_sha=$(git -C "$repo_root" rev-parse --verify HEAD)
9 
receipt_root="$repo_root/.git/openagents/staging-isolation"
10 
receipt_path="$receipt_root/$git_sha.json"
11
12 
: "${staging_project:?OPENAGENTS_STAGING_PROJECT_ID is required}"
13 
: "${production_project:?OPENAGENTS_PRODUCTION_PROJECT_ID is required}"
14
15 
case "$staging_project" in
16 
  *stag*) ;;
17 
  *) echo "staging project ID must contain 'stag'" >&2; exit 1 ;;
18 
esac
19
20 
if [ "$staging_project" = "$production_project" ]; then
21 
  echo "staging and production project IDs must differ" >&2
22 
  exit 1
23 
fi
24
25 
for command_name in gcloud git jq sha256sum; do
26 
  if ! command -v "$command_name" >/dev/null 2>&1; then
27 
    echo "$command_name is required" >&2
28 
    exit 1
29 
  fi
30 
done
31
32 
gcloud auth print-access-token >/dev/null
33
34 
run_root=$(mktemp -d /tmp/openagents-staging-isolation.XXXXXX)
35 
cleanup() {
36 
  find "$run_root" -depth -delete
37 
}
38 
trap cleanup EXIT INT TERM
39
40 
staging_number=$(gcloud projects describe "$staging_project" --format='value(projectNumber)')
41 
production_number=$(gcloud projects describe "$production_project" --format='value(projectNumber)')
42
43 
if [ "$staging_number" = "$production_number" ]; then
44 
  echo "staging and production resolve to the same project number" >&2
45 
  exit 1
46 
fi
47
48 
gcloud sql instances list --project="$staging_project" --format=json >"$run_root/sql.json"
49 
gcloud sql users list --project="$staging_project" \
50 
  --instance=openagents-staging-postgres --format=json >"$run_root/sql-users.json"
51 
gcloud compute instances list --project="$staging_project" --format=json >"$run_root/instances.json"
52 
gcloud storage buckets list --project="$staging_project" --format=json >"$run_root/buckets.json"
53 
gcloud secrets list --project="$staging_project" --format=json >"$run_root/secrets.json"
54 
gcloud iam service-accounts list --project="$staging_project" --format=json >"$run_root/accounts.json"
55 
gcloud dns managed-zones list --project="$staging_project" --format=json >"$run_root/dns.json"
56 
gcloud compute networks list --project="$staging_project" --format=json >"$run_root/networks.json"
57 
gcloud projects get-iam-policy "$staging_project" --format=json >"$run_root/staging-iam.json"
58 
gcloud iam service-accounts list --project="$production_project" --format=json >"$run_root/production-accounts.json"
59
60 
jq -e '
61 
  length == 1 and
62 
  .[0].name == "openagents-staging-postgres" and
63 
  .[0].state == "RUNNABLE" and
64 
  ([.[0].ipAddresses[]? | select(.type == "PRIMARY")] | length) == 0 and
65 
  .[0].settings.userLabels.environment == "staging"
66 
' "$run_root/sql.json" >/dev/null
67
68 
jq -e '
69 
  any(.[]; .name == "openagents_staging" and .type == "BUILT_IN")
70 
' "$run_root/sql-users.json" >/dev/null
71
72 
jq -e '
73 
  [
74 
    .[] |
75 
    select(.labels.environment == "staging" and .labels.lane == "distributed")
76 
  ] as $fleet |
77 
  [
78 
    .[] |
79 
    select(.labels.environment == "staging" and .labels.lane == "deployer")
80 
  ] as $deployer |
81 
  ($fleet | length) == 3 and
82 
  all($fleet[]; ([.networkInterfaces[].accessConfigs[]?] | length) == 0) and
83 
  ($deployer | length) == 1 and
84 
  all($deployer[]; ([.networkInterfaces[].accessConfigs[]?] | length) == 0) and
85 
  ([$fleet[].name] | sort) == [
86 
    "openagents-fleet-1",
87 
    "openagents-fleet-2",
88 
    "openagents-fleet-3"
89 
  ]
90 
' "$run_root/instances.json" >/dev/null
91
92 
jq -e --arg project "$staging_project" '
93 
  [.[].name] as $names |
94 
  all([
95 
    "\($project)-openagents-artifacts",
96 
    "\($project)-openagents-evidence",
97 
    "\($project)-openagents-forge-wal",
98 
    "\($project)-openagents-recordings"
99 
  ][]; . as $required | $names | index($required))
100 
' "$run_root/buckets.json" >/dev/null
101
102 
jq -e '
103 
  [.[].name | split("/")[-1]] as $names |
104 
  all([
105 
    "openagents-staging-builder-config",
106 
    "openagents-staging-database-url",
107 
    "openagents-staging-fleet-config",
108 
    "openagents-staging-forge-operator-token",
109 
    "openagents-staging-github-client-secret",
110 
    "openagents-staging-github-vault-active",
111 
    "openagents-staging-github-vault-previous",
112 
    "openagents-staging-openai-api-key",
113 
    "openagents-staging-release-cookie",
114 
    "openagents-staging-secret-key-base",
115 
    "openagents-staging-voice-recording-key",
116 
    "openagents-staging-web-config"
117 
  ][]; . as $required | $names | index($required))
118 
' "$run_root/secrets.json" >/dev/null
119
120 
jq -e '
121 
  [.[].email | split("@") | first] as $names |
122 
  all([
123 
    "openagents-staging-deployer",
124 
    "openagents-staging-fleet",
125 
    "openagents-staging-web"
126 
  ][]; . as $required | $names | index($required))
127 
' "$run_root/accounts.json" >/dev/null
128
129 
jq -e 'any(.[]; .name == "openagents-staging-internal" and .visibility == "private")' \
130 
  "$run_root/dns.json" >/dev/null
131 
jq -e 'any(.[]; .name == "openagents-staging")' "$run_root/networks.json" >/dev/null
132
133 
jq -n \
134 
  --slurpfile policy "$run_root/staging-iam.json" \
135 
  --slurpfile production "$run_root/production-accounts.json" '
136 
    [$production[0][].email | "serviceAccount:" + .] as $production_members |
137 
    [$policy[0].bindings[].members[]?] as $staging_members |
138 
    [$staging_members[] | select(. as $member | $production_members | index($member))] |
139 
    length == 0
140 
  ' | jq -e . >/dev/null
141
142 
project_fingerprint=$(printf '%s' "$staging_project" | sha256sum | cut -d ' ' -f 1)
143 
production_fingerprint=$(printf '%s' "$production_project" | sha256sum | cut -d ' ' -f 1)
144 
mkdir -p "$receipt_root"
145 
umask 077
146
147 
jq -n \
148 
  --arg sha "$git_sha" \
149 
  --arg staging "$project_fingerprint" \
150 
  --arg production "$production_fingerprint" '
151 
  {
152 
    schema: "openagents.staging-isolation.v1",
153 
    git_sha: $sha,
154 
    status: "passed",
155 
    staging_project_fingerprint: $staging,
156 
    production_project_fingerprint: $production,
157 
    checks: {
158 
      distinct_projects: "passed",
159 
      private_database_instance: "passed",
160 
      staging_database_role: "passed",
161 
      three_private_fleet_nodes: "passed",
162 
      staging_buckets: "passed",
163 
      staging_secrets: "passed",
164 
      split_service_accounts: "passed",
165 
      private_dns_and_network: "passed",
166 
      no_production_service_accounts: "passed"
167 
    }
168 
  }
169 
' >"$receipt_path"
170
171 
echo "Staging isolation validation passed for $git_sha"
172 
echo "Receipt: .git/openagents/staging-isolation/$git_sha.json"
test/openagents/cluster_test.exs modified +13

@@ -10,12 +10,25 @@ defmodule OpenAgents.ClusterTest do

10 10 
    assert report["schema"] == "openagents.cluster_health.v1"
11 11 
    assert report["node"] == to_string(Node.self())
12 12 
    assert report["revision"] == OpenAgents.BuildInfo.revision()
13 
    assert report["image_digest"] == OpenAgents.BuildInfo.image_digest()
14 
    assert report["admission_ready"] == true
13 15 
    assert report["version"] == to_string(Application.spec(:openagents, :vsn) || "unknown")
14 16 
    assert report["live"] == true
15 17 
    assert report["ready"] == true
16 18 
    assert is_integer(report["uptime_ms"])
17 19 
  end
18 20
21 
  test "admission fences local readiness" do
22 
    on_exit(&OpenAgents.Cluster.Admission.restore/0)
23
24 
    assert :ok = OpenAgents.Cluster.Admission.remove()
25 
    refute Cluster.local_report()["ready"]
26 
    refute Cluster.local_report()["admission_ready"]
27
28 
    assert :ok = OpenAgents.Cluster.Admission.restore()
29 
    assert Cluster.local_report()["admission_ready"]
30 
  end
31
19 32 
  test "quorum and snapshot in single-node mode" do
20 33 
    assert Cluster.size() == 1
21 34 
    assert Cluster.members() == [Node.self()]
test/openagents/forge/gate_receipt_test.exs modified +1

@@ -16,6 +16,7 @@ defmodule OpenAgents.Forge.GateReceiptTest do

16 16 
    interrupted_install
17 17 
    rolling_replacement
18 18 
    contracts
19 
    staging_infra
19 20 
    release_smoke
20 21 
  )
21 22
test/openagents/forge/rolling_provider/gcp/compute_test.exs added +91

@@ -0,0 +1,91 @@

1 
defmodule OpenAgents.Forge.RollingProvider.Gcp.ComputeTest do
2 
  use ExUnit.Case, async: true
3
4 
  alias OpenAgents.Forge.RollingProvider.Gcp.Compute
5
6 
  setup {Req.Test, :verify_on_exit!}
7
8 
  @sha String.duplicate("a", 40)
9 
  @digest "sha256:" <> String.duplicate("b", 64)
10
11 
  test "updates bounded identity metadata and resets one exact instance" do
12 
    Req.Test.expect(__MODULE__, fn conn ->
13 
      assert conn.method == "GET"
14
15 
      assert conn.request_path ==
16 
               "/compute/v1/projects/staging-project/zones/us-central1-a/instances/fleet-1"
17
18 
      assert Plug.Conn.get_req_header(conn, "authorization") == ["Bearer test-token"]
19
20 
      Req.Test.json(conn, %{
21 
        "metadata" => %{
22 
          "fingerprint" => "fingerprint-1",
23 
          "items" => [
24 
            %{"key" => "retained", "value" => "yes"},
25 
            %{"key" => "openagents-sha", "value" => "old"}
26 
          ]
27 
        }
28 
      })
29 
    end)
30
31 
    Req.Test.expect(__MODULE__, fn conn ->
32 
      assert conn.method == "POST"
33
34 
      assert conn.request_path ==
35 
               "/compute/v1/projects/staging-project/zones/us-central1-a/instances/fleet-1/setMetadata"
36
37 
      body = conn |> Req.Test.raw_body() |> Jason.decode!()
38 
      assert body["fingerprint"] == "fingerprint-1"
39
40 
      assert Map.new(body["items"], &{&1["key"], &1["value"]}) == %{
41 
               "openagents-image" =>
42 
                 "us-central1-docker.pkg.dev/staging-project/openagents/app@#{@digest}",
43 
               "openagents-image-digest" => @digest,
44 
               "openagents-sha" => @sha,
45 
               "retained" => "yes"
46 
             }
47
48 
      Req.Test.json(conn, %{"name" => "set-metadata-1", "status" => "DONE"})
49 
    end)
50
51 
    Req.Test.expect(__MODULE__, fn conn ->
52 
      assert conn.method == "POST"
53
54 
      assert conn.request_path ==
55 
               "/compute/v1/projects/staging-project/zones/us-central1-a/instances/fleet-1/reset"
56
57 
      Req.Test.json(conn, %{"name" => "reset-1", "status" => "DONE"})
58 
    end)
59
60 
    assert :ok = Compute.replace("fleet-1", @sha, @digest, config())
61 
  end
62
63 
  test "returns only bounded status when the API response contains private data" do
64 
    Req.Test.expect(__MODULE__, fn conn ->
65 
      conn
66 
      |> Plug.Conn.put_status(500)
67 
      |> Req.Test.json(%{"access_token" => "private-sentinel"})
68 
    end)
69
70 
    result = Compute.replace("fleet-1", @sha, @digest, config())
71 
    assert result == {:error, {:compute_api_error, :get_instance, 500}}
72 
    refute inspect(result) =~ "private-sentinel"
73 
  end
74
75 
  test "refuses malformed identity before making a request" do
76 
    assert {:error, :invalid_replacement_identity} =
77 
             Compute.replace("fleet-1", "not-a-sha", @digest, config())
78 
  end
79
80 
  defp config do
81 
    [
82 
      project_id: "staging-project",
83 
      zone: "us-central1-a",
84 
      image_repository: "us-central1-docker.pkg.dev/staging-project/openagents/app",
85 
      token_provider: fn -> {:ok, "test-token"} end,
86 
      request_options: [plug: {Req.Test, __MODULE__}],
87 
      operation_attempts: 1,
88 
      operation_interval_ms: 0
89 
    ]
90 
  end
91 
end
test/openagents/forge/rolling_provider/gcp_test.exs added +156

@@ -0,0 +1,156 @@

1 
defmodule OpenAgents.Forge.RollingProvider.GcpTest do
2 
  use ExUnit.Case, async: false
3
4 
  alias OpenAgents.Cluster.Admission
5 
  alias OpenAgents.Cluster.Drain
6 
  alias OpenAgents.Forge.RollingNodeProbe
7 
  alias OpenAgents.Forge.RollingProvider.Gcp
8 
  alias OpenAgents.Test.RollingGcpDriver
9
10 
  @sha String.duplicate("a", 40)
11 
  @previous_sha String.duplicate("b", 40)
12 
  @digest "sha256:" <> String.duplicate("c", 64)
13 
  @previous_digest "sha256:" <> String.duplicate("d", 64)
14 
  @nodes [:"openagents@fleet-1", :"openagents@fleet-2", :"openagents@fleet-3"]
15
16 
  setup do
17 
    previous = Application.get_env(:openagents, Gcp)
18 
    on_exit(fn -> restore_config(previous) end)
19 
    :ok
20 
  end
21
22 
  test "fences readiness, checks quorum, and replaces the exact mapped instance" do
23 
    owner = self()
24
25 
    rpc = fn node, module, function, arguments, _timeout ->
26 
      send(owner, {:rpc, node, module, function, arguments})
27
28 
      case {module, function} do
29 
        {Admission, :remove} ->
30 
          :ok
31
32 
        {Admission, :restore} ->
33 
          :ok
34
35 
        {Drain, :drain} ->
36 
          {:ok, 0}
37
38 
        {RollingNodeProbe, :status} ->
39 
          probe(node)
40
41 
        {RollingGcpDriver, :replace} ->
42 
          [instance, sha, digest, _config] = arguments
43 
          send(owner, {:gcp_replace, instance, sha, digest})
44 
          :ok
45 
      end
46 
    end
47
48 
    put_config(rpc)
49 
    context = context()
50
51 
    assert :ok = Gcp.remove_readiness(hd(@nodes), context)
52 
    assert {:ok, 0} = Gcp.drain(hd(@nodes), context)
53 
    assert {:ok, %{ready: 2, quorum: true}} = Gcp.capacity(tl(@nodes), context)
54
55 
    assert {:ok,
56 
            %{
57 
              member: true,
58 
              ready: true,
59 
              boot_converged: true,
60 
              database_ready: true,
61 
              sha: @sha,
62 
              image_digest: @digest
63 
            }} = Gcp.status(hd(@nodes), context)
64
65 
    assert :ok = Gcp.replace(hd(@nodes), @digest, context)
66
67 
    assert_receive {:rpc, :"openagents-deployer@openagents-deployer.staging.internal",
68 
                    RollingGcpDriver, :replace,
69 
                    ["openagents-fleet-1", @sha, @digest, compute_config]}
70
71 
    assert compute_config[:project_id] == "openagents-staging-project"
72 
    refute Keyword.has_key?(compute_config, :rpc)
73 
    assert_receive {:gcp_replace, "openagents-fleet-1", @sha, @digest}
74
75 
    assert :ok = Gcp.rollback(hd(@nodes), @previous_digest, context)
76 
    assert_receive {:gcp_replace, "openagents-fleet-1", @previous_sha, @previous_digest}
77 
  end
78
79 
  test "refuses a staging project that matches production" do
80 
    put_config(fn _node, _module, _function, _arguments, _timeout -> :ok end,
81 
      project_id: "production-project",
82 
      production_project_id: "production-project"
83 
    )
84
85 
    assert {:error, :staging_project_matches_production} =
86 
             Gcp.remove_readiness(hd(@nodes), context())
87
88 
    refute_receive {:gcp_replace, _instance, _sha, _digest}
89 
  end
90
91 
  test "refuses an unrecognized deployer node" do
92 
    put_config(fn _node, _module, _function, _arguments, _timeout -> :ok end,
93 
      deployer_node: :"openagents-deployer@untrusted.internal"
94 
    )
95
96 
    assert {:error, :invalid_deployer_node} = Gcp.remove_readiness(hd(@nodes), context())
97 
  end
98
99 
  test "fails capacity closed when Ra quorum is absent" do
100 
    rpc = fn _node, RollingNodeProbe, :status, [_expected], _timeout ->
101 
      Map.put(probe(hd(@nodes)), :ra_quorum, false)
102 
    end
103
104 
    put_config(rpc)
105 
    assert {:ok, %{ready: 2, quorum: false}} = Gcp.capacity(tl(@nodes), context())
106 
  end
107
108 
  defp context do
109 
    %{
110 
      sha: @sha,
111 
      previous_sha: @previous_sha,
112 
      image_digest: @digest,
113 
      previous_image_digest: @previous_digest,
114 
      expected_nodes: @nodes
115 
    }
116 
  end
117
118 
  defp probe(node) do
119 
    %{
120 
      member: true,
121 
      ready: true,
122 
      boot_converged: true,
123 
      database_ready: true,
124 
      sha: @sha,
125 
      image_digest: @digest,
126 
      ra_quorum: true,
127 
      node: node
128 
    }
129 
  end
130
131 
  defp put_config(rpc, overrides \\ []) do
132 
    instances =
133 
      @nodes
134 
      |> Enum.with_index(1)
135 
      |> Map.new(fn {node, index} -> {to_string(node), "openagents-fleet-#{index}"} end)
136
137 
    config = [
138 
      project_id: "openagents-staging-project",
139 
      production_project_id: "production-project",
140 
      zone: "us-central1-a",
141 
      instances: instances,
142 
      image_repository:
143 
        "us-central1-docker.pkg.dev/openagents-staging-project/openagents/openagents",
144 
      deployer_node: :"openagents-deployer@openagents-deployer.staging.internal",
145 
      driver: RollingGcpDriver,
146 
      rpc: rpc,
147 
      rpc_timeout_ms: 100,
148 
      compute_timeout_ms: 1_000
149 
    ]
150
151 
    Application.put_env(:openagents, Gcp, Keyword.merge(config, overrides))
152 
  end
153
154 
  defp restore_config(nil), do: Application.delete_env(:openagents, Gcp)
155 
  defp restore_config(config), do: Application.put_env(:openagents, Gcp, config)
156 
end
test/openagents/runtime_config_test.exs modified +66

@@ -51,6 +51,57 @@ defmodule OpenAgents.RuntimeConfigTest do

51 51 
    assert {:error, %{setting: :staging_gate}} = RuntimeConfig.validate(settings)
52 52 
  end
53 53
54 
  test "Gate 12 requires exact packaged source and image identities" do
55 
    settings = staging_settings() |> Map.put(:staging_gate, 12)
56
57 
    assert {:error, %{setting: :build_revision}} = RuntimeConfig.validate(settings)
58
59 
    settings = Map.put(settings, :build_revision, String.duplicate("a", 40))
60 
    assert {:error, %{setting: :image_digest}} = RuntimeConfig.validate(settings)
61
62 
    assert {:ok, _config} =
63 
             settings
64 
             |> Map.put(:image_digest, "sha256:" <> String.duplicate("b", 64))
65 
             |> RuntimeConfig.validate()
66 
  end
67
68 
  test "fleet deployment requires the isolated GCP rolling provider" do
69 
    settings =
70 
      staging_settings()
71 
      |> Map.merge(%{
72 
        staging_gate: 13,
73 
        build_revision: String.duplicate("a", 40),
74 
        image_digest: "sha256:" <> String.duplicate("b", 64),
75 
        forge_enabled: true,
76 
        forge_deploy_lane_enabled: true,
77 
        forge_boot_converge_enabled: true,
78 
        forge_expected_fleet_size: 3,
79 
        forge_operator_token: "staging-operator-token",
80 
        forge_rolling_provider: OpenAgents.Forge.RollingProvider.Gcp,
81 
        forge_wal_dir: "/var/lib/openagents/forge-wal",
82 
        ra_enabled: true,
83 
        dns_cluster_query: "openagents-fleet.staging.internal",
84 
        distribution: [
85 
          enabled: true,
86 
          node_configured: true,
87 
          cookie_configured: true,
88 
          port_min: 9_100,
89 
          port_max: 9_115
90 
        ]
91 
      })
92 
      |> Map.put(OpenAgents.Forge.RollingProvider.Gcp, rolling_gcp_config())
93
94 
    assert {:ok, _config} = RuntimeConfig.validate(settings)
95
96 
    assert {:error, %{setting: :forge_rolling_provider}} =
97 
             settings
98 
             |> Map.put(
99 
               OpenAgents.Forge.RollingProvider.Gcp,
100 
               Keyword.put(rolling_gcp_config(), :project_id, "production-project")
101 
             )
102 
             |> RuntimeConfig.validate()
103 
  end
104
54 105 
  test "enabled OpenAI features require the centralized provider secret" do
55 106 
    settings =
56 107 
      staging_settings()

@@ -226,6 +277,21 @@ defmodule OpenAgents.RuntimeConfigTest do

226 277 
    |> put_nested(:shadow_programs, :enabled, false)
227 278 
  end
228 279
280 
  defp rolling_gcp_config do
281 
    [
282 
      project_id: "staging-project",
283 
      production_project_id: "production-project",
284 
      zone: "us-central1-a",
285 
      instances: %{
286 
        "openagents@fleet-1.staging.internal" => "openagents-fleet-1",
287 
        "openagents@fleet-2.staging.internal" => "openagents-fleet-2",
288 
        "openagents@fleet-3.staging.internal" => "openagents-fleet-3"
289 
      },
290 
      image_repository: "us-central1-docker.pkg.dev/staging-project/openagents/app",
291 
      deployer_node: :"openagents-deployer@openagents-deployer.staging.internal"
292 
    ]
293 
  end
294
229 295 
  defp update_oauth(settings, key, value) do
230 296 
    Map.update!(settings, :github_oauth, &Keyword.put(&1, key, value))
231 297 
  end
test/openagents_web/controllers/health_controller_test.exs modified +17 -1

@@ -35,7 +35,23 @@ defmodule OpenAgentsWeb.HealthControllerTest do

35 35 
             "status" => "unavailable",
36 36 
             "reason" => "runtime_not_ready",
37 37 
             "boot_converged" => false,
38 
             "deployment_ready" => true
38 
             "deployment_ready" => true,
39 
             "admission_ready" => true
40 
           }
41 
  end
42
43 
  test "refuses readiness while a rolling provider drains the node", %{conn: conn} do
44 
    on_exit(&OpenAgents.Cluster.Admission.restore/0)
45 
    :ok = OpenAgents.Cluster.Admission.remove()
46
47 
    conn = get(conn, ~p"/healthz")
48
49 
    assert json_response(conn, 503) == %{
50 
             "status" => "unavailable",
51 
             "reason" => "runtime_not_ready",
52 
             "boot_converged" => true,
53 
             "deployment_ready" => true,
54 
             "admission_ready" => false
39 55 
           }
40 56 
  end
41 57 
end
test/support/openagents/test/rolling_gcp_driver.ex added +8

@@ -0,0 +1,8 @@

1 
defmodule OpenAgents.Test.RollingGcpDriver do
2 
  @moduledoc false
3
4 
  def replace(instance, sha, digest, config) do
5 
    send(Keyword.fetch!(config, :test_pid), {:gcp_replace, instance, sha, digest})
6 
    Keyword.get(config, :driver_result, :ok)
7 
  end
8 
end

This page updates live while a promote is in flight · changelog