infra/staging/templates/fleet-startup.sh.tftpl

main at 58e6347eeb72 · 9 KB

#!/bin/bash
set -euo pipefail

exec >>/var/log/openagents-startup.log 2>&1

metadata() {
  curl --fail --silent --show-error \
    --header 'Metadata-Flavor: Google' \
    "http://metadata.google.internal/computeMetadata/v1/$1"
}

metadata_attribute() {
  metadata "instance/attributes/$1" 2>/dev/null || true
}

state_device=/dev/disk/by-id/google-openagents-state
state_root=/var/lib/openagents

if ! blkid "$state_device" >/dev/null 2>&1; then
  mkfs.ext4 -F "$state_device"
fi

mkdir -p "$state_root"
mountpoint -q "$state_root" || mount "$state_device" "$state_root"
mkdir -p \
  "$state_root/artifacts" \
  "$state_root/coding-jobs" \
  "$state_root/forge" \
  "$state_root/forge-wal" \
  "$state_root/ra" \
  "$state_root/workspace/build" \
  "$state_root/workspace/build-queue"
chown -R 65534:65534 "$state_root"

image=$(metadata_attribute openagents-image)
image_digest=$(metadata_attribute openagents-image-digest)
builder_image=$(metadata_attribute openagents-builder-image)
builder_digest=$(metadata_attribute openagents-builder-digest)
source_sha=$(metadata_attribute openagents-sha)
runtime_secret=$(metadata_attribute openagents-runtime-secret)
builder_secret=$(metadata_attribute openagents-builder-secret)
instance_ip=$(metadata instance/network-interfaces/0/ip)

# Container-Optimized OS denies unsolicited host traffic by default. Match the
# VPC admission rule locally so only the staging subnet can reach Phoenix,
# EPMD, and the fixed Erlang distribution range.
iptables -C INPUT -s "${network_cidr}" -p tcp -m multiport \
  --dports 4000,4369,9100:9115 -j ACCEPT 2>/dev/null || \
  iptables -I INPUT -s "${network_cidr}" -p tcp -m multiport \
    --dports 4000,4369,9100:9115 -j ACCEPT

if [ -z "$image" ] || [ -z "$image_digest" ] || [ -z "$source_sha" ]; then
  echo "No staging candidate is assigned; leaving the node fenced."
  exit 0
fi

application_image_prefix="${region}-docker.pkg.dev/${project_id}/openagents-staging/openagents@"
builder_image_prefix="${region}-docker.pkg.dev/${project_id}/openagents-staging/openagents-builder@"

case "$image" in
  "$application_image_prefix"sha256:????????????????????????????????????????????????????????????????) ;;
  *) echo "Assigned application image is not digest-addressed" >&2; exit 1 ;;
esac

case "$source_sha" in
  ????????????????????????????????????????) ;;
  *) echo "Assigned source revision is malformed" >&2; exit 1 ;;
esac

case "$source_sha" in
  *[!0-9a-f]*) echo "Assigned source revision is malformed" >&2; exit 1 ;;
esac

case "$image_digest" in
  sha256:????????????????????????????????????????????????????????????????) ;;
  *) echo "Assigned application digest is malformed" >&2; exit 1 ;;
esac

image_digest_hex=$${image_digest#sha256:}
case "$image_digest_hex" in
  *[!0-9a-f]*) echo "Assigned application digest is malformed" >&2; exit 1 ;;
esac

case "$image" in
  *"@$image_digest") ;;
  *) echo "Assigned application image and digest disagree" >&2; exit 1 ;;
esac

access_token=$(metadata instance/service-accounts/default/token | jq -r '.access_token')

fetch_secret() {
  secret_name=$1
  destination=$2

  curl --fail --silent --show-error \
    --header "Authorization: Bearer $access_token" \
    "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$secret_name/versions/latest:access" \
    | jq -r '.payload.data' \
    | tr '_-' '/+' \
    | base64 -d >"$destination"

  chmod 0600 "$destination"
}

append_secret() {
  environment_name=$1
  secret_name=$2
  destination=$3
  value=$(
    curl --fail --silent --show-error \
      --header "Authorization: Bearer $access_token" \
      "https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets/$secret_name/versions/latest:access" \
      | jq -r '.payload.data' \
      | tr '_-' '/+' \
      | base64 -d
  )

  if [ -z "$value" ] || [ "$(printf %s "$value" | wc -l)" -ne 0 ]; then
    echo "Runtime secrets must be nonempty single-line values" >&2
    exit 1
  fi

  printf '%s=%s\n' "$environment_name" "$value" >>"$destination"
}

mkdir -p /run/openagents
fetch_secret "$runtime_secret" /run/openagents/runtime.env

if grep -Eq '^(DATABASE_URL|SECRET_KEY_BASE|GITHUB_CLIENT_SECRET|GITHUB_TOKEN_ENCRYPTION_KEY|GITHUB_TOKEN_DECRYPTION_KEYS_JSON|OPENAI_API_KEY|VOICE_RECORDING_ENCRYPTION_KEY|CONTENT_ENCRYPTION_KEY|OPENAGENTS_FORGE_OPERATOR_TOKEN|RELEASE_COOKIE)=' /run/openagents/runtime.env; then
  echo "Fleet configuration must not duplicate named secrets" >&2
  exit 1
fi

append_secret DATABASE_URL openagents-staging-fleet-database-url /run/openagents/runtime.env
append_secret SECRET_KEY_BASE openagents-staging-secret-key-base /run/openagents/runtime.env
append_secret GITHUB_CLIENT_SECRET openagents-staging-github-client-secret /run/openagents/runtime.env
append_secret GITHUB_TOKEN_ENCRYPTION_KEY openagents-staging-github-vault-active /run/openagents/runtime.env
append_secret GITHUB_TOKEN_DECRYPTION_KEYS_JSON openagents-staging-github-vault-previous /run/openagents/runtime.env
append_secret OPENAI_API_KEY openagents-staging-openai-api-key /run/openagents/runtime.env
append_secret VOICE_RECORDING_ENCRYPTION_KEY openagents-staging-voice-recording-key /run/openagents/runtime.env
append_secret CONTENT_ENCRYPTION_KEY openagents-staging-content-vault-active /run/openagents/runtime.env
append_secret OPENAGENTS_FORGE_OPERATOR_TOKEN openagents-staging-forge-operator-token /run/openagents/runtime.env
append_secret RELEASE_COOKIE openagents-staging-release-cookie /run/openagents/runtime.env

cat >>/run/openagents/runtime.env <<EOF
DNS_CLUSTER_QUERY=openagents-fleet.staging.internal
OPENAGENTS_IMAGE_DIGEST=$image_digest
OPENAGENTS_NODE_HOST=$instance_ip
RELEASE_DISTRIBUTION=name
RELEASE_NODE=openagents@$instance_ip
EOF

# Optional one-way GitHub mirrors, keyed by repository name. The value is
# deployment policy (credential-free URLs by construction -- RuntimeConfig
# refuses anything else), so it rides the plain environment when the
# operator sets it.
if [ -n "${forge_mirror_urls_json}" ]; then
  echo "OPENAGENTS_FORGE_MIRROR_URLS_JSON=${forge_mirror_urls_json}" >>/run/openagents/runtime.env
fi

export DOCKER_CONFIG=/run/openagents/docker-config
mkdir -p "$DOCKER_CONFIG"
chmod 0700 "$DOCKER_CONFIG"
docker-credential-gcr configure-docker --registries=${region}-docker.pkg.dev

# Fleet images are immutable and recoverable from Artifact Registry. Reclaim
# only images that no running container uses before pulling the next candidate.
docker rm --force sarah-builder sarah-breakglass 2>/dev/null || true
docker image prune --all --force

cloud_sql_proxy_image=gcr.io/cloud-sql-connectors/cloud-sql-proxy@sha256:825d5e4ce70d38bd0006c9eea15a6a2e2983e87b31ac6924d33e2dba56eafc9f
docker pull "$cloud_sql_proxy_image"
docker rm --force openagents-cloud-sql-proxy 2>/dev/null || true
docker run --detach \
  --name openagents-cloud-sql-proxy \
  --network host \
  --restart always \
  "$cloud_sql_proxy_image" \
  --address 127.0.0.1 \
  --health-check \
  --http-address 127.0.0.1 \
  --http-port 9090 \
  --port 5432 \
  --private-ip \
  "${project_id}:${region}:openagents-staging-postgres"

proxy_ready=false
for _attempt in $(seq 1 60); do
  if curl --fail --silent http://127.0.0.1:9090/readiness >/dev/null; then
    proxy_ready=true
    break
  fi
  sleep 1
done

if [ "$proxy_ready" != true ]; then
  echo "Cloud SQL Auth Proxy did not become ready" >&2
  exit 1
fi

docker pull "$image"
docker rm --force openagents 2>/dev/null || true
# Codex creates a nested Bubblewrap user namespace for each repository command.
# COS blocks that setup through Docker's default profiles. Remove only those
# profiles; keep the container unprivileged and add no Linux capabilities.
docker run --detach \
  --name openagents \
  --network host \
  --restart always \
  --security-opt seccomp=unconfined \
  --security-opt apparmor=unconfined \
  --env-file /run/openagents/runtime.env \
  --volume "$state_root:$state_root" \
  "$image"

if [ -n "$builder_image" ] || [ -n "$builder_digest" ]; then
  case "$builder_image" in
    "$builder_image_prefix"sha256:????????????????????????????????????????????????????????????????) ;;
    *) echo "Assigned builder image is not digest-addressed" >&2; exit 1 ;;
  esac


  case "$builder_digest" in
    sha256:????????????????????????????????????????????????????????????????) ;;
    *) echo "Assigned builder digest is malformed" >&2; exit 1 ;;
  esac

  builder_digest_hex=$${builder_digest#sha256:}
  case "$builder_digest_hex" in
    *[!0-9a-f]*) echo "Assigned builder digest is malformed" >&2; exit 1 ;;
  esac

  case "$builder_image" in
    *"@$builder_digest") ;;
    *) echo "Assigned builder image and digest disagree" >&2; exit 1 ;;
  esac

  fetch_secret "$builder_secret" /run/openagents/builder.env
  if grep -Eq '^OPENAGENTS_FORGE_OPERATOR_TOKEN=' /run/openagents/builder.env; then
    echo "Builder configuration must not duplicate its named secret" >&2
    exit 1
  fi
  append_secret OPENAGENTS_FORGE_OPERATOR_TOKEN openagents-staging-forge-operator-token /run/openagents/builder.env
  # The small COS boot disk cannot retain two builder images. The running
  # application remains available while this disposable sidecar is replaced.
  docker rm --force openagents-builder 2>/dev/null || true
  docker image prune --all --force
  docker pull "$builder_image"
  docker run --detach \
    --name openagents-builder \
    --network host \
    --restart always \
    --user 0:65534 \
    --env-file /run/openagents/builder.env \
    --volume "$state_root/workspace:$state_root/workspace" \
    --volume "$state_root/artifacts:$state_root/artifacts" \
    "$builder_image"
fi