#!/bin/sh
# Roll one revision onto the production fleet, one node at a time.
#
# Promotion needs an operator, not a browser: there is no button on
# /admin/forge that does it, and telling someone to look for one wastes their
# time. This drives the same operator API a console would, through `rpc` on a
# node, so the whole release runs from a terminal.
#
# It refuses rather than guesses. A revision with no passing gate receipt, an
# image whose embedded revision is not the Git SHA, or a node that comes back
# on the wrong revision each stop the release where it stands, because a
# half-rolled fleet that reports success is worse than one that stops.
#
# Usage: ops/deploy/release-to-production.sh <git-sha>

set -eu

sha=${1:-}
[ -n "$sha" ] || { echo "usage: $0 <git-sha>" >&2; exit 2; }

project=openagentsgemini
registry=us-central1-docker.pkg.dev/openagents-staging-20260820/openagents-staging/openagents

# One release at a time. Two runs rolling the same three nodes can take down
# more than one at once, which is an outage rather than a rolling replacement.
# The lock is a directory because mkdir is atomic on every filesystem this
# runs on, and it carries the pid and sha so a stale one names its owner.
lock_dir=${TMPDIR:-/tmp}/openagents-release.lock
if ! mkdir "$lock_dir" 2>/dev/null; then
  echo "a release is already running: $(cat "$lock_dir/owner" 2>/dev/null || echo 'unknown')" >&2
  echo "if that is stale, remove $lock_dir and run again" >&2
  exit 1
fi
echo "pid=$$ sha=${1:-} started=$(date -u +%FT%TZ)" > "$lock_dir/owner"
trap 'rm -rf "$lock_dir"' EXIT INT TERM
# The automation service account: the interactive account hits Workspace
# reauthentication and cannot refresh in a headless run.
CLOUDSDK_CONFIG=${CLOUDSDK_CONFIG:-/Users/christopherdavid/work/.secrets/gcloud-sa-config}
export CLOUDSDK_CONFIG

# Zone is part of a node's identity here, so the three are named together.
nodes='sarah-fleet-1:us-central1-a sarah-fleet-2:us-central1-b sarah-fleet-3:us-central1-c'

on_node() {
  instance=$1
  zone=$2
  shift 2
  gcloud compute ssh "$instance" --zone="$zone" --project="$project" \
    --tunnel-through-iap --command="$*"
}

# Inline `rpc` quoting does not survive the trip through ssh and docker, so
# every remote evaluation goes as a file.
rpc_file() {
  instance=$1
  zone=$2
  script=$3
  gcloud compute scp "$script" "$instance:/tmp/rpc.exs" --zone="$zone" \
    --project="$project" >/dev/null
  on_node "$instance" "$zone" \
    "chmod 644 /tmp/rpc.exs && docker cp /tmp/rpc.exs openagents:/tmp/rpc.exs && docker exec openagents /app/bin/openagents rpc 'Code.eval_file(\"/tmp/rpc.exs\")'"
}

health_of() {
  on_node "$1" "$2" 'curl -s http://127.0.0.1:8080/health' 2>/dev/null |
    grep -o '"revision":"[0-9a-f]*"' | head -1 | cut -d'"' -f4
}

echo "==> release $sha"

# 1. The gate receipt. Read it rather than re-running the gate, so a release
#    cannot be talked into trusting a run nobody kept.
receipt="$(git rev-parse --path-format=absolute --git-common-dir)/openagents/release-gate-receipts/$sha.json"
[ -f "$receipt" ] || { echo "no gate receipt for $sha; run ops/ci/gate.sh first" >&2; exit 1; }
jq -e '.status == "passed"' "$receipt" >/dev/null ||
  { echo "gate receipt for $sha is not passed" >&2; exit 1; }
echo "gate receipt: passed"

# 2. The image, identified by digest from here on. A tag can be moved; a
#    digest names the bytes the fleet will actually run.
digest=$(gcloud artifacts docker images describe "$registry:$sha" \
  --format='value(image_summary.digest)' --project="$project" 2>/dev/null || true)
[ -n "$digest" ] || { echo "no image for $sha; build it first" >&2; exit 1; }
echo "image digest: $digest"

# Nothing to do is a valid outcome, and it has to be checked before anything
# is promoted. A release already live on this sha used to fall through to a
# fresh promotion, which builds a new target and rolls a fleet that is already
# serving exactly what was asked for.
live_now=$(mktemp)
cat > "$live_now" <<ELIXIR
live = OpenAgents.Forge.Targets.live("openagents.com")
IO.puts(if live && live.sha == "$sha", do: "already-live", else: "roll-needed")
ELIXIR
if rpc_file sarah-fleet-1 us-central1-a "$live_now" 2>/dev/null | grep -q "already-live"; then
  rm -f "$live_now"
  echo "$sha is already live on this fleet; nothing to do"
  exit 0
fi
rm -f "$live_now"

# What the fleet is replacing, resolved before the promotion so the authority
# can be checked against something true. The live target's details do not
# reliably carry an image digest — a target settled after its builder wrote
# build details comes back with none — and the old fallback used this
# release's own digest, which is the one value the authority check refuses:
# a rolling replacement whose previous image equals its next image is not a
# replacement. The registry knows what the live sha was built as, so ask it.
echo "==> previous release"
live_sha=$(mktemp)
cat > "$live_sha" <<'ELIXIR'
live = OpenAgents.Forge.Targets.live("openagents.com")
IO.puts("live-sha=" <> ((live && live.sha) || "none"))
ELIXIR
previous_sha=$(rpc_file sarah-fleet-1 us-central1-a "$live_sha" 2>/dev/null |
  grep -o 'live-sha=[0-9a-f]*' | head -1 | cut -d= -f2)
rm -f "$live_sha"

[ -n "$previous_sha" ] && [ "$previous_sha" != "none" ] ||
  { echo "no live release to replace; refusing to guess a previous image" >&2; exit 1; }

previous_digest=$(gcloud artifacts docker images describe "$registry:$previous_sha" \
  --format='value(image_summary.digest)' --project="$project" 2>/dev/null || true)

[ -n "$previous_digest" ] ||
  { echo "no image in the registry for the live release $previous_sha" >&2; exit 1; }

[ "$previous_digest" != "$digest" ] ||
  { echo "the live release already runs this image; nothing to replace" >&2; exit 1; }

echo "previous: $previous_sha ($previous_digest)"

echo "==> promote"
promote=$(mktemp)
cat > "$promote" <<ELIXIR
previous_sha = "$previous_sha"
previous_image_digest = "$previous_digest"
expected_nodes = Enum.sort(["openagents@10.128.0.4", "openagents@10.128.0.110", "openagents@10.128.0.111"])

# The forge builds a promoted target itself: OpenAgents.Forge.Builder
# subscribes to the promotion broadcast, builds the artifact, and writes the
# complete build receipt that finish_rolling_replacement/2 requires. This used
# to advance promoted -> building -> built -> needs_rolling_replace by hand,
# which raced the builder past its own window: the statuses said a build had
# happened while no receipt existed, every node rolled, and settle then refused
# with complete_build_receipt_not_found on a fleet already serving the new
# image. Promote, then wait for the builder to say what it found.
#
# No backticks anywhere in this heredoc: it is unquoted so the shell expands it,
# and a backticked word in a comment runs as a command.
import Ecto.Query

built? = fn target ->
  OpenAgents.Repo.exists?(
    from b in OpenAgents.Forge.BuildReceipt,
      where: b.target_id == ^target.id and b.status == "complete"
  )
end

await_built = fn await_built, remaining ->
  t = OpenAgents.Forge.Targets.current("openagents.com")

  cond do
    t == nil or t.sha != "$sha" ->
      raise "current target is #{inspect(t && t.sha)}, not $sha"

    t.status == "failed" ->
      raise "target #{t.id} failed to build"

    t.status in ["needs_rolling_replace", "built"] and built?.(t) ->
      t

    remaining > 0 ->
      Process.sleep(5_000)
      await_built.(await_built, remaining - 1)

    true ->
      raise "target #{t.id} is #{t.status} with no complete build receipt"
  end
end

promote_fresh = fn ->
  {:ok, _} =
    OpenAgents.Forge.Targets.promote("openagents.com", "$sha", "operator:14167547",
      details: %{"source" => "operator_console"}
    )

  await_built.(await_built, 120)
end

# A target only counts as reusable when the receipt settlement needs is really
# there. One parked at needs_rolling_replace with no receipt is what the old
# hand-advance left behind, and reusing it walks into the same refusal, so it
# is promoted again rather than waited on.
target =
  case OpenAgents.Forge.Targets.current("openagents.com") do
    %{sha: "$sha", status: "needs_rolling_replace"} = t ->
      if built?.(t), do: t, else: promote_fresh.()

    %{sha: "$sha", status: status} when status in ["promoted", "building", "built"] ->
      await_built.(await_built, 120)

    _ ->
      promote_fresh.()
  end

{:ok, authorized} =
  OpenAgents.Forge.Targets.authorize_rolling_replacement(target.id, %{
    "sha" => "$sha",
    "image_digest" => "$digest",
    "previous_sha" => previous_sha,
    "previous_image_digest" => previous_image_digest,
    "expected_nodes" => expected_nodes,
    "authorized_by" => "operator:14167547"
  })

IO.puts("target=#{authorized.id} status=#{authorized.status}")
ELIXIR
rpc_file sarah-fleet-1 us-central1-a "$promote"
rm -f "$promote"

echo "==> roll"

# The startup script pins the image by digest, so it is generated per release
# rather than edited by hand. A stale digest here is how a "rolled" node comes
# back running the previous release while every check reports success.
startup=$(mktemp)
sed "s|__IMAGE_DIGEST__|$digest|g" "$(dirname "$0")/fleet-startup.template.sh" > "$startup"
grep -q '__IMAGE_DIGEST__' "$startup" && { echo "startup template not fully filled" >&2; exit 1; }

# Restarting a node that already serves this release is not a no-op: the
# startup runner stops the container and brings it back, so the node answers
# 502 for the length of that restart. Re-running a release that had already
# rolled is what took production down for a window, and the roll looked
# harmless in the log because the metadata write reported "No change
# requested" while the restart happened anyway.
#
# So a node is rolled only when it actually needs rolling: either its metadata
# does not yet pin this digest, or it is not serving this revision. Both have
# to be already true to skip it, because a node can serve the right revision
# from a hand restart while its metadata still pins the old digest, and that
# node would revert on its next reboot.
rolled=0
skipped=0

for entry in $nodes; do
  instance=${entry%%:*}
  zone=${entry##*:}
  echo "--> $instance ($zone)"

  pinned=$(gcloud compute instances describe "$instance" --zone="$zone" \
    --project="$project" \
    --format="value(metadata.items.filter(\"key:startup-script\").extract(value))" 2>/dev/null |
    grep -c "$digest" || true)
  serving=$(health_of "$instance" "$zone")

  if [ "$pinned" != "0" ] && [ "$serving" = "$sha" ]; then
    echo "    already on $sha with this digest pinned; not restarting"
    skipped=$((skipped + 1))
    continue
  fi

  # Never take a node down while another is already down. The roll is only
  # safe because the rest of the fleet is serving; without this a second node
  # can go before the first is back, and two of three down is an outage.
  for other in $nodes; do
    other_instance=${other%%:*}
    other_zone=${other##*:}
    [ "$other_instance" = "$instance" ] && continue
    if [ -z "$(health_of "$other_instance" "$other_zone")" ]; then
      echo "$other_instance is not answering; refusing to restart $instance too" >&2
      exit 1
    fi
  done

  rolled=$((rolled + 1))
  gcloud compute instances add-metadata "$instance" --zone="$zone" \
    --project="$project" --metadata-from-file=startup-script="$startup" >/dev/null
  on_node "$instance" "$zone" 'sudo google_metadata_script_runner startup >/tmp/roll.log 2>&1 &'

  # Wait for the node to come back on the revision it was asked for. A node
  # that answers on the OLD revision is not "still starting" — it is a node
  # that did not take the release, and continuing past it would leave the
  # fleet split while reporting success.
  ok=
  i=0
  while [ "$i" -lt 30 ]; do
    sleep 20
    if [ "$(health_of "$instance" "$zone")" = "$sha" ]; then ok=1; break; fi
    i=$((i + 1))
  done
  [ -n "$ok" ] || { echo "$instance did not reach $sha; stopping with the fleet split" >&2; exit 1; }
  echo "    healthy on $sha"
done
rm -f "$startup"

echo "rolled $rolled node(s), skipped $skipped already on $sha"

echo "==> settle"
settle=$(mktemp)
cat > "$settle" <<ELIXIR
target = OpenAgents.Forge.Targets.current("openagents.com")
sha = "$sha"
image_digest = "$digest"
authority = OpenAgents.Forge.Targets.rolling_authority("openagents.com")
expected = authority["expected_nodes"]

# Ask each node what it is running. Recording an assumed identity would make
# the log say something nobody checked, which is the one thing settlement
# exists to prevent.
observed =
  Map.new(expected, fn name ->
    {name, :rpc.call(String.to_atom(name), OpenAgents.BuildInfo, :revision, [])}
  end)

wrong = for {name, revision} <- observed, revision != sha, do: {name, revision}

if wrong != [] do
  IO.inspect(wrong, label: :refused_nodes_not_on_this_revision)
else
  for {name, _revision} <- observed do
    {:ok, _} =
      OpenAgents.Forge.Targets.record_rolling_node(target.id, name, %{
        sha: sha,
        image_digest: image_digest
      })
  end

  result = %{
    schema: "openagents.rolling-replacement.v1",
    sha: sha,
    previous_sha: authority["previous_sha"],
    image_digest: image_digest,
    previous_image_digest: authority["previous_image_digest"],
    expected_nodes: expected,
    status: "live",
    node_results: Map.new(expected, &{&1, "ready"})
  }

  IO.inspect(OpenAgents.Forge.Targets.finish_rolling_replacement(target.id, result) |> elem(0),
    label: :finish
  )

  settled = OpenAgents.Forge.Targets.current("openagents.com")
  IO.puts("status=#{settled.status} sha=#{String.slice(settled.sha || "", 0, 12)}")
end
ELIXIR
rpc_file sarah-fleet-1 us-central1-a "$settle"
rm -f "$settle"

curl -s -o /dev/null -w 'openagents.com: %{http_code}\n' https://openagents.com/