halkyon/deploy/redeploy.sh

#!/bin/sh
# Deploys the instance (halkyon.li, ADR 0039, D13; learn.halkyon.eu until November 2026)
# from main, by hand, once the commit's night's CI run passed. The only command deploy's
# keys may run: deploy's ~/.ssh/authorized_keys on the instance's host pins the
# developer's key to this script:
#
#   restrict,command="/home/deploy/halkyon/deploy/redeploy.sh" ssh-ed25519 … (deploys by hand)
#
# The developer passes the commit. This deploys it only
#   - while it is still the tip of main (otherwise the newer tip is deployed instead),
#   - as a fast-forward of the deployed commit,
#   - if every new commit is signed by a key in deploy/allowed-signers, as the deployed
#     checkout lists them (so no commit can add its own key),
# dumps the database first (the last 3 dumps stay in ~/backups; Train's, Todo's and Meditate's
# too where .env switches each on, compose.train.yaml, compose.todo.yaml,
# compose.meditate.yaml), applies the migrations
# before switching to the new server, starts it once more after a migration (see below),
# and fails unless that server answers afterwards. A failure leaves the checkout at the
# deployed commit, so running it again retries. From melchior, at home or through ramiel:
#
#   ssh balthasar-deploy <commit>
set -eu

main() {
  commit=${SSH_ORIGINAL_COMMAND:-${1:-}}
  case $commit in
    '' | *[!0-9a-f]*) fail 'expects the full commit to deploy (40 hex digits)' ;;
  esac
  [ ${#commit} -eq 40 ] || fail 'expects the full commit to deploy (40 hex digits)'

  cd "$HOME/halkyon"
  # One deploy at a time.
  exec 9>"$HOME/.redeploy.lock"
  flock 9

  git fetch -q origin main
  tip=$(git rev-parse origin/main)
  if [ "$tip" != "$commit" ]; then
    echo "main is at $tip now, not $commit: the CI run of $tip deploys that."
    return 0
  fi
  deployed=$(git rev-parse HEAD)
  if [ "$deployed" = "$commit" ]; then
    echo "$commit is deployed already."
    return 0
  fi
  git merge-base --is-ancestor "$deployed" "$commit" ||
    fail "$commit is no fast-forward of the deployed $deployed"

  unsigned=$(git log --format='%H %G? %GP' "$deployed..$commit" |
    while read -r each status key; do
      case $status in
        # G: good signature; U: good, by a key deploy's keyring doesn't vouch for
        # (the list below does the vouching).
        G | U) [ -n "$key" ] && grep -qx "$key" deploy/allowed-signers && continue ;;
      esac
      echo "$each"
    done)
  [ -z "$unsigned" ] || fail "not signed by a key in deploy/allowed-signers: $unsigned"

  echo "Deploying $commit (was $deployed)."
  git merge -q --ff-only "$commit"
  trap back_out EXIT
  cd deploy
  # The apps' servers beside Learn's, where .env switches each on (compose.train.yaml,
  # compose.todo.yaml, compose.meditate.yaml): built, dumped, migrated and waited for as
  # Learn's.
  services=$(docker compose config --services)
  apps=
  for app in train todo meditate; do
    if printf '%s\n' "$services" | grep -qx "$app-server"; then
      apps="$apps $app"
    fi
  done
  # Build while the running server keeps serving; then dump, migrate, switch.
  docker compose build -q server dictionaries
  for app in $apps; do
    docker compose build -q "$app-server"
  done
  # Train's routes, with its server.
  case " $apps " in *' train '*) docker compose build -q brouter ;; esac
  # Train's own map, where .env switches it on (compose.maps.yaml).
  if docker compose config --services | grep -qx map-tiles; then
    docker compose build -q map-tiles
  fi
  # Train's place search, where .env switches it on (compose.search.yaml).
  if docker compose config --services | grep -qx photon; then
    docker compose build -q photon
  fi
  dump halkyon_learn
  for app in $apps; do
    dump "halkyon_$app"
  done
  # Migrations as a run of their own, before the switch: Serverpod's maintenance role
  # applies them and exits with 0 only if that worked. Each migration is one
  # transaction, so a failing one stops the deploy here and the running server keeps
  # serving the schema it knows. (Until the switch, the old server serves the new
  # schema; migrations that only add, as they should, keep that harmless.)
  docker compose run --rm --no-deps -e role=maintenance server --apply-migrations
  for app in $apps; do
    docker compose run --rm --no-deps -e role=maintenance "$app-server" --apply-migrations
  done
  switched=yes
  docker compose up -d --remove-orphans
  # Caddy reads its Caddyfile when it starts, and `up` doesn't restart it for a
  # changed file it mounts. Nor would `caddy reload` see the change: Docker
  # mounts a single file by its inode, and git writes a changed file anew, so
  # the container keeps the old one (found 2026-10-10). A changed Caddyfile, or a
  # sidecar's site, therefore recreates the container, which mounts the new file; the
  # certificates are in a volume, and the instance is away for seconds.
  if ! git diff --quiet "$deployed" "$commit" -- Caddyfile 'sidecars/*.caddy' forwarded.caddy; then
    docker compose up -d --force-recreate --no-deps caddy
  fi
  wait_until_answering server
  for app in $apps; do
    wait_until_answering "$app-server"
  done
  # After a migration the new server is started once more. The sync engine
  # (serverpod_offline_sync v0.0.10) reconciles its registry of synced tables as a
  # server first uses the database; when a migration changed a synced table, that
  # changes the registry, and the server then rebuilds every space's projections in
  # every session until it stops (issue #150): after v0.13.0's deploy every request
  # took a minute or two, and syncs failed, until the server was restarted
  # (2026-10-13). The maintenance run doesn't spare it. Started again, the server
  # finds the registry as the first start left it.
  if ! git diff --quiet "$deployed" "$commit" -- ../apps/learn/halkyon_learn_server/migrations; then
    docker compose restart server
    wait_until_answering server
  fi
  # The apps' servers too: they sync through the same engine.
  for app in $apps; do
    if ! git diff --quiet "$deployed" "$commit" -- "../apps/$app/halkyon_${app}_server/migrations"; then
      docker compose restart "$app-server"
      wait_until_answering "$app-server"
    fi
  done
  trap - EXIT
  docker image prune -f >/dev/null
  echo "Deployed $commit."
}

# Dumps the database $1 (halkyon_learn, halkyon_train, halkyon_todo, halkyon_meditate), as it is before the new server
# applies its migrations; as the instance's superuser, halkyon_learn.
dump() {
  mkdir -p "$HOME/backups"
  file="$HOME/backups/$1-$(git rev-parse --short "$deployed")-$(date -u +%Y%m%dT%H%M%SZ).dump"
  (
    umask 077
    docker compose exec -T postgres pg_dump -U halkyon_learn -Fc "$1" >"$file"
  )
  echo "Database dumped to $file ($(du -h "$file" | cut -f1))."
  # Only the newest 3 (decided 2026-10-02): they are the deploys' way back, not backups;
  # those go off-site. Every file people keep is in each of them, so fewer leave room.
  # (ls is safe here: the names are ours, letters, digits and dashes.)
  # shellcheck disable=SC2012
  ls -1t "$HOME"/backups/"$1"-*.dump | tail -n +4 | xargs -r rm --
}

# Waits up to a minute for the new server $1 (server, train-server, todo-server, meditate-server) to answer inside the
# stack's network.
wait_until_answering() {
  tries=0
  while [ $tries -lt 30 ]; do
    if docker compose exec -T caddy wget -q -O /dev/null "http://$1:8080/" 2>/dev/null; then
      echo "The $1 answers."
      return 0
    fi
    tries=$((tries + 1))
    sleep 2
  done
  docker compose logs --tail 40 "$1" >&2
  fail "the $1 doesn't answer a minute after the switch"
}

# A failed deploy puts the checkout back at the deployed commit, so running the CI job
# again retries. Failing after the switch, it also brings the containers back as that
# commit has them (found 2026-10-14: Caddy couldn't start on the new network, and the
# instance stayed down until they were brought back by hand), on the new images, which
# fit the database the migrations have changed. Before the switch, the running
# containers were never touched.
back_out() {
  cd "$HOME/halkyon"
  git reset -q --hard "$deployed"
  echo "The checkout is back at $deployed." >&2
  if [ "${switched:-}" = yes ]; then
    (cd deploy && docker compose up -d --remove-orphans) >&2 &&
      echo "The containers are back as $deployed has them." >&2
  fi
}

fail() {
  echo "Error: $*" >&2
  exit 1
}

# All of the above is read before anything runs, so the script can update its own file.
main "$@"
exit