halkyon/deploy/redeploy.sh

#!/bin/sh
# Deploys learn.halkyon.eu from main. The only command CI's deploy key may run: deploy's
# ~/.ssh/authorized_keys on sahaquiel pins that key to this script, from sahaquiel itself:
#
#   restrict,from="127.0.0.1,::1",command="/home/deploy/halkyon-learn/deploy/redeploy.sh" ssh-ed25519 … (CI deploy trigger)
#
# CI passes the commit it has checked. This deploys it only
#   - while it is still the tip of main (otherwise the CI run of the newer tip deploys),
#   - as a fast-forward of the deployed commit,
#   - if every new commit is signed by a key in deploy/allowed-signers, as the deployed
#     checkout lists them (so no commit can add its own key),
# dumps the database first (the last 10 dumps stay in ~/backups), applies the migrations
# before switching to the new server, and fails unless that server answers afterwards. A failure leaves the checkout at the deployed commit, so
# running the CI job again retries. By hand, with your own key:
#
#   ssh sahaquiel-deploy halkyon-learn/deploy/redeploy.sh <commit>
set -eu

main() {
  commit=${SSH_ORIGINAL_COMMAND:-${1:-}}
  case $commit in
    '' | *[!0-9a-f]*) fail 'expects the full commit to deploy (40 hex digits)' ;;
  esac
  [ ${#commit} -eq 40 ] || fail 'expects the full commit to deploy (40 hex digits)'

  cd "$HOME/halkyon-learn"
  # One deploy at a time.
  exec 9>"$HOME/.redeploy.lock"
  flock 9

  git fetch -q origin main
  tip=$(git rev-parse origin/main)
  if [ "$tip" != "$commit" ]; then
    echo "main is at $tip now, not $commit: the CI run of $tip deploys that."
    return 0
  fi
  deployed=$(git rev-parse HEAD)
  if [ "$deployed" = "$commit" ]; then
    echo "$commit is deployed already."
    return 0
  fi
  git merge-base --is-ancestor "$deployed" "$commit" ||
    fail "$commit is no fast-forward of the deployed $deployed"

  unsigned=$(git log --format='%H %G? %GP' "$deployed..$commit" |
    while read -r each status key; do
      case $status in
        # G: good signature; U: good, by a key deploy's keyring doesn't vouch for
        # (the list below does the vouching).
        G | U) [ -n "$key" ] && grep -qx "$key" deploy/allowed-signers && continue ;;
      esac
      echo "$each"
    done)
  [ -z "$unsigned" ] || fail "not signed by a key in deploy/allowed-signers: $unsigned"

  echo "Deploying $commit (was $deployed)."
  git merge -q --ff-only "$commit"
  trap 'git reset -q --hard "$deployed"; echo "The checkout is back at $deployed." >&2' EXIT
  cd deploy
  # Build while the running server keeps serving; then dump, migrate, switch.
  docker compose build -q server
  dump
  # Migrations as a run of their own, before the switch: Serverpod's maintenance role
  # applies them and exits with 0 only if that worked. Each migration is one
  # transaction, so a failing one stops the deploy here and the running server keeps
  # serving the schema it knows. (Until the switch, the old server serves the new
  # schema; migrations that only add, as they should, keep that harmless.)
  docker compose run --rm --no-deps -e role=maintenance server --apply-migrations
  docker compose up -d --remove-orphans
  wait_until_answering
  trap - EXIT
  docker image prune -f >/dev/null
  echo "Deployed $commit."
}

# Dumps the database, as it is before the new server applies its migrations.
dump() {
  mkdir -p "$HOME/backups"
  file="$HOME/backups/halkyon_learn-$(git rev-parse --short "$deployed")-$(date -u +%Y%m%dT%H%M%SZ).dump"
  (
    umask 077
    docker compose exec -T postgres pg_dump -U halkyon_learn -Fc halkyon_learn >"$file"
  )
  echo "Database dumped to $file ($(du -h "$file" | cut -f1))."
  # Only the newest 10. (ls is safe here: the names are ours, letters, digits and dashes.)
  # shellcheck disable=SC2012
  ls -1t "$HOME"/backups/halkyon_learn-*.dump | tail -n +11 | xargs -r rm --
}

# Waits up to a minute for the new server to answer inside the stack's network.
wait_until_answering() {
  tries=0
  while [ $tries -lt 30 ]; do
    if docker compose exec -T caddy wget -q -O /dev/null http://server:8080/ 2>/dev/null; then
      echo 'The server answers.'
      return 0
    fi
    tries=$((tries + 1))
    sleep 2
  done
  docker compose logs --tail 40 server >&2
  fail "the server doesn't answer a minute after the switch"
}

fail() {
  echo "Error: $*" >&2
  exit 1
}

# All of the above is read before anything runs, so the script can update its own file.
main "$@"
exit