halkyon/deploy/backup.sh

#!/bin/sh
# Backs up learn.halkyon.eu off-site. Runs every hour from
# deploy/backup/halkyon-backup.timer, as deploy on the server; deploy/backups.md says
# how it was set up and how to restore.
#
# Everything people keep is in the database, their files too (Serverpod's database
# cloud storage), and in Train's and Todo's where .env switches each on (compose.train.yaml,
# Halkyon Train's ADR 0030, D46; compose.todo.yaml, Halkyon Todo's ADR 0033), but for the
# releases of published studysets, which are git
# repositories in the server's volume: a dump and a bundle of each repository cover
# the instance; the secrets in deploy/.env are kept in Proton Pass instead. restic stores the dump encrypted in a Hetzner Storage Box
# and keeps 24 hourly, 7 daily, 4 weekly and 3 monthly snapshots (decided 2026-10-03).
# Hourly, because a restore loses what came after the backup, and for someone who
# started since, that is everything: their app no longer finds its session and starts
# anew (deploy/backups.md). After 3 months, a deleted account is gone from every
# backup (the privacy page says so). The box's own automatic snapshots, which only
# Hetzner's Console can remove, keep the backups safe from a taken-over server,
# which can delete what restic wrote.
#
# Failures go to the journal only (decided 2026-10-03):
#
#   systemctl status halkyon-backup; journalctl -u halkyon-backup
#
# restic finds the repository and its password in deploy's files
# ~/.config/restic/repository and ~/.config/restic/password (both 0600); a run by hand
# can point elsewhere with RESTIC_REPOSITORY_FILE and RESTIC_PASSWORD_FILE.
set -eu

export RESTIC_REPOSITORY_FILE="${RESTIC_REPOSITORY_FILE:-$HOME/.config/restic/repository}"
export RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-$HOME/.config/restic/password}"

cd "$(dirname "$0")"

# The dump goes to a file first: Ubuntu 24.04's restic (v0.16) would store a dump
# piped into it even when pg_dump fails halfway, and the file is the newest copy at
# hand for a quick restore. Uncompressed, so restic finds what hasn't changed since
# the last hour and stores it only once; restic compresses what it stores.
mkdir -p -m 700 "$HOME/backups"
dump="$HOME/backups/latest.dump"
# The apps' servers .env switches on beside Learn's, each with its database
# (halkyon_<app>) dumped to latest-<app>.dump.
services=$(docker compose config --services)
apps=
for app in train todo meditate; do
  if printf '%s\n' "$services" | grep -qx "$app-server"; then
    apps="$apps $app"
  fi
done
(
  umask 077
  # Never during a deploy (redeploy.sh holds this lock): its migrations would wait
  # for the dump, or the dump for them.
  exec 9>"$HOME/.redeploy.lock"
  flock 9
  docker compose exec -T postgres pg_dump -U halkyon_learn -Fc -Z 0 halkyon_learn >"$dump.partial"
  for app in $apps; do
    docker compose exec -T postgres pg_dump -U halkyon_learn -Fc -Z 0 "halkyon_$app" \
      >"$HOME/backups/latest-$app.dump.partial"
  done
)
mv "$dump.partial" "$dump"
set -- "$dump"
for app in $apps; do
  mv "$HOME/backups/latest-$app.dump.partial" "$HOME/backups/latest-$app.dump"
  set -- "$@" "$HOME/backups/latest-$app.dump"
done

# Each repository as a git bundle, after the dump: a release the dump lacks is never
# numbered again (the server numbers past the highest tag), while one the bundles
# lacked would be lost. A bundle is whole whenever it is made: git reads the refs, then
# packs what they reach. A repository being created this moment has nothing to bundle
# yet, and comes next hour.
releases="$HOME/backups/repositories.tar"
(
  umask 077
  docker compose exec -T server sh -c '
    out=$(mktemp -d)
    for repository in /app/repositories/*.git; do
      [ -d "$repository" ] || continue
      git -C "$repository" bundle create --quiet \
        "$out/$(basename "$repository" .git).bundle" --all ||
        echo "Not bundled: $repository" >&2
    done
    tar -C "$out" -cf - .
    rm -rf "$out"
  ' >"$releases.partial"
)
mv "$releases.partial" "$releases"

restic backup --quiet "$@" "$releases"
# By host alone: restic groups by the paths backed up too, so a path added (Train's
# dump, Todo's) would leave the snapshots before it a group of their own, never forgotten, and
# a deleted account in them for good.
restic forget --quiet --group-by host --keep-hourly 24 --keep-daily 7 --keep-weekly 4 --keep-monthly 3

# Once a day, in the run at 03:00: free the space of forgotten snapshots, and read the
# repository's structure; on Sundays also a tenth of the stored data, chosen at random,
# so damage to what was stored long ago shows within weeks.
if [ "$(date +%H)" = 03 ]; then
  restic prune --quiet
  if [ "$(date +%u)" = 7 ]; then
    restic check --read-data-subset=10%
  else
    restic check
  fi
fi
restic snapshots --compact --latest 1