halkyon/deploy/photon/data.sh

#!/bin/sh
# Brings Photon's database of PHOTON_REGION (`europe` by default; a path under
# GraphHopper's downloads, `europe/germany` for Germany alone) for Halkyon Train's
# place search (ADR 0030, D50): GraphHopper's weekly build for Photon 1.x, downloaded
# where none is there or the one there is older than PHOTON_REFRESH_DAYS days (30),
# unpacked as it comes into /data/next and checked against its MD5 on the way, then
# marked ready for serve.sh, which puts it in place. Its archive is never kept: only
# the old database and the new one take room at once. Looked at daily; after a failed
# download, again a day later, the wait doubling up to a week while downloads keep
# failing.
set -eu

base=https://download1.graphhopper.com/public
region=${PHOTON_REGION:-europe}
days=${PHOTON_REFRESH_DAYS:-30}
name=$(basename "$region")
file="$base/$region/photon-db-$name-1.0-latest.tar.bz2"
cd /data

# The database in use came less than $days days ago (serve.sh moves the mark with it).
fresh() {
  [ -e fetched ] && [ -z "$(find fetched -mtime +"$days" 2>/dev/null)" ]
}

fetch() {
  failed=1
  if [ -e next/ready ] || fresh; then
    failed=0
    return 0
  fi
  rm -rf next
  mkdir -p next
  expected=$(curl -fsS "$file.md5" | cut -d' ' -f1) || expected=
  if [ -z "$expected" ]; then
    echo "The database's MD5 didn't come; trying again later." >&2
    return 0
  fi
  mkfifo next/stream
  md5sum <next/stream >next/md5 &
  sum=$!
  if curl -fsS --retry 3 "$file" | tee next/stream | pbzip2 -dc | tar -x -C next &&
    wait "$sum" && [ "$(cut -d' ' -f1 next/md5)" = "$expected" ] &&
    [ -d next/photon_data ]; then
    rm -f next/stream next/md5
    : >next/fetched
    : >next/ready
    echo "Photon's database of $region is ready, $(du -sh next/photon_data | cut -f1)."
    failed=0
  else
    wait "$sum" 2>/dev/null || true
    rm -rf next
    echo "Photon's database of $region didn't come whole; trying again later." >&2
  fi
}

# Seconds to wait after the $1th failed download in a row: $2, doubling up to $3.
backoff() {
  pause=$2 n=1
  while [ "$n" -lt "$1" ] && [ "$pause" -lt "$3" ]; do
    pause=$((pause * 2)) n=$((n + 1))
  done
  if [ "$pause" -gt "$3" ]; then pause=$3; fi
  echo "$pause"
}

failures=0
while :; do
  fetch
  if [ "$failed" -eq 0 ]; then
    failures=0
    sleep 86400
  else
    failures=$((failures + 1))
    pause=$(backoff "$failures" 86400 604800)
    echo "Trying again in $((pause / 3600)) hours." >&2
    sleep "$pause"
  fi
done