halkyon/deploy/brouter/segments.sh

#!/bin/sh
# Keeps BRouter's segments of the operator's region current (ADR 0030, D11, D44):
# every 5°×5° segment of BRouter's project whose south-west corner lies in
# BROUTER_REGION (`west,south,east,north` in whole degrees, multiples of 5; Europe's
# `-25,35,40,70` by default), downloaded where the project has a newer one than the
# volume holds, then again every BROUTER_REFRESH_DAYS days (30); after a round where the
# listing or a segment didn't come, again in an hour, the wait doubling up to a day
# while rounds keep failing. Each is written beside
# its old self and moved over it, so BRouter never reads half a file. Segments of
# the sea, which the project doesn't have, are skipped.
set -eu

base=https://brouter.de/brouter/segments4
region=${BROUTER_REGION:--25,35,40,70}
days=${BROUTER_REFRESH_DAYS:-30}
cd /segments4

IFS=, read -r west south east north <<END
$region
END

name() {
  lon=$1 lat=$2
  if [ "$lon" -lt 0 ]; then ew="W$((-lon))"; else ew="E$lon"; fi
  if [ "$lat" -lt 0 ]; then ns="S$((-lat))"; else ns="N$lat"; fi
  echo "${ew}_${ns}.rd5"
}

refresh() {
  complete=0
  listing=$(curl -fsS "$base/") || { echo "The segments' listing didn't come." >&2; return 0; }
  fetched=0 kept=0 failed=0
  lon=$west
  while [ "$lon" -le "$east" ]; do
    lat=$south
    while [ "$lat" -le "$north" ]; do
      file=$(name "$lon" "$lat")
      if echo "$listing" | grep -q "href=\"$file\""; then
        # Only where the project's is newer than ours (If-Modified-Since): a
        # current one comes back empty.
        if [ -e "$file" ]; then set -- -z "$file"; else set --; fi
        if ! curl -fsS -R --retry 3 "$@" -o "$file.part" "$base/$file"; then
          rm -f "$file.part"
          echo "Segment $file didn't come." >&2
          failed=$((failed + 1))
        elif [ -s "$file.part" ]; then
          mv "$file.part" "$file"
          fetched=$((fetched + 1))
        else
          rm -f "$file.part"
          kept=$((kept + 1))
        fi
      fi
      lat=$((lat + 5))
    done
    lon=$((lon + 5))
  done
  echo "Segments: $fetched fetched, $kept current, $failed failed, $(du -sh . | cut -f1) in all."
  if [ "$failed" -eq 0 ]; then complete=1; fi
}

# Seconds to wait after the $1th incomplete round in a row: $2, doubling up to $3.
backoff() {
  pause=$2 n=1
  while [ "$n" -lt "$1" ] && [ "$pause" -lt "$3" ]; do
    pause=$((pause * 2)) n=$((n + 1))
  done
  if [ "$pause" -gt "$3" ]; then pause=$3; fi
  echo "$pause"
}

failures=0
while :; do
  refresh
  if [ "$complete" -eq 1 ]; then
    failures=0
    sleep $((days * 86400))
  else
    failures=$((failures + 1))
    pause=$(backoff "$failures" 3600 86400)
    echo "The round wasn't complete; trying again in $((pause / 60)) minutes." >&2
    sleep "$pause"
  fi
done