vps-health-api/agent/pg-backup-pull/run-pg-backup-pull.sh

#!/bin/sh
#
# Daily pull of the encrypted PostgreSQL backups from the VPS onto this
# workstation (la-compagnie-maximus#220). Run by the systemd user timer
# maximus-pg-backup-pull.timer, unattended.
#
# The VPS (backup-postgres.sh, 03:00 UTC) age-encrypts one copy of every dump
# into /home/pgbackup/outbox/<date>/. This script fetches that outbox over the
# classic sshd on 100.86.30.21:2222, where the key can only run `rrsync -ro`.
# It never decrypts anything: the age private key is not on this machine.
#
# NEVER add `set -x`. The output goes to journald, and the shell would echo the
# `. "$ENV_FILE"` expansion — the ingest token — into it for good. Nothing below
# ever echoes a variable that holds a secret.

set -eu

# Overridable for testing only; the timer uses the defaults.
ENV_FILE="${PG_BACKUP_PULL_ENV_FILE:-$HOME/.config/maximus-pg-backup-pull.env}"
DEST="${PG_BACKUP_PULL_DEST:-$HOME/backups/postgres}"
STATE_DIR="${PG_BACKUP_PULL_STATE_DIR:-$HOME/.local/state/maximus-pg-backup-pull}"
SSH_KEY="${PG_BACKUP_PULL_SSH_KEY:-$HOME/.ssh/pgbackup_pull_ed25519}"
KNOWN_HOSTS="${PG_BACKUP_PULL_KNOWN_HOSTS:-$HOME/.ssh/known_hosts_pgbackup}"
TODAY="${PG_BACKUP_PULL_TODAY:-$(date -u +%F)}"

SOURCE="pgbackup@100.86.30.21:"
SSH_PORT=2222
RETENTION_DAYS=30
RSYNC_TIMEOUT_SECONDS=900
POST_TIMEOUT_SECONDS=20

EXIT_INSTALL=1
EXIT_PULL=2
EXIT_REPORT=3

log() { echo "pg-backup-pull: $*"; }

for bin in rsync ssh flock timeout; do
  if ! command -v "$bin" >/dev/null 2>&1; then
    log "$bin not found"
    exit "$EXIT_INSTALL"
  fi
done
for file in "$SSH_KEY" "$KNOWN_HOSTS"; do
  if [ ! -f "$file" ]; then
    log "missing $file"
    exit "$EXIT_INSTALL"
  fi
done

mkdir -p "$STATE_DIR" "$DEST"
chmod 700 "$STATE_DIR" "$DEST"

exec 9>"$STATE_DIR/run.lock"
if ! flock -n 9; then
  log "a run is already in progress, skipping"
  exit 0
fi

# The ingest token sits in this file for the length of one curl call. The trap
# removes it on any exit, a kill by systemd included: the signal traps turn
# TERM/INT/HUP into a plain exit, which runs the EXIT trap (dash does not run
# it on a signal by itself).
header_file="$STATE_DIR/auth.header"
trap 'rm -f "$header_file"' EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM

# The env file is optional until the ingest route is deployed: without a token,
# the pull still runs and only the report to the health API is skipped.
BACKUPS_API_URL=""
BACKUPS_INGEST_TOKEN=""
if [ -f "$ENV_FILE" ]; then
  FILE_MODE=$(stat -c "%a" "$ENV_FILE" 2>/dev/null || echo "")
  if [ -n "$FILE_MODE" ] && [ "$FILE_MODE" != "600" ]; then
    log "warning: $ENV_FILE is mode $FILE_MODE, expected 600"
  fi
  # shellcheck source=/dev/null
  . "$ENV_FILE"
fi

report() {
  # report <ok:true|false> <files> [error-code]
  if [ -z "$BACKUPS_INGEST_TOKEN" ] || [ -z "$BACKUPS_API_URL" ]; then
    log "no BACKUPS_INGEST_TOKEN/BACKUPS_API_URL in $ENV_FILE, report skipped"
    return 0
  fi
  if ! command -v curl >/dev/null 2>&1; then
    log "curl not found, report skipped"
    return 1
  fi
  pulled_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  if [ -n "${3:-}" ]; then
    body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s,"error":"%s"}' "$pulled_at" "$2" "$1" "$3")
  else
    body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s}' "$pulled_at" "$2" "$1")
  fi
  # The token goes through a header file, never through argv (visible in ps).
  (umask 077 && printf 'Authorization: Bearer %s\n' "$BACKUPS_INGEST_TOKEN" > "$header_file")
  status=$(printf '%s' "$body" | curl -sS -o /dev/null -w '%{http_code}' \
      --max-time "$POST_TIMEOUT_SECONDS" \
      -H @"$header_file" -H 'Content-Type: application/json' \
      --data-binary @- "$BACKUPS_API_URL/backups/pull" 2>/dev/null) || status="000"
  rm -f "$header_file"
  case "$status" in
    2??) log "report ok status=$status"; return 0 ;;
    *) log "report failed status=$status"; return 1 ;;
  esac
}

# --- Pull ---------------------------------------------------------------------
# Only directories and .age files cross: a misconfigured outbox can never land
# a clear dump here. No --delete: the VPS keeps 7 days, this machine keeps 30.
# --prune-empty-dirs: a dated directory with no .age file (encryption failed on
# the VPS) is not created here, so it cannot pass for today's backup.
# --ignore-existing: a file already pulled is never rewritten, whatever the VPS
# serves under its name later. The VPS writes each dump to a .tmp and renames
# it (the .tmp never passes the filter), so what lands here is already whole.
if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --prune-empty-dirs --ignore-existing --partial-dir=.rsync-partial \
    --include='*/' --include='*.age' --exclude='*' \
    -e "ssh -p $SSH_PORT -i $SSH_KEY -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" \
    "$SOURCE" "$DEST/"; then
  log "pull failed"
  report false 0 pull_failed || true
  exit "$EXIT_PULL"
fi
chmod -R go-rwx "$DEST"

# Nothing the VPS sends may delete or alter a copy already here: directory
# names come from the outbox, so they are checked before they steer anything.
#
# Plain shell over the glob, no `find -printf` nor `date -d "... days"`: both
# are GNU-only, and the CI runs this script under busybox (node:22-alpine).
#
# A name is dated only if `date` gives it back unchanged: GNU date refuses
# 2026-13-45, busybox turns 2026-02-30 into 2026-03-02. Anything else is never
# picked as latest, never removed, and must not stop the pass.
is_dated() {
  case "$1" in
    [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;;
    *) return 1 ;;
  esac
  [ "$(date -u -d "$1" +%F 2>/dev/null)" = "$1" ]
}
# YYYY-MM-DD as a YYYYMMDD integer: `[ a \< b ]` is not POSIX, and /bin/sh is dash.
day_number() { echo "$1" | tr -d -; }

# latest is the newest dated directory not after today: a future name (VPS
# clock off, or a VPS gone rogue) is kept but steers neither the report nor the
# rotation below.
latest=""
for dir in "$DEST"/*/; do
  [ -d "$dir" ] || continue
  name=$(basename "$dir")
  # Only names of the dated shape are logged: they hold digits and dashes, so a
  # name from the VPS cannot forge a journal line.
  case "$name" in [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) continue ;; esac
  if ! is_dated "$name"; then
    log "warning: ignoring $name, not a valid date"
    continue
  fi
  if [ "$(day_number "$name")" -gt "$(day_number "$TODAY")" ]; then
    log "warning: ignoring $name, after $TODAY"
    continue
  fi
  if [ -z "$latest" ] || [ "$(day_number "$name")" -gt "$(day_number "$latest")" ]; then
    latest="$name"
  fi
done
files=0
if [ -n "$latest" ]; then
  # -maxdepth 1: a partial file left in .rsync-partial/ is not a backup.
  files=$(find "$DEST/$latest" -maxdepth 1 -type f -name '*.age' | wc -l | tr -d ' ')
fi
log "pull ok latest=${latest:-none} files=$files"
# The report counts the newest local directory: if the VPS stops producing, a
# pass still says ok on an old one. GET /backups judges freshness on the VPS
# side; this line names the gap in the journal.
if [ "${latest:-}" != "$TODAY" ]; then
  log "warning: no directory for $TODAY, latest is ${latest:-none}"
fi
if [ -n "$latest" ] && [ "$files" -eq 0 ]; then
  log "warning: no .age file in $latest"
fi

# --- Rotation (fails loudly) --------------------------------------------------
# Anchored on latest, the newest backup held here that is not after today. Not
# on today alone: when the VPS stops producing (dead cron, failing pg_dump), or
# this machine's clock jumps ahead, a today-based cutoff would delete the last
# off-site copy. Not on the newest name alone: a future-dated directory from
# the VPS would push the window forward and delete every real backup. This
# keeps latest and the 29 days before it; a future or invalid name is newer
# than the cutoff or not dated, so it is never removed.
# Epoch arithmetic: `date -d YYYY-MM-DD` and `date -d @N` are read by GNU date
# and busybox alike.
rotation_ok=true
cutoff=""
if [ -n "$latest" ]; then
  latest_epoch=$(date -u -d "$latest" +%s)
  cutoff=$(date -u -d "@$((latest_epoch - (RETENTION_DAYS - 1) * 86400))" +%F)
fi
# No latest means no dated directory: the loop below then compares nothing.
for dir in "$DEST"/*/; do
  [ -d "$dir" ] || continue
  name=$(basename "$dir")
  is_dated "$name" || continue
  if [ "$(day_number "$name")" -lt "$(day_number "$cutoff")" ]; then
    if rm -rf -- "$dir"; then
      log "rotation: removed $name"
    else
      log "rotation: could not remove $name"
      rotation_ok=false
    fi
  fi
done

if [ "$rotation_ok" != true ]; then
  report false "$files" rotation_failed || true
  exit "$EXIT_INSTALL"
fi

if ! report true "$files"; then
  exit "$EXIT_REPORT"
fi