vps-health-api/agent/pg-backup-pull/run-pg-backup-pull.sh

#!/bin/sh
#
# Daily pull of the encrypted PostgreSQL backups from the VPS onto this
# workstation (la-compagnie-maximus#220). Run by the systemd user timer
# maximus-pg-backup-pull.timer, unattended.
#
# The VPS (backup-postgres.sh, 03:00 UTC) age-encrypts one copy of every dump
# into /home/pgbackup/outbox/<date>/. This script fetches that outbox over the
# classic sshd on 100.86.30.21:2222, where the key can only run `rrsync -ro`.
# It never decrypts anything: the age private key is not on this machine.
#
# NEVER add `set -x`. The output goes to journald, and the shell would echo the
# `. "$ENV_FILE"` expansion — the ingest token — into it for good. Nothing below
# ever echoes a variable that holds a secret.

set -eu

# Overridable for testing only; the timer uses the defaults.
ENV_FILE="${PG_BACKUP_PULL_ENV_FILE:-$HOME/.config/maximus-pg-backup-pull.env}"
DEST="${PG_BACKUP_PULL_DEST:-$HOME/backups/postgres}"
STATE_DIR="${PG_BACKUP_PULL_STATE_DIR:-$HOME/.local/state/maximus-pg-backup-pull}"
SSH_KEY="${PG_BACKUP_PULL_SSH_KEY:-$HOME/.ssh/pgbackup_pull_ed25519}"
KNOWN_HOSTS="${PG_BACKUP_PULL_KNOWN_HOSTS:-$HOME/.ssh/known_hosts_pgbackup}"
TODAY="${PG_BACKUP_PULL_TODAY:-$(date -u +%F)}"

SOURCE="pgbackup@100.86.30.21:"
SSH_PORT=2222
RETENTION_DAYS=30
RSYNC_TIMEOUT_SECONDS=900
POST_TIMEOUT_SECONDS=20

EXIT_INSTALL=1
EXIT_PULL=2
EXIT_REPORT=3

log() { echo "pg-backup-pull: $*"; }

for bin in rsync ssh flock timeout; do
  if ! command -v "$bin" >/dev/null 2>&1; then
    log "$bin not found"
    exit "$EXIT_INSTALL"
  fi
done
for file in "$SSH_KEY" "$KNOWN_HOSTS"; do
  if [ ! -f "$file" ]; then
    log "missing $file"
    exit "$EXIT_INSTALL"
  fi
done

mkdir -p "$STATE_DIR" "$DEST"
chmod 700 "$STATE_DIR" "$DEST"

exec 9>"$STATE_DIR/run.lock"
if ! flock -n 9; then
  log "a run is already in progress, skipping"
  exit 0
fi

# The env file is optional until the ingest route is deployed: without a token,
# the pull still runs and only the report to the health API is skipped.
BACKUPS_API_URL=""
BACKUPS_INGEST_TOKEN=""
if [ -f "$ENV_FILE" ]; then
  FILE_MODE=$(stat -c "%a" "$ENV_FILE" 2>/dev/null || echo "")
  if [ -n "$FILE_MODE" ] && [ "$FILE_MODE" != "600" ]; then
    log "warning: $ENV_FILE is mode $FILE_MODE, expected 600"
  fi
  # shellcheck source=/dev/null
  . "$ENV_FILE"
fi

report() {
  # report <ok:true|false> <files> [error-code]
  if [ -z "$BACKUPS_INGEST_TOKEN" ] || [ -z "$BACKUPS_API_URL" ]; then
    log "no BACKUPS_INGEST_TOKEN/BACKUPS_API_URL in $ENV_FILE, report skipped"
    return 0
  fi
  if ! command -v curl >/dev/null 2>&1; then
    log "curl not found, report skipped"
    return 1
  fi
  pulled_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  if [ -n "${3:-}" ]; then
    body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s,"error":"%s"}' "$pulled_at" "$2" "$1" "$3")
  else
    body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s}' "$pulled_at" "$2" "$1")
  fi
  # The token goes through a header file, never through argv (visible in ps).
  header_file="$STATE_DIR/auth.header"
  (umask 077 && printf 'Authorization: Bearer %s\n' "$BACKUPS_INGEST_TOKEN" > "$header_file")
  status=$(printf '%s' "$body" | curl -sS -o /dev/null -w '%{http_code}' \
      --max-time "$POST_TIMEOUT_SECONDS" \
      -H @"$header_file" -H 'Content-Type: application/json' \
      --data-binary @- "$BACKUPS_API_URL/backups/pull" 2>/dev/null) || status="000"
  rm -f "$header_file"
  case "$status" in
    2??) log "report ok status=$status"; return 0 ;;
    *) log "report failed status=$status"; return 1 ;;
  esac
}

# --- Pull ---------------------------------------------------------------------
# Only directories and .age files cross: a misconfigured outbox can never land
# a clear dump here. No --delete: the VPS keeps 7 days, this machine keeps 30.
if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --partial-dir=.rsync-partial \
    --include='*/' --include='*.age' --exclude='*' \
    -e "ssh -p $SSH_PORT -i $SSH_KEY -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" \
    "$SOURCE" "$DEST/"; then
  log "pull failed"
  report false 0 pull_failed || true
  exit "$EXIT_PULL"
fi
chmod -R go-rwx "$DEST"

latest=$(find "$DEST" -mindepth 1 -maxdepth 1 -type d -name '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' -printf '%f\n' | sort | tail -1)
files=0
if [ -n "$latest" ]; then
  files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l)
fi
log "pull ok latest=${latest:-none} files=$files"

# --- Rotation (fails loudly) --------------------------------------------------
cutoff=$(date -u -d "$TODAY -$((RETENTION_DAYS - 1)) days" +%F)
rotation_ok=true
for dir in "$DEST"/*/; do
  [ -d "$dir" ] || continue
  name=$(basename "$dir")
  case "$name" in
    [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;;
    *) continue ;;
  esac
  # Compared as YYYYMMDD integers: `[ a \< b ]` is not POSIX, and /bin/sh is dash.
  if [ "$(echo "$name" | tr -d -)" -lt "$(echo "$cutoff" | tr -d -)" ]; then
    if rm -rf -- "$dir"; then
      log "rotation: removed $name"
    else
      log "rotation: could not remove $name"
      rotation_ok=false
    fi
  fi
done

if [ "$rotation_ok" != true ]; then
  report false "$files" rotation_failed || true
  exit "$EXIT_INSTALL"
fi

if ! report true "$files"; then
  exit "$EXIT_REPORT"
fi