feat(agent): daily pull of the encrypted PostgreSQL backups

POSIX sh agent run by a systemd user timer (Persistent=true, retries on
failure): rsync over the classic sshd on 100.86.30.21:2222 with a pinned
host key, .age files only, 30-day rotation that fails loudly, and a
report to POST /backups/pull when the ingest token is configured (sent
through a header file, never argv). Never decrypts.

Refs maximus/la-compagnie-maximus#220

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

__tests__/agent-pg-backup-pull.test.js

@@ -0,0 +1,207 @@
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const { spawnSync } = require("node:child_process");
+
+// Coverage for the PostgreSQL backup pull agent (la-compagnie-maximus#220).
+//
+// The script runs under a systemd user timer and talks to the VPS over
+// rsync/ssh, then reports to POST /backups/pull. Both are replaced here by fake
+// binaries on PATH, which record their argv so the tests can check what would
+// have crossed the wire — in particular that the ingest token never appears in
+// a command line nor in the script's own output (journald keeps it forever).
+
+const SCRIPT = path.join(__dirname, "..", "agent", "pg-backup-pull", "run-pg-backup-pull.sh");
+const TOKEN = "ingest-token-0123456789abcdef";
+const TODAY = "2026-10-11";
+
+let tmp;
+let bin;
+
+function writeExecutable(file, content) {
+  fs.writeFileSync(file, content, { mode: 0o755 });
+}
+
+beforeEach(() => {
+  tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pg-backup-pull-"));
+  bin = path.join(tmp, "bin");
+  fs.mkdirSync(bin);
+  fs.writeFileSync(path.join(tmp, "key"), "fake key", { mode: 0o600 });
+  fs.writeFileSync(path.join(tmp, "known_hosts"), "[100.86.30.21]:2222 ssh-ed25519 AAAA");
+
+  // Fake rsync: records argv, then lands two .age files in today's directory.
+  writeExecutable(
+    path.join(bin, "rsync"),
+    `#!/bin/sh
+printf '%s\\n' "$@" > "$FAKE_DIR/rsync.argv"
+[ -n "\${FAKE_RSYNC_FAIL:-}" ] && exit 23
+for last; do :; done
+mkdir -p "$last/${TODAY}"
+echo x > "$last/${TODAY}/maximus_api.dump.age"
+echo x > "$last/${TODAY}/globals.sql.age"
+`,
+  );
+  // Fake curl: records argv, the header file it was handed and the body.
+  writeExecutable(
+    path.join(bin, "curl"),
+    `#!/bin/sh
+printf '%s\\n' "$@" > "$FAKE_DIR/curl.argv"
+prev=""
+for a; do
+  case "$prev" in -H) case "$a" in @*) cp "\${a#@}" "$FAKE_DIR/curl.header" ;; esac ;; esac
+  prev="$a"
+done
+cat > "$FAKE_DIR/curl.body"
+printf '%s' "\${FAKE_CURL_STATUS:-204}"
+`,
+  );
+});
+
+afterEach(() => {
+  fs.rmSync(tmp, { recursive: true, force: true });
+});
+
+function run({ envFile, extraEnv = {} } = {}) {
+  const dest = path.join(tmp, "dest");
+  const result = spawnSync("sh", [SCRIPT], {
+    encoding: "utf8",
+    env: {
+      PATH: `${bin}:/usr/bin:/bin`,
+      HOME: tmp,
+      FAKE_DIR: tmp,
+      PG_BACKUP_PULL_ENV_FILE: envFile ?? path.join(tmp, "absent.env"),
+      PG_BACKUP_PULL_DEST: dest,
+      PG_BACKUP_PULL_STATE_DIR: path.join(tmp, "state"),
+      PG_BACKUP_PULL_SSH_KEY: path.join(tmp, "key"),
+      PG_BACKUP_PULL_KNOWN_HOSTS: path.join(tmp, "known_hosts"),
+      PG_BACKUP_PULL_TODAY: TODAY,
+      ...extraEnv,
+    },
+  });
+  return { ...result, output: result.stdout + result.stderr, dest };
+}
+
+function writeEnv() {
+  const file = path.join(tmp, "agent.env");
+  fs.writeFileSync(
+    file,
+    `BACKUPS_API_URL=https://health.example.test\nBACKUPS_INGEST_TOKEN=${TOKEN}\n`,
+    { mode: 0o600 },
+  );
+  return file;
+}
+
+const read = (name) => fs.readFileSync(path.join(tmp, name), "utf8");
+
+describe("pull", () => {
+  test("pulls over port 2222 with a pinned host key, .age files only, never --delete", () => {
+    const { status } = run();
+    expect(status).toBe(0);
+    const argv = read("rsync.argv").split("\n");
+    expect(argv).toContain("--include=*.age");
+    expect(argv).toContain("--exclude=*");
+    expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*"));
+    expect(argv).not.toContain("--delete");
+    expect(argv).toContain("pgbackup@100.86.30.21:");
+    const ssh = argv.find((a) => a.startsWith("ssh "));
+    expect(ssh).toContain("-p 2222");
+    expect(ssh).toContain("StrictHostKeyChecking=yes");
+    expect(ssh).toContain("BatchMode=yes");
+    expect(ssh).toContain("IdentitiesOnly=yes");
+  });
+
+  test("without a token, still pulls and exits 0, saying the report was skipped", () => {
+    const { status, output, dest } = run();
+    expect(status).toBe(0);
+    expect(output).toContain("report skipped");
+    expect(fs.existsSync(path.join(dest, TODAY, "maximus_api.dump.age"))).toBe(true);
+    expect(fs.existsSync(path.join(tmp, "curl.argv"))).toBe(false);
+  });
+
+  test("keeps the destination private", () => {
+    const { dest } = run();
+    expect(fs.statSync(dest).mode & 0o777).toBe(0o700);
+    expect(fs.statSync(path.join(dest, TODAY, "globals.sql.age")).mode & 0o077).toBe(0);
+  });
+
+  test("a failed pull exits 2 and reports pull_failed", () => {
+    const { status } = run({ envFile: writeEnv(), extraEnv: { FAKE_RSYNC_FAIL: "1" } });
+    expect(status).toBe(2);
+    expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, files: 0, error: "pull_failed" });
+  });
+
+  test("missing ssh key is an install problem: exit 1, nothing pulled", () => {
+    fs.rmSync(path.join(tmp, "key"));
+    const { status } = run();
+    expect(status).toBe(1);
+    expect(fs.existsSync(path.join(tmp, "rsync.argv"))).toBe(false);
+  });
+});
+
+describe("report", () => {
+  test("posts pulledAt, files and ok to /backups/pull", () => {
+    const { status } = run({ envFile: writeEnv() });
+    expect(status).toBe(0);
+    const body = JSON.parse(read("curl.body"));
+    expect(Object.keys(body).sort()).toEqual(["files", "ok", "pulledAt"]);
+    expect(body).toMatchObject({ ok: true, files: 2 });
+    expect(body.pulledAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/);
+    expect(read("curl.argv").split("\n")).toContain("https://health.example.test/backups/pull");
+  });
+
+  test("the token travels in a header file, never in argv nor in the output", () => {
+    const { output } = run({ envFile: writeEnv() });
+    expect(read("curl.header")).toBe(`Authorization: Bearer ${TOKEN}\n`);
+    expect(read("curl.argv")).not.toContain(TOKEN.slice(0, 12));
+    expect(output).not.toContain(TOKEN.slice(0, 12));
+    expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header");
+  });
+
+  test("a non-2xx answer exits 3", () => {
+    const { status, output } = run({ envFile: writeEnv(), extraEnv: { FAKE_CURL_STATUS: "401" } });
+    expect(status).toBe(3);
+    expect(output).toContain("report failed status=401");
+  });
+});
+
+describe("rotation", () => {
+  test("keeps 30 dated directories, today included, and ignores other names", () => {
+    const dest = path.join(tmp, "dest");
+    for (const d of ["2026-09-11", "2026-09-12", "notes"]) fs.mkdirSync(path.join(dest, d), { recursive: true });
+    const { status } = run();
+    expect(status).toBe(0);
+    expect(fs.existsSync(path.join(dest, "2026-09-11"))).toBe(false);
+    expect(fs.existsSync(path.join(dest, "2026-09-12"))).toBe(true);
+    expect(fs.existsSync(path.join(dest, "notes"))).toBe(true);
+  });
+
+  test("a directory that cannot be removed fails loudly and reports rotation_failed", () => {
+    const dest = path.join(tmp, "dest");
+    const old = path.join(dest, "2026-09-01");
+    fs.mkdirSync(path.join(old, "locked"), { recursive: true });
+    fs.writeFileSync(path.join(old, "locked", "f.age"), "x");
+    fs.chmodSync(path.join(old, "locked"), 0o500);
+    try {
+      const { status } = run({ envFile: writeEnv() });
+      expect(status).toBe(1);
+      expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" });
+    } finally {
+      fs.chmodSync(path.join(old, "locked"), 0o700);
+    }
+  });
+});
+
+describe("the script itself", () => {
+  test("never enables shell tracing — `set -x` would echo the token into journald", () => {
+    const traced = fs
+      .readFileSync(SCRIPT, "utf8")
+      .split("\n")
+      .filter((line) => !line.trim().startsWith("#"))
+      .filter((line) => /\bset\b[^#\n]*-[a-z]*x/.test(line));
+    expect(traced).toEqual([]);
+  });
+
+  test("is executable", () => {
+    expect(fs.statSync(SCRIPT).mode & 0o111).toBeGreaterThan(0);
+  });
+});

agent/README.md

@@ -361,3 +361,77 @@ agent itself does not change.
    `GET /claude-usage/history?from=<first month>&to=<last month>`.
 
 Then delete `~/claude-usage-backfill.json` on the workstation.
+
+## PostgreSQL backup pull (`agent/pg-backup-pull/`)
+
+A third, independent agent on the Lenovo. Once a day it pulls the
+age-encrypted dumps of the shared PostgreSQL cluster from the VPS, keeps 30
+days of them in `~/backups/postgres/`, and reports the pull to
+`POST /backups/pull`. The VPS side (`backup-postgres.sh`, 03:00 UTC), the
+`status.json` contract and the restore procedure live in
+`la-compagnie-maximus/docs/postgres-ops.md`, section Backup
+(la-compagnie-maximus#220).
+
+| File | Role |
+|------|------|
+| `run-pg-backup-pull.sh` | rsync over ssh, rotation, report; POSIX `sh`, no node |
+| `maximus-pg-backup-pull.service` | systemd user unit, oneshot, retries on failure |
+| `maximus-pg-backup-pull.timer` | `OnCalendar=daily`, `Persistent=true` |
+
+**The only departure from the cron convention of the other agents is the
+timer.** A cron line missed while the laptop is off is simply lost;
+`Persistent=true` runs the missed pass at the next boot. That catch-up run can
+start before Tailscale is up, hence `Restart=on-failure` every 15 minutes (8
+tries at most in 4 hours).
+
+**What it never does.** It never decrypts: the age private key is not on this
+machine (Vaultwarden plus a paper copy). It never deletes on the VPS side, and
+rsync only lets directories and `*.age` files through, so a misconfigured
+outbox cannot land a clear dump here. The key it uses
+(`~/.ssh/pgbackup_pull_ed25519`) can only run `rrsync -ro` on the outbox, from
+the Lenovo's Tailscale IP, on the classic sshd at `100.86.30.21:2222`; the VPS
+host key is pinned in `~/.ssh/known_hosts_pgbackup` (no trust on first use).
+
+### Install
+
+```
+mkdir -p ~/.local/share/maximus-pg-backup-pull ~/.config/systemd/user
+cp agent/pg-backup-pull/run-pg-backup-pull.sh ~/.local/share/maximus-pg-backup-pull/
+chmod +x ~/.local/share/maximus-pg-backup-pull/run-pg-backup-pull.sh
+cp agent/pg-backup-pull/maximus-pg-backup-pull.{service,timer} ~/.config/systemd/user/
+systemctl --user daemon-reload
+systemctl --user enable --now maximus-pg-backup-pull.timer
+```
+
+As for the other agents, the copy step *is* the deployment: never point the
+unit at the git working tree. The user manager must run without an open
+session: `loginctl show-user $USER -p Linger` must say `yes`.
+
+The env file `~/.config/maximus-pg-backup-pull.env` (mode `600`) is optional
+until the ingest route is deployed. Without it the pull runs and the report is
+skipped, with one log line. Then, written with `printf`:
+
+```
+BACKUPS_API_URL=https://health.lacompagniemaximus.com
+BACKUPS_INGEST_TOKEN=<the backups ingest token>
+```
+
+The token reaches curl through a header file created `0600` and removed right
+after, never through argv.
+
+### Check
+
+```
+systemctl --user start maximus-pg-backup-pull.service
+journalctl --user -t pg-backup-pull --since -1h
+systemctl --user list-timers maximus-pg-backup-pull.timer
+```
+
+### Exit codes
+
+| Code | Meaning |
+|------|---------|
+| 0 | Pulled, rotated, reported (or report skipped without a token) |
+| 1 | Install problem (missing key, known_hosts, binary) or rotation failure |
+| 2 | The pull failed (`pull_failed` reported) |
+| 3 | Pulled and rotated, but the report did not land |

agent/pg-backup-pull/maximus-pg-backup-pull.service

@@ -0,0 +1,15 @@
+# Daily pull of the encrypted PostgreSQL backups (la-compagnie-maximus#220).
+# Installed as a systemd *user* unit, see agent/README.md.
+[Unit]
+Description=Pull the encrypted PostgreSQL backups from the VPS
+# A run caught up at boot may start before Tailscale is up: retry a few times.
+StartLimitIntervalSec=4h
+StartLimitBurst=8
+
+[Service]
+Type=oneshot
+ExecStart=%h/.local/share/maximus-pg-backup-pull/run-pg-backup-pull.sh
+SyslogIdentifier=pg-backup-pull
+Nice=10
+Restart=on-failure
+RestartSec=15min

agent/pg-backup-pull/maximus-pg-backup-pull.timer

@@ -0,0 +1,13 @@
+# Daily, after the VPS run (03:00 UTC). Persistent=true catches up a run missed
+# while the workstation was off, as soon as it is back (needs `loginctl
+# enable-linger` for the user manager to run without an open session).
+[Unit]
+Description=Daily pull of the encrypted PostgreSQL backups
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+RandomizedDelaySec=15min
+
+[Install]
+WantedBy=timers.target

agent/pg-backup-pull/run-pg-backup-pull.sh

@@ -0,0 +1,151 @@
+#!/bin/sh
+#
+# Daily pull of the encrypted PostgreSQL backups from the VPS onto this
+# workstation (la-compagnie-maximus#220). Run by the systemd user timer
+# maximus-pg-backup-pull.timer, unattended.
+#
+# The VPS (backup-postgres.sh, 03:00 UTC) age-encrypts one copy of every dump
+# into /home/pgbackup/outbox/<date>/. This script fetches that outbox over the
+# classic sshd on 100.86.30.21:2222, where the key can only run `rrsync -ro`.
+# It never decrypts anything: the age private key is not on this machine.
+#
+# NEVER add `set -x`. The output goes to journald, and the shell would echo the
+# `. "$ENV_FILE"` expansion — the ingest token — into it for good. Nothing below
+# ever echoes a variable that holds a secret.
+
+set -eu
+
+# Overridable for testing only; the timer uses the defaults.
+ENV_FILE="${PG_BACKUP_PULL_ENV_FILE:-$HOME/.config/maximus-pg-backup-pull.env}"
+DEST="${PG_BACKUP_PULL_DEST:-$HOME/backups/postgres}"
+STATE_DIR="${PG_BACKUP_PULL_STATE_DIR:-$HOME/.local/state/maximus-pg-backup-pull}"
+SSH_KEY="${PG_BACKUP_PULL_SSH_KEY:-$HOME/.ssh/pgbackup_pull_ed25519}"
+KNOWN_HOSTS="${PG_BACKUP_PULL_KNOWN_HOSTS:-$HOME/.ssh/known_hosts_pgbackup}"
+TODAY="${PG_BACKUP_PULL_TODAY:-$(date -u +%F)}"
+
+SOURCE="pgbackup@100.86.30.21:"
+SSH_PORT=2222
+RETENTION_DAYS=30
+RSYNC_TIMEOUT_SECONDS=900
+POST_TIMEOUT_SECONDS=20
+
+EXIT_INSTALL=1
+EXIT_PULL=2
+EXIT_REPORT=3
+
+log() { echo "pg-backup-pull: $*"; }
+
+for bin in rsync ssh flock timeout; do
+  if ! command -v "$bin" >/dev/null 2>&1; then
+    log "$bin not found"
+    exit "$EXIT_INSTALL"
+  fi
+done
+for file in "$SSH_KEY" "$KNOWN_HOSTS"; do
+  if [ ! -f "$file" ]; then
+    log "missing $file"
+    exit "$EXIT_INSTALL"
+  fi
+done
+
+mkdir -p "$STATE_DIR" "$DEST"
+chmod 700 "$STATE_DIR" "$DEST"
+
+exec 9>"$STATE_DIR/run.lock"
+if ! flock -n 9; then
+  log "a run is already in progress, skipping"
+  exit 0
+fi
+
+# The env file is optional until the ingest route is deployed: without a token,
+# the pull still runs and only the report to the health API is skipped.
+BACKUPS_API_URL=""
+BACKUPS_INGEST_TOKEN=""
+if [ -f "$ENV_FILE" ]; then
+  FILE_MODE=$(stat -c "%a" "$ENV_FILE" 2>/dev/null || echo "")
+  if [ -n "$FILE_MODE" ] && [ "$FILE_MODE" != "600" ]; then
+    log "warning: $ENV_FILE is mode $FILE_MODE, expected 600"
+  fi
+  # shellcheck source=/dev/null
+  . "$ENV_FILE"
+fi
+
+report() {
+  # report <ok:true|false> <files> [error-code]
+  if [ -z "$BACKUPS_INGEST_TOKEN" ] || [ -z "$BACKUPS_API_URL" ]; then
+    log "no BACKUPS_INGEST_TOKEN/BACKUPS_API_URL in $ENV_FILE, report skipped"
+    return 0
+  fi
+  if ! command -v curl >/dev/null 2>&1; then
+    log "curl not found, report skipped"
+    return 1
+  fi
+  pulled_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+  if [ -n "${3:-}" ]; then
+    body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s,"error":"%s"}' "$pulled_at" "$2" "$1" "$3")
+  else
+    body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s}' "$pulled_at" "$2" "$1")
+  fi
+  # The token goes through a header file, never through argv (visible in ps).
+  header_file="$STATE_DIR/auth.header"
+  (umask 077 && printf 'Authorization: Bearer %s\n' "$BACKUPS_INGEST_TOKEN" > "$header_file")
+  status=$(printf '%s' "$body" | curl -sS -o /dev/null -w '%{http_code}' \
+      --max-time "$POST_TIMEOUT_SECONDS" \
+      -H @"$header_file" -H 'Content-Type: application/json' \
+      --data-binary @- "$BACKUPS_API_URL/backups/pull" 2>/dev/null) || status="000"
+  rm -f "$header_file"
+  case "$status" in
+    2??) log "report ok status=$status"; return 0 ;;
+    *) log "report failed status=$status"; return 1 ;;
+  esac
+}
+
+# --- Pull ---------------------------------------------------------------------
+# Only directories and .age files cross: a misconfigured outbox can never land
+# a clear dump here. No --delete: the VPS keeps 7 days, this machine keeps 30.
+if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --partial-dir=.rsync-partial \
+    --include='*/' --include='*.age' --exclude='*' \
+    -e "ssh -p $SSH_PORT -i $SSH_KEY -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" \
+    "$SOURCE" "$DEST/"; then
+  log "pull failed"
+  report false 0 pull_failed || true
+  exit "$EXIT_PULL"
+fi
+chmod -R go-rwx "$DEST"
+
+latest=$(find "$DEST" -mindepth 1 -maxdepth 1 -type d -name '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' -printf '%f\n' | sort | tail -1)
+files=0
+if [ -n "$latest" ]; then
+  files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l)
+fi
+log "pull ok latest=${latest:-none} files=$files"
+
+# --- Rotation (fails loudly) --------------------------------------------------
+cutoff=$(date -u -d "$TODAY -$((RETENTION_DAYS - 1)) days" +%F)
+rotation_ok=true
+for dir in "$DEST"/*/; do
+  [ -d "$dir" ] || continue
+  name=$(basename "$dir")
+  case "$name" in
+    [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;;
+    *) continue ;;
+  esac
+  # Compared as YYYYMMDD integers: `[ a \< b ]` is not POSIX, and /bin/sh is dash.
+  if [ "$(echo "$name" | tr -d -)" -lt "$(echo "$cutoff" | tr -d -)" ]; then
+    if rm -rf -- "$dir"; then
+      log "rotation: removed $name"
+    else
+      log "rotation: could not remove $name"
+      rotation_ok=false
+    fi
+  fi
+done
+
+if [ "$rotation_ok" != true ]; then
+  report false "$files" rotation_failed || true
+  exit "$EXIT_INSTALL"
+fi
+
+if ! report true "$files"; then
+  exit "$EXIT_REPORT"
+fi