From 59b963220df6dc8728a453f79fa734bdb9f0f3b2 Mon Sep 17 00:00:00 2001 From: le king fu Date: Sat, 10 Oct 2026 16:18:17 -0400 Subject: [PATCH 1/4] feat(agent): daily pull of the encrypted PostgreSQL backups POSIX sh agent run by a systemd user timer (Persistent=true, retries on failure): rsync over the classic sshd on 100.86.30.21:2222 with a pinned host key, .age files only, 30-day rotation that fails loudly, and a report to POST /backups/pull when the ingest token is configured (sent through a header file, never argv). Never decrypts. Refs maximus/la-compagnie-maximus#220 Co-Authored-By: Claude Opus 5.5 --- __tests__/agent-pg-backup-pull.test.js | 207 ++++++++++++++++++ agent/README.md | 74 +++++++ .../maximus-pg-backup-pull.service | 15 ++ .../maximus-pg-backup-pull.timer | 13 ++ agent/pg-backup-pull/run-pg-backup-pull.sh | 151 +++++++++++++ 5 files changed, 460 insertions(+) create mode 100644 __tests__/agent-pg-backup-pull.test.js create mode 100644 agent/pg-backup-pull/maximus-pg-backup-pull.service create mode 100644 agent/pg-backup-pull/maximus-pg-backup-pull.timer create mode 100755 agent/pg-backup-pull/run-pg-backup-pull.sh diff --git a/__tests__/agent-pg-backup-pull.test.js b/__tests__/agent-pg-backup-pull.test.js new file mode 100644 index 0000000..ccc7fc5 --- /dev/null +++ b/__tests__/agent-pg-backup-pull.test.js @@ -0,0 +1,207 @@ +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +// Coverage for the PostgreSQL backup pull agent (la-compagnie-maximus#220). +// +// The script runs under a systemd user timer and talks to the VPS over +// rsync/ssh, then reports to POST /backups/pull. Both are replaced here by fake +// binaries on PATH, which record their argv so the tests can check what would +// have crossed the wire — in particular that the ingest token never appears in +// a command line nor in the script's own output (journald keeps it forever). + +const SCRIPT = path.join(__dirname, "..", "agent", "pg-backup-pull", "run-pg-backup-pull.sh"); +const TOKEN = "ingest-token-0123456789abcdef"; +const TODAY = "2026-10-11"; + +let tmp; +let bin; + +function writeExecutable(file, content) { + fs.writeFileSync(file, content, { mode: 0o755 }); +} + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pg-backup-pull-")); + bin = path.join(tmp, "bin"); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(tmp, "key"), "fake key", { mode: 0o600 }); + fs.writeFileSync(path.join(tmp, "known_hosts"), "[100.86.30.21]:2222 ssh-ed25519 AAAA"); + + // Fake rsync: records argv, then lands two .age files in today's directory. + writeExecutable( + path.join(bin, "rsync"), + `#!/bin/sh +printf '%s\\n' "$@" > "$FAKE_DIR/rsync.argv" +[ -n "\${FAKE_RSYNC_FAIL:-}" ] && exit 23 +for last; do :; done +mkdir -p "$last/${TODAY}" +echo x > "$last/${TODAY}/maximus_api.dump.age" +echo x > "$last/${TODAY}/globals.sql.age" +`, + ); + // Fake curl: records argv, the header file it was handed and the body. + writeExecutable( + path.join(bin, "curl"), + `#!/bin/sh +printf '%s\\n' "$@" > "$FAKE_DIR/curl.argv" +prev="" +for a; do + case "$prev" in -H) case "$a" in @*) cp "\${a#@}" "$FAKE_DIR/curl.header" ;; esac ;; esac + prev="$a" +done +cat > "$FAKE_DIR/curl.body" +printf '%s' "\${FAKE_CURL_STATUS:-204}" +`, + ); +}); + +afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); +}); + +function run({ envFile, extraEnv = {} } = {}) { + const dest = path.join(tmp, "dest"); + const result = spawnSync("sh", [SCRIPT], { + encoding: "utf8", + env: { + PATH: `${bin}:/usr/bin:/bin`, + HOME: tmp, + FAKE_DIR: tmp, + PG_BACKUP_PULL_ENV_FILE: envFile ?? path.join(tmp, "absent.env"), + PG_BACKUP_PULL_DEST: dest, + PG_BACKUP_PULL_STATE_DIR: path.join(tmp, "state"), + PG_BACKUP_PULL_SSH_KEY: path.join(tmp, "key"), + PG_BACKUP_PULL_KNOWN_HOSTS: path.join(tmp, "known_hosts"), + PG_BACKUP_PULL_TODAY: TODAY, + ...extraEnv, + }, + }); + return { ...result, output: result.stdout + result.stderr, dest }; +} + +function writeEnv() { + const file = path.join(tmp, "agent.env"); + fs.writeFileSync( + file, + `BACKUPS_API_URL=https://health.example.test\nBACKUPS_INGEST_TOKEN=${TOKEN}\n`, + { mode: 0o600 }, + ); + return file; +} + +const read = (name) => fs.readFileSync(path.join(tmp, name), "utf8"); + +describe("pull", () => { + test("pulls over port 2222 with a pinned host key, .age files only, never --delete", () => { + const { status } = run(); + expect(status).toBe(0); + const argv = read("rsync.argv").split("\n"); + expect(argv).toContain("--include=*.age"); + expect(argv).toContain("--exclude=*"); + expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*")); + expect(argv).not.toContain("--delete"); + expect(argv).toContain("pgbackup@100.86.30.21:"); + const ssh = argv.find((a) => a.startsWith("ssh ")); + expect(ssh).toContain("-p 2222"); + expect(ssh).toContain("StrictHostKeyChecking=yes"); + expect(ssh).toContain("BatchMode=yes"); + expect(ssh).toContain("IdentitiesOnly=yes"); + }); + + test("without a token, still pulls and exits 0, saying the report was skipped", () => { + const { status, output, dest } = run(); + expect(status).toBe(0); + expect(output).toContain("report skipped"); + expect(fs.existsSync(path.join(dest, TODAY, "maximus_api.dump.age"))).toBe(true); + expect(fs.existsSync(path.join(tmp, "curl.argv"))).toBe(false); + }); + + test("keeps the destination private", () => { + const { dest } = run(); + expect(fs.statSync(dest).mode & 0o777).toBe(0o700); + expect(fs.statSync(path.join(dest, TODAY, "globals.sql.age")).mode & 0o077).toBe(0); + }); + + test("a failed pull exits 2 and reports pull_failed", () => { + const { status } = run({ envFile: writeEnv(), extraEnv: { FAKE_RSYNC_FAIL: "1" } }); + expect(status).toBe(2); + expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, files: 0, error: "pull_failed" }); + }); + + test("missing ssh key is an install problem: exit 1, nothing pulled", () => { + fs.rmSync(path.join(tmp, "key")); + const { status } = run(); + expect(status).toBe(1); + expect(fs.existsSync(path.join(tmp, "rsync.argv"))).toBe(false); + }); +}); + +describe("report", () => { + test("posts pulledAt, files and ok to /backups/pull", () => { + const { status } = run({ envFile: writeEnv() }); + expect(status).toBe(0); + const body = JSON.parse(read("curl.body")); + expect(Object.keys(body).sort()).toEqual(["files", "ok", "pulledAt"]); + expect(body).toMatchObject({ ok: true, files: 2 }); + expect(body.pulledAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/); + expect(read("curl.argv").split("\n")).toContain("https://health.example.test/backups/pull"); + }); + + test("the token travels in a header file, never in argv nor in the output", () => { + const { output } = run({ envFile: writeEnv() }); + expect(read("curl.header")).toBe(`Authorization: Bearer ${TOKEN}\n`); + expect(read("curl.argv")).not.toContain(TOKEN.slice(0, 12)); + expect(output).not.toContain(TOKEN.slice(0, 12)); + expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header"); + }); + + test("a non-2xx answer exits 3", () => { + const { status, output } = run({ envFile: writeEnv(), extraEnv: { FAKE_CURL_STATUS: "401" } }); + expect(status).toBe(3); + expect(output).toContain("report failed status=401"); + }); +}); + +describe("rotation", () => { + test("keeps 30 dated directories, today included, and ignores other names", () => { + const dest = path.join(tmp, "dest"); + for (const d of ["2026-09-11", "2026-09-12", "notes"]) fs.mkdirSync(path.join(dest, d), { recursive: true }); + const { status } = run(); + expect(status).toBe(0); + expect(fs.existsSync(path.join(dest, "2026-09-11"))).toBe(false); + expect(fs.existsSync(path.join(dest, "2026-09-12"))).toBe(true); + expect(fs.existsSync(path.join(dest, "notes"))).toBe(true); + }); + + test("a directory that cannot be removed fails loudly and reports rotation_failed", () => { + const dest = path.join(tmp, "dest"); + const old = path.join(dest, "2026-09-01"); + fs.mkdirSync(path.join(old, "locked"), { recursive: true }); + fs.writeFileSync(path.join(old, "locked", "f.age"), "x"); + fs.chmodSync(path.join(old, "locked"), 0o500); + try { + const { status } = run({ envFile: writeEnv() }); + expect(status).toBe(1); + expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" }); + } finally { + fs.chmodSync(path.join(old, "locked"), 0o700); + } + }); +}); + +describe("the script itself", () => { + test("never enables shell tracing — `set -x` would echo the token into journald", () => { + const traced = fs + .readFileSync(SCRIPT, "utf8") + .split("\n") + .filter((line) => !line.trim().startsWith("#")) + .filter((line) => /\bset\b[^#\n]*-[a-z]*x/.test(line)); + expect(traced).toEqual([]); + }); + + test("is executable", () => { + expect(fs.statSync(SCRIPT).mode & 0o111).toBeGreaterThan(0); + }); +}); diff --git a/agent/README.md b/agent/README.md index 821b2ac..2b99b52 100644 --- a/agent/README.md +++ b/agent/README.md @@ -361,3 +361,77 @@ agent itself does not change. `GET /claude-usage/history?from=&to=`. Then delete `~/claude-usage-backfill.json` on the workstation. + +## PostgreSQL backup pull (`agent/pg-backup-pull/`) + +A third, independent agent on the Lenovo. Once a day it pulls the +age-encrypted dumps of the shared PostgreSQL cluster from the VPS, keeps 30 +days of them in `~/backups/postgres/`, and reports the pull to +`POST /backups/pull`. The VPS side (`backup-postgres.sh`, 03:00 UTC), the +`status.json` contract and the restore procedure live in +`la-compagnie-maximus/docs/postgres-ops.md`, section Backup +(la-compagnie-maximus#220). + +| File | Role | +|------|------| +| `run-pg-backup-pull.sh` | rsync over ssh, rotation, report; POSIX `sh`, no node | +| `maximus-pg-backup-pull.service` | systemd user unit, oneshot, retries on failure | +| `maximus-pg-backup-pull.timer` | `OnCalendar=daily`, `Persistent=true` | + +**The only departure from the cron convention of the other agents is the +timer.** A cron line missed while the laptop is off is simply lost; +`Persistent=true` runs the missed pass at the next boot. That catch-up run can +start before Tailscale is up, hence `Restart=on-failure` every 15 minutes (8 +tries at most in 4 hours). + +**What it never does.** It never decrypts: the age private key is not on this +machine (Vaultwarden plus a paper copy). It never deletes on the VPS side, and +rsync only lets directories and `*.age` files through, so a misconfigured +outbox cannot land a clear dump here. The key it uses +(`~/.ssh/pgbackup_pull_ed25519`) can only run `rrsync -ro` on the outbox, from +the Lenovo's Tailscale IP, on the classic sshd at `100.86.30.21:2222`; the VPS +host key is pinned in `~/.ssh/known_hosts_pgbackup` (no trust on first use). + +### Install + +``` +mkdir -p ~/.local/share/maximus-pg-backup-pull ~/.config/systemd/user +cp agent/pg-backup-pull/run-pg-backup-pull.sh ~/.local/share/maximus-pg-backup-pull/ +chmod +x ~/.local/share/maximus-pg-backup-pull/run-pg-backup-pull.sh +cp agent/pg-backup-pull/maximus-pg-backup-pull.{service,timer} ~/.config/systemd/user/ +systemctl --user daemon-reload +systemctl --user enable --now maximus-pg-backup-pull.timer +``` + +As for the other agents, the copy step *is* the deployment: never point the +unit at the git working tree. The user manager must run without an open +session: `loginctl show-user $USER -p Linger` must say `yes`. + +The env file `~/.config/maximus-pg-backup-pull.env` (mode `600`) is optional +until the ingest route is deployed. Without it the pull runs and the report is +skipped, with one log line. Then, written with `printf`: + +``` +BACKUPS_API_URL=https://health.lacompagniemaximus.com +BACKUPS_INGEST_TOKEN= +``` + +The token reaches curl through a header file created `0600` and removed right +after, never through argv. + +### Check + +``` +systemctl --user start maximus-pg-backup-pull.service +journalctl --user -t pg-backup-pull --since -1h +systemctl --user list-timers maximus-pg-backup-pull.timer +``` + +### Exit codes + +| Code | Meaning | +|------|---------| +| 0 | Pulled, rotated, reported (or report skipped without a token) | +| 1 | Install problem (missing key, known_hosts, binary) or rotation failure | +| 2 | The pull failed (`pull_failed` reported) | +| 3 | Pulled and rotated, but the report did not land | diff --git a/agent/pg-backup-pull/maximus-pg-backup-pull.service b/agent/pg-backup-pull/maximus-pg-backup-pull.service new file mode 100644 index 0000000..98964bd --- /dev/null +++ b/agent/pg-backup-pull/maximus-pg-backup-pull.service @@ -0,0 +1,15 @@ +# Daily pull of the encrypted PostgreSQL backups (la-compagnie-maximus#220). +# Installed as a systemd *user* unit, see agent/README.md. +[Unit] +Description=Pull the encrypted PostgreSQL backups from the VPS +# A run caught up at boot may start before Tailscale is up: retry a few times. +StartLimitIntervalSec=4h +StartLimitBurst=8 + +[Service] +Type=oneshot +ExecStart=%h/.local/share/maximus-pg-backup-pull/run-pg-backup-pull.sh +SyslogIdentifier=pg-backup-pull +Nice=10 +Restart=on-failure +RestartSec=15min diff --git a/agent/pg-backup-pull/maximus-pg-backup-pull.timer b/agent/pg-backup-pull/maximus-pg-backup-pull.timer new file mode 100644 index 0000000..21f1a80 --- /dev/null +++ b/agent/pg-backup-pull/maximus-pg-backup-pull.timer @@ -0,0 +1,13 @@ +# Daily, after the VPS run (03:00 UTC). Persistent=true catches up a run missed +# while the workstation was off, as soon as it is back (needs `loginctl +# enable-linger` for the user manager to run without an open session). +[Unit] +Description=Daily pull of the encrypted PostgreSQL backups + +[Timer] +OnCalendar=daily +Persistent=true +RandomizedDelaySec=15min + +[Install] +WantedBy=timers.target diff --git a/agent/pg-backup-pull/run-pg-backup-pull.sh b/agent/pg-backup-pull/run-pg-backup-pull.sh new file mode 100755 index 0000000..d2d949d --- /dev/null +++ b/agent/pg-backup-pull/run-pg-backup-pull.sh @@ -0,0 +1,151 @@ +#!/bin/sh +# +# Daily pull of the encrypted PostgreSQL backups from the VPS onto this +# workstation (la-compagnie-maximus#220). Run by the systemd user timer +# maximus-pg-backup-pull.timer, unattended. +# +# The VPS (backup-postgres.sh, 03:00 UTC) age-encrypts one copy of every dump +# into /home/pgbackup/outbox//. This script fetches that outbox over the +# classic sshd on 100.86.30.21:2222, where the key can only run `rrsync -ro`. +# It never decrypts anything: the age private key is not on this machine. +# +# NEVER add `set -x`. The output goes to journald, and the shell would echo the +# `. "$ENV_FILE"` expansion — the ingest token — into it for good. Nothing below +# ever echoes a variable that holds a secret. + +set -eu + +# Overridable for testing only; the timer uses the defaults. +ENV_FILE="${PG_BACKUP_PULL_ENV_FILE:-$HOME/.config/maximus-pg-backup-pull.env}" +DEST="${PG_BACKUP_PULL_DEST:-$HOME/backups/postgres}" +STATE_DIR="${PG_BACKUP_PULL_STATE_DIR:-$HOME/.local/state/maximus-pg-backup-pull}" +SSH_KEY="${PG_BACKUP_PULL_SSH_KEY:-$HOME/.ssh/pgbackup_pull_ed25519}" +KNOWN_HOSTS="${PG_BACKUP_PULL_KNOWN_HOSTS:-$HOME/.ssh/known_hosts_pgbackup}" +TODAY="${PG_BACKUP_PULL_TODAY:-$(date -u +%F)}" + +SOURCE="pgbackup@100.86.30.21:" +SSH_PORT=2222 +RETENTION_DAYS=30 +RSYNC_TIMEOUT_SECONDS=900 +POST_TIMEOUT_SECONDS=20 + +EXIT_INSTALL=1 +EXIT_PULL=2 +EXIT_REPORT=3 + +log() { echo "pg-backup-pull: $*"; } + +for bin in rsync ssh flock timeout; do + if ! command -v "$bin" >/dev/null 2>&1; then + log "$bin not found" + exit "$EXIT_INSTALL" + fi +done +for file in "$SSH_KEY" "$KNOWN_HOSTS"; do + if [ ! -f "$file" ]; then + log "missing $file" + exit "$EXIT_INSTALL" + fi +done + +mkdir -p "$STATE_DIR" "$DEST" +chmod 700 "$STATE_DIR" "$DEST" + +exec 9>"$STATE_DIR/run.lock" +if ! flock -n 9; then + log "a run is already in progress, skipping" + exit 0 +fi + +# The env file is optional until the ingest route is deployed: without a token, +# the pull still runs and only the report to the health API is skipped. +BACKUPS_API_URL="" +BACKUPS_INGEST_TOKEN="" +if [ -f "$ENV_FILE" ]; then + FILE_MODE=$(stat -c "%a" "$ENV_FILE" 2>/dev/null || echo "") + if [ -n "$FILE_MODE" ] && [ "$FILE_MODE" != "600" ]; then + log "warning: $ENV_FILE is mode $FILE_MODE, expected 600" + fi + # shellcheck source=/dev/null + . "$ENV_FILE" +fi + +report() { + # report [error-code] + if [ -z "$BACKUPS_INGEST_TOKEN" ] || [ -z "$BACKUPS_API_URL" ]; then + log "no BACKUPS_INGEST_TOKEN/BACKUPS_API_URL in $ENV_FILE, report skipped" + return 0 + fi + if ! command -v curl >/dev/null 2>&1; then + log "curl not found, report skipped" + return 1 + fi + pulled_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + if [ -n "${3:-}" ]; then + body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s,"error":"%s"}' "$pulled_at" "$2" "$1" "$3") + else + body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s}' "$pulled_at" "$2" "$1") + fi + # The token goes through a header file, never through argv (visible in ps). + header_file="$STATE_DIR/auth.header" + (umask 077 && printf 'Authorization: Bearer %s\n' "$BACKUPS_INGEST_TOKEN" > "$header_file") + status=$(printf '%s' "$body" | curl -sS -o /dev/null -w '%{http_code}' \ + --max-time "$POST_TIMEOUT_SECONDS" \ + -H @"$header_file" -H 'Content-Type: application/json' \ + --data-binary @- "$BACKUPS_API_URL/backups/pull" 2>/dev/null) || status="000" + rm -f "$header_file" + case "$status" in + 2??) log "report ok status=$status"; return 0 ;; + *) log "report failed status=$status"; return 1 ;; + esac +} + +# --- Pull --------------------------------------------------------------------- +# Only directories and .age files cross: a misconfigured outbox can never land +# a clear dump here. No --delete: the VPS keeps 7 days, this machine keeps 30. +if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --partial-dir=.rsync-partial \ + --include='*/' --include='*.age' --exclude='*' \ + -e "ssh -p $SSH_PORT -i $SSH_KEY -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" \ + "$SOURCE" "$DEST/"; then + log "pull failed" + report false 0 pull_failed || true + exit "$EXIT_PULL" +fi +chmod -R go-rwx "$DEST" + +latest=$(find "$DEST" -mindepth 1 -maxdepth 1 -type d -name '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' -printf '%f\n' | sort | tail -1) +files=0 +if [ -n "$latest" ]; then + files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l) +fi +log "pull ok latest=${latest:-none} files=$files" + +# --- Rotation (fails loudly) -------------------------------------------------- +cutoff=$(date -u -d "$TODAY -$((RETENTION_DAYS - 1)) days" +%F) +rotation_ok=true +for dir in "$DEST"/*/; do + [ -d "$dir" ] || continue + name=$(basename "$dir") + case "$name" in + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; + *) continue ;; + esac + # Compared as YYYYMMDD integers: `[ a \< b ]` is not POSIX, and /bin/sh is dash. + if [ "$(echo "$name" | tr -d -)" -lt "$(echo "$cutoff" | tr -d -)" ]; then + if rm -rf -- "$dir"; then + log "rotation: removed $name" + else + log "rotation: could not remove $name" + rotation_ok=false + fi + fi +done + +if [ "$rotation_ok" != true ]; then + report false "$files" rotation_failed || true + exit "$EXIT_INSTALL" +fi + +if ! report true "$files"; then + exit "$EXIT_REPORT" +fi -- 2.45.2 From b87b3d72535089362f3655ea6babfb6181e1ae3d Mon Sep 17 00:00:00 2001 From: le king fu Date: Sat, 10 Oct 2026 18:56:05 -0400 Subject: [PATCH 2/4] fix(agent): run the backup pull under busybox, catch an empty filter The CI runs the tests in node:22-alpine, where 9 of the 12 cases failed: no ssh on PATH, and busybox has neither `find -printf` nor `date -d "... days"`. The newest dated directory is now found by a loop over the glob, and the rotation cutoff by epoch arithmetic, which GNU date and busybox both read. The tests add a fake ssh, and a fake flock where the host has none. The rotation_failed case failed the pass through a read-only directory, which root ignores: it now uses a fake rm. Review suggestions: the rsync argv must put --include=*/ before --exclude=* (without it nothing crosses and the pass still says ok), and a case runs the real rsync through the fake ssh when rsync is present. A pass whose newest directory is not today's logs a warning. Refs maximus/la-compagnie-maximus#220 Co-Authored-By: Claude Opus 5.5 --- __tests__/agent-pg-backup-pull.test.js | 90 +++++++++++++++++++--- agent/README.md | 9 +++ agent/pg-backup-pull/run-pg-backup-pull.sh | 42 +++++++--- 3 files changed, 120 insertions(+), 21 deletions(-) diff --git a/__tests__/agent-pg-backup-pull.test.js b/__tests__/agent-pg-backup-pull.test.js index ccc7fc5..e287398 100644 --- a/__tests__/agent-pg-backup-pull.test.js +++ b/__tests__/agent-pg-backup-pull.test.js @@ -41,6 +41,28 @@ echo x > "$last/${TODAY}/maximus_api.dump.age" echo x > "$last/${TODAY}/globals.sql.age" `, ); + // Fake ssh: what rrsync does on the VPS, minus the restriction. Skips the + // options, drops the host, and runs the remote rsync command inside + // $FAKE_OUTBOX. Only reached by the real-rsync case; the fake rsync above + // never calls it, but the script requires ssh on PATH (none in node:22-alpine). + writeExecutable( + path.join(bin, "ssh"), + `#!/bin/sh +while [ $# -gt 0 ]; do + case "$1" in + -p|-i|-o|-l) shift 2 ;; + -*) shift ;; + *) shift; break ;; + esac +done +cd "$FAKE_OUTBOX" && exec "$@" +`, + ); + // No flock in the CI's busybox: a pass-through stand-in, only where the host + // has none (no case here races two runs). + if (!["/usr/bin/flock", "/bin/flock"].some((f) => fs.existsSync(f))) { + writeExecutable(path.join(bin, "flock"), "#!/bin/sh\nexit 0\n"); + } // Fake curl: records argv, the header file it was handed and the body. writeExecutable( path.join(bin, "curl"), @@ -100,6 +122,10 @@ describe("pull", () => { const argv = read("rsync.argv").split("\n"); expect(argv).toContain("--include=*.age"); expect(argv).toContain("--exclude=*"); + // rsync keeps the first matching rule: without `--include=*/` first, the + // dated directories hit `--exclude=*` and nothing crosses, silently. + expect(argv.indexOf("--include=*/")).toBeGreaterThan(-1); + expect(argv.indexOf("--include=*/")).toBeLessThan(argv.indexOf("--exclude=*")); expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*")); expect(argv).not.toContain("--delete"); expect(argv).toContain("pgbackup@100.86.30.21:"); @@ -130,6 +156,43 @@ describe("pull", () => { expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, files: 0, error: "pull_failed" }); }); + test("logs a warning when the newest directory is not today's", () => { + fs.writeFileSync( + path.join(bin, "rsync"), + `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/2026-10-09"\necho x > "$last/2026-10-09/a.age"\n`, + ); + const { status, output } = run(); + expect(status).toBe(0); + expect(output).toContain("latest=2026-10-09 files=1"); + expect(output).toContain(`warning: no directory for ${TODAY}, latest is 2026-10-09`); + }); + + test("no warning when today's directory is there", () => { + const { output } = run(); + expect(output).toContain(`latest=${TODAY} files=2`); + expect(output).not.toContain("warning"); + }); + + // The real rsync with the script's own filter, against a local outbox through + // the fake ssh: only .age files land, inside their dated directory. Skipped + // where rsync is absent (the CI image); the argv order test above runs there. + test.skipIf(!["/usr/bin/rsync", "/bin/rsync"].some((f) => fs.existsSync(f)))( + "with the real rsync, only .age files cross, inside their dated directory", + () => { + fs.rmSync(path.join(bin, "rsync")); + const outbox = path.join(tmp, "outbox"); + fs.mkdirSync(path.join(outbox, TODAY), { recursive: true }); + fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump.age"), "cipher"); + fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump"), "clear"); + fs.writeFileSync(path.join(outbox, "stray.sql"), "clear"); + const { status, output, dest } = run({ extraEnv: { FAKE_OUTBOX: outbox } }); + expect(status).toBe(0); + expect(fs.readdirSync(dest)).toEqual([TODAY]); + expect(fs.readdirSync(path.join(dest, TODAY))).toEqual(["maximus_api.dump.age"]); + expect(output).toContain(`latest=${TODAY} files=1`); + }, + ); + test("missing ssh key is an install problem: exit 1, nothing pulled", () => { fs.rmSync(path.join(tmp, "key")); const { status } = run(); @@ -168,26 +231,29 @@ describe("rotation", () => { test("keeps 30 dated directories, today included, and ignores other names", () => { const dest = path.join(tmp, "dest"); for (const d of ["2026-09-11", "2026-09-12", "notes"]) fs.mkdirSync(path.join(dest, d), { recursive: true }); - const { status } = run(); + const { status, output } = run(); expect(status).toBe(0); + // The newest of several dated directories is the one counted. + expect(output).toContain(`latest=${TODAY} files=2`); expect(fs.existsSync(path.join(dest, "2026-09-11"))).toBe(false); expect(fs.existsSync(path.join(dest, "2026-09-12"))).toBe(true); expect(fs.existsSync(path.join(dest, "notes"))).toBe(true); }); + // The failure comes from a fake rm, not from file modes: the CI runs as + // root, which ignores a read-only directory. test("a directory that cannot be removed fails loudly and reports rotation_failed", () => { const dest = path.join(tmp, "dest"); - const old = path.join(dest, "2026-09-01"); - fs.mkdirSync(path.join(old, "locked"), { recursive: true }); - fs.writeFileSync(path.join(old, "locked", "f.age"), "x"); - fs.chmodSync(path.join(old, "locked"), 0o500); - try { - const { status } = run({ envFile: writeEnv() }); - expect(status).toBe(1); - expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" }); - } finally { - fs.chmodSync(path.join(old, "locked"), 0o700); - } + fs.mkdirSync(path.join(dest, "2026-09-01"), { recursive: true }); + writeExecutable( + path.join(bin, "rm"), + `#!/bin/sh\ncase "$*" in *2026-09-01*) exit 1 ;; esac\nexec /bin/rm "$@"\n`, + ); + const { status, output } = run({ envFile: writeEnv() }); + expect(status).toBe(1); + expect(output).toContain("rotation: could not remove 2026-09-01"); + expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" }); + expect(fs.existsSync(path.join(dest, "2026-09-01"))).toBe(true); }); }); diff --git a/agent/README.md b/agent/README.md index 2b99b52..ee2a449 100644 --- a/agent/README.md +++ b/agent/README.md @@ -427,6 +427,15 @@ journalctl --user -t pg-backup-pull --since -1h systemctl --user list-timers maximus-pg-backup-pull.timer ``` +A pass whose newest dated directory is not today's (UTC) logs +`warning: no directory for , latest is `: the VPS stopped +producing, or the pass ran before its 03:00 UTC run. The report still says +`ok` with the newest directory's count; freshness is judged on the VPS side +by `GET /backups`. + +The script sticks to what busybox offers (no `find -printf`, no +`date -d "... days"`): the CI runs its tests in `node:22-alpine`. + ### Exit codes | Code | Meaning | diff --git a/agent/pg-backup-pull/run-pg-backup-pull.sh b/agent/pg-backup-pull/run-pg-backup-pull.sh index d2d949d..c4cc2c0 100755 --- a/agent/pg-backup-pull/run-pg-backup-pull.sh +++ b/agent/pg-backup-pull/run-pg-backup-pull.sh @@ -113,25 +113,49 @@ if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --partial-dir=.rsync-partial \ fi chmod -R go-rwx "$DEST" -latest=$(find "$DEST" -mindepth 1 -maxdepth 1 -type d -name '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' -printf '%f\n' | sort | tail -1) +# Plain shell over the glob, no `find -printf` nor `date -d "... days"`: both +# are GNU-only, and the CI runs this script under busybox (node:22-alpine). +is_dated() { + case "$1" in + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) return 0 ;; + *) return 1 ;; + esac +} +# YYYY-MM-DD as a YYYYMMDD integer: `[ a \< b ]` is not POSIX, and /bin/sh is dash. +day_number() { echo "$1" | tr -d -; } + +latest="" +for dir in "$DEST"/*/; do + [ -d "$dir" ] || continue + name=$(basename "$dir") + is_dated "$name" || continue + if [ -z "$latest" ] || [ "$(day_number "$name")" -gt "$(day_number "$latest")" ]; then + latest="$name" + fi +done files=0 if [ -n "$latest" ]; then - files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l) + files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l | tr -d ' ') fi log "pull ok latest=${latest:-none} files=$files" +# The report counts the newest local directory: if the VPS stops producing, a +# pass still says ok on an old one. GET /backups judges freshness on the VPS +# side; this line names the gap in the journal. +if [ "${latest:-}" != "$TODAY" ]; then + log "warning: no directory for $TODAY, latest is ${latest:-none}" +fi # --- Rotation (fails loudly) -------------------------------------------------- -cutoff=$(date -u -d "$TODAY -$((RETENTION_DAYS - 1)) days" +%F) +# Epoch arithmetic: `date -d YYYY-MM-DD` and `date -d @N` are read by GNU date +# and busybox alike. +today_epoch=$(date -u -d "$TODAY" +%s) +cutoff=$(date -u -d "@$((today_epoch - (RETENTION_DAYS - 1) * 86400))" +%F) rotation_ok=true for dir in "$DEST"/*/; do [ -d "$dir" ] || continue name=$(basename "$dir") - case "$name" in - [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; - *) continue ;; - esac - # Compared as YYYYMMDD integers: `[ a \< b ]` is not POSIX, and /bin/sh is dash. - if [ "$(echo "$name" | tr -d -)" -lt "$(echo "$cutoff" | tr -d -)" ]; then + is_dated "$name" || continue + if [ "$(day_number "$name")" -lt "$(day_number "$cutoff")" ]; then if rm -rf -- "$dir"; then log "rotation: removed $name" else -- 2.45.2 From f81a80e3257a325496a5200c48ae5d58154014ab Mon Sep 17 00:00:00 2001 From: le king fu Date: Sat, 10 Oct 2026 19:03:48 -0400 Subject: [PATCH 3/4] fix(agent): anchor the backup rotation on the newest backup, not today The rotation cutoff was TODAY minus 29 days. When the VPS stops producing (dead cron, failing pg_dump) or this machine's clock jumps ahead, the last off-site copy aged out after 30 days, the pass exited 0 and the report said ok on a directory it had just deleted. The cutoff now comes from the newest dated directory held here, which is never removed. Review suggestions: - --prune-empty-dirs: a dated directory with no .age file is not created, and one found empty logs a warning; - files counts the dated directory only (-maxdepth 1), not partial files left in .rsync-partial/; - an EXIT trap removes the header file holding the ingest token, with TERM/INT/HUP turned into an exit so a stop by systemd runs it. Refs maximus/la-compagnie-maximus#220 Co-Authored-By: Claude Opus 5.5 --- __tests__/agent-pg-backup-pull.test.js | 72 +++++++++++++++++++++- agent/README.md | 16 +++-- agent/pg-backup-pull/run-pg-backup-pull.sh | 33 ++++++++-- 3 files changed, 110 insertions(+), 11 deletions(-) diff --git a/__tests__/agent-pg-backup-pull.test.js b/__tests__/agent-pg-backup-pull.test.js index e287398..1f00adb 100644 --- a/__tests__/agent-pg-backup-pull.test.js +++ b/__tests__/agent-pg-backup-pull.test.js @@ -128,6 +128,7 @@ describe("pull", () => { expect(argv.indexOf("--include=*/")).toBeLessThan(argv.indexOf("--exclude=*")); expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*")); expect(argv).not.toContain("--delete"); + expect(argv).toContain("--prune-empty-dirs"); expect(argv).toContain("pgbackup@100.86.30.21:"); const ssh = argv.find((a) => a.startsWith("ssh ")); expect(ssh).toContain("-p 2222"); @@ -167,6 +168,24 @@ describe("pull", () => { expect(output).toContain(`warning: no directory for ${TODAY}, latest is 2026-10-09`); }); + test("a dated directory without any .age file logs a warning", () => { + fs.writeFileSync(path.join(bin, "rsync"), `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/${TODAY}"\n`); + const { status, output } = run(); + expect(status).toBe(0); + expect(output).toContain(`latest=${TODAY} files=0`); + expect(output).toContain(`warning: no .age file in ${TODAY}`); + }); + + test("a partial file left by rsync is not counted", () => { + fs.writeFileSync( + path.join(bin, "rsync"), + `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/${TODAY}/.rsync-partial"\n` + + `echo x > "$last/${TODAY}/a.age"\necho x > "$last/${TODAY}/.rsync-partial/b.age"\n`, + ); + const { output } = run(); + expect(output).toContain(`latest=${TODAY} files=1`); + }); + test("no warning when today's directory is there", () => { const { output } = run(); expect(output).toContain(`latest=${TODAY} files=2`); @@ -174,10 +193,11 @@ describe("pull", () => { }); // The real rsync with the script's own filter, against a local outbox through - // the fake ssh: only .age files land, inside their dated directory. Skipped + // the fake ssh: only .age files land, inside their dated directory, and a + // dated directory holding no .age file is not created. Skipped // where rsync is absent (the CI image); the argv order test above runs there. test.skipIf(!["/usr/bin/rsync", "/bin/rsync"].some((f) => fs.existsSync(f)))( - "with the real rsync, only .age files cross, inside their dated directory", + "with the real rsync, only .age files cross, inside their dated directory, never an empty one", () => { fs.rmSync(path.join(bin, "rsync")); const outbox = path.join(tmp, "outbox"); @@ -185,6 +205,8 @@ describe("pull", () => { fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump.age"), "cipher"); fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump"), "clear"); fs.writeFileSync(path.join(outbox, "stray.sql"), "clear"); + fs.mkdirSync(path.join(outbox, "2026-10-12")); + fs.writeFileSync(path.join(outbox, "2026-10-12", "maximus_api.dump"), "clear"); const { status, output, dest } = run({ extraEnv: { FAKE_OUTBOX: outbox } }); expect(status).toBe(0); expect(fs.readdirSync(dest)).toEqual([TODAY]); @@ -220,6 +242,32 @@ describe("report", () => { expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header"); }); + // systemd stops a unit with SIGTERM. The fake curl sends it to the script + // while the header file exists; the trap must still remove it. + test("a TERM while the report is in flight still removes the header file", () => { + writeExecutable( + path.join(bin, "curl"), + `#!/bin/sh +cat > /dev/null +# Climb from the parent (the script, or a subshell of it, same cmdline) while +# the parent is still the script: the last one is the script itself. +ppid_of() { sed 's/.*) //' "/proc/$1/stat" | cut -d' ' -f2; } +script=$PPID +while parent=$(ppid_of "$script") && tr '\\0' ' ' < "/proc/$parent/cmdline" | grep -q run-pg-backup-pull; do + script=$parent +done +[ -f "$FAKE_DIR/state/auth.header" ] && echo present > "$FAKE_DIR/header.seen" +kill -TERM "$script" +sleep 1 +printf 204 +`, + ); + const { status } = run({ envFile: writeEnv() }); + expect(read("header.seen")).toBe("present\n"); + expect(status).toBe(143); + expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header"); + }); + test("a non-2xx answer exits 3", () => { const { status, output } = run({ envFile: writeEnv(), extraEnv: { FAKE_CURL_STATUS: "401" } }); expect(status).toBe(3); @@ -240,6 +288,26 @@ describe("rotation", () => { expect(fs.existsSync(path.join(dest, "notes"))).toBe(true); }); + // The VPS stopped producing on 2026-09-01 (dead cron, failing pg_dump): its + // outbox yields nothing new. Anchored on today, the rotation would delete the + // last off-site copy; anchored on the newest backup, it keeps it and the 29 + // days before, and only removes what is older. + test("never removes the newest backup, however old: the window ends at the newest directory", () => { + const dest = path.join(tmp, "dest"); + for (const d of ["2026-08-02", "2026-08-03", "2026-09-01"]) { + fs.mkdirSync(path.join(dest, d), { recursive: true }); + fs.writeFileSync(path.join(dest, d, "a.age"), "x"); + } + fs.writeFileSync(path.join(bin, "rsync"), "#!/bin/sh\nexit 0\n"); + const { status, output } = run({ envFile: writeEnv() }); + expect(status).toBe(0); + expect(output).toContain("latest=2026-09-01 files=1"); + expect(fs.existsSync(path.join(dest, "2026-09-01", "a.age"))).toBe(true); + expect(fs.existsSync(path.join(dest, "2026-08-03"))).toBe(true); + expect(fs.existsSync(path.join(dest, "2026-08-02"))).toBe(false); + expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: true, files: 1 }); + }); + // The failure comes from a fake rm, not from file modes: the CI runs as // root, which ignores a read-only directory. test("a directory that cannot be removed fails loudly and reports rotation_failed", () => { diff --git a/agent/README.md b/agent/README.md index ee2a449..3fcc4b5 100644 --- a/agent/README.md +++ b/agent/README.md @@ -417,7 +417,8 @@ BACKUPS_INGEST_TOKEN= ``` The token reaches curl through a header file created `0600` and removed right -after, never through argv. +after, never through argv. An `EXIT` trap removes it too when the run is +stopped mid-report (systemd's `SIGTERM` is turned into an exit first). ### Check @@ -429,9 +430,16 @@ systemctl --user list-timers maximus-pg-backup-pull.timer A pass whose newest dated directory is not today's (UTC) logs `warning: no directory for , latest is `: the VPS stopped -producing, or the pass ran before its 03:00 UTC run. The report still says -`ok` with the newest directory's count; freshness is judged on the VPS side -by `GET /backups`. +producing, or the pass ran before its 03:00 UTC run. A dated directory with no +`.age` file is never created (`--prune-empty-dirs`), and one found empty logs +`warning: no .age file in `. The report still says `ok` with the newest +directory's count (partial files in `.rsync-partial/` not counted); freshness +is judged on the VPS side by `GET /backups`. + +**Rotation is anchored on the newest backup held here, not on today's date.** +It keeps that directory and the 29 days before it. If the VPS stops producing +(dead cron, failing `pg_dump`) or this machine's clock jumps ahead, the last +off-site copy stays put instead of aging out after 30 days. The script sticks to what busybox offers (no `find -printf`, no `date -d "... days"`): the CI runs its tests in `node:22-alpine`. diff --git a/agent/pg-backup-pull/run-pg-backup-pull.sh b/agent/pg-backup-pull/run-pg-backup-pull.sh index c4cc2c0..8e06838 100755 --- a/agent/pg-backup-pull/run-pg-backup-pull.sh +++ b/agent/pg-backup-pull/run-pg-backup-pull.sh @@ -57,6 +57,16 @@ if ! flock -n 9; then exit 0 fi +# The ingest token sits in this file for the length of one curl call. The trap +# removes it on any exit, a kill by systemd included: the signal traps turn +# TERM/INT/HUP into a plain exit, which runs the EXIT trap (dash does not run +# it on a signal by itself). +header_file="$STATE_DIR/auth.header" +trap 'rm -f "$header_file"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + # The env file is optional until the ingest route is deployed: without a token, # the pull still runs and only the report to the health API is skipped. BACKUPS_API_URL="" @@ -87,7 +97,6 @@ report() { body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s}' "$pulled_at" "$2" "$1") fi # The token goes through a header file, never through argv (visible in ps). - header_file="$STATE_DIR/auth.header" (umask 077 && printf 'Authorization: Bearer %s\n' "$BACKUPS_INGEST_TOKEN" > "$header_file") status=$(printf '%s' "$body" | curl -sS -o /dev/null -w '%{http_code}' \ --max-time "$POST_TIMEOUT_SECONDS" \ @@ -103,7 +112,9 @@ report() { # --- Pull --------------------------------------------------------------------- # Only directories and .age files cross: a misconfigured outbox can never land # a clear dump here. No --delete: the VPS keeps 7 days, this machine keeps 30. -if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --partial-dir=.rsync-partial \ +# --prune-empty-dirs: a dated directory with no .age file (encryption failed on +# the VPS) is not created here, so it cannot pass for today's backup. +if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --prune-empty-dirs --partial-dir=.rsync-partial \ --include='*/' --include='*.age' --exclude='*' \ -e "ssh -p $SSH_PORT -i $SSH_KEY -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" \ "$SOURCE" "$DEST/"; then @@ -135,7 +146,8 @@ for dir in "$DEST"/*/; do done files=0 if [ -n "$latest" ]; then - files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l | tr -d ' ') + # -maxdepth 1: a partial file left in .rsync-partial/ is not a backup. + files=$(find "$DEST/$latest" -maxdepth 1 -type f -name '*.age' | wc -l | tr -d ' ') fi log "pull ok latest=${latest:-none} files=$files" # The report counts the newest local directory: if the VPS stops producing, a @@ -144,13 +156,24 @@ log "pull ok latest=${latest:-none} files=$files" if [ "${latest:-}" != "$TODAY" ]; then log "warning: no directory for $TODAY, latest is ${latest:-none}" fi +if [ -n "$latest" ] && [ "$files" -eq 0 ]; then + log "warning: no .age file in $latest" +fi # --- Rotation (fails loudly) -------------------------------------------------- +# Anchored on the newest backup held here, never on today's date: when the VPS +# stops producing (dead cron, failing pg_dump), or this machine's clock jumps +# ahead, a TODAY-based cutoff would delete the last off-site copy. This keeps +# the 30 days before the newest backup, which itself is never removed. # Epoch arithmetic: `date -d YYYY-MM-DD` and `date -d @N` are read by GNU date # and busybox alike. -today_epoch=$(date -u -d "$TODAY" +%s) -cutoff=$(date -u -d "@$((today_epoch - (RETENTION_DAYS - 1) * 86400))" +%F) rotation_ok=true +cutoff="" +if [ -n "$latest" ]; then + latest_epoch=$(date -u -d "$latest" +%s) + cutoff=$(date -u -d "@$((latest_epoch - (RETENTION_DAYS - 1) * 86400))" +%F) +fi +# No latest means no dated directory: the loop below then compares nothing. for dir in "$DEST"/*/; do [ -d "$dir" ] || continue name=$(basename "$dir") -- 2.45.2 From 5208957ce91ca34e8947e84afbf4e0f81a1c33a5 Mon Sep 17 00:00:00 2001 From: le king fu Date: Sat, 10 Oct 2026 19:13:27 -0400 Subject: [PATCH 4/4] fix(agent): nothing the VPS sends deletes or alters a pulled backup Anchoring the rotation on the newest directory name let the VPS steer it: one future-dated directory (VPS clock off, or a VPS gone rogue) pushed the window forward and deleted every real backup, with exit 0 and an ok report. Two more inputs from the outbox could do harm: rsync -rt rewrote an already pulled .age whenever the VPS served another version, and an impossible dated name (2026-13-45) failed `date -d` under set -e, stopping every later pass for good. - latest is the newest valid dated directory not after today; the rotation keeps it and the 29 days before it; - a name is dated only if `date` gives it back unchanged (busybox turns 2026-02-30 into 2026-03-02); an invalid or future name is logged, never counted, never removed, and the pass goes on; - --ignore-existing: a pulled file is never rewritten. The VPS writes a .tmp and renames it, and the filter drops the .tmp. Refs maximus/la-compagnie-maximus#220 Co-Authored-By: Claude Opus 5.5 --- __tests__/agent-pg-backup-pull.test.js | 55 ++++++++++++++++++++-- agent/README.md | 19 ++++++-- agent/pg-backup-pull/run-pg-backup-pull.sh | 41 +++++++++++++--- 3 files changed, 101 insertions(+), 14 deletions(-) diff --git a/__tests__/agent-pg-backup-pull.test.js b/__tests__/agent-pg-backup-pull.test.js index 1f00adb..b0fada2 100644 --- a/__tests__/agent-pg-backup-pull.test.js +++ b/__tests__/agent-pg-backup-pull.test.js @@ -129,6 +129,7 @@ describe("pull", () => { expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*")); expect(argv).not.toContain("--delete"); expect(argv).toContain("--prune-empty-dirs"); + expect(argv).toContain("--ignore-existing"); expect(argv).toContain("pgbackup@100.86.30.21:"); const ssh = argv.find((a) => a.startsWith("ssh ")); expect(ssh).toContain("-p 2222"); @@ -197,7 +198,7 @@ describe("pull", () => { // dated directory holding no .age file is not created. Skipped // where rsync is absent (the CI image); the argv order test above runs there. test.skipIf(!["/usr/bin/rsync", "/bin/rsync"].some((f) => fs.existsSync(f)))( - "with the real rsync, only .age files cross, inside their dated directory, never an empty one", + "with the real rsync, only .age files cross, inside their dated directory, never an empty one, never over a pulled file", () => { fs.rmSync(path.join(bin, "rsync")); const outbox = path.join(tmp, "outbox"); @@ -207,10 +208,17 @@ describe("pull", () => { fs.writeFileSync(path.join(outbox, "stray.sql"), "clear"); fs.mkdirSync(path.join(outbox, "2026-10-12")); fs.writeFileSync(path.join(outbox, "2026-10-12", "maximus_api.dump"), "clear"); - const { status, output, dest } = run({ extraEnv: { FAKE_OUTBOX: outbox } }); + fs.mkdirSync(path.join(outbox, "2026-10-10")); + fs.writeFileSync(path.join(outbox, "2026-10-10", "globals.sql.age"), "garbage"); + const dest = path.join(tmp, "dest"); + fs.mkdirSync(path.join(dest, "2026-10-10"), { recursive: true }); + fs.writeFileSync(path.join(dest, "2026-10-10", "globals.sql.age"), "good-backup"); + const { status, output } = run({ extraEnv: { FAKE_OUTBOX: outbox } }); expect(status).toBe(0); - expect(fs.readdirSync(dest)).toEqual([TODAY]); + expect(fs.readdirSync(dest).sort()).toEqual(["2026-10-10", TODAY]); expect(fs.readdirSync(path.join(dest, TODAY))).toEqual(["maximus_api.dump.age"]); + // A file already pulled is never rewritten by what the VPS serves later. + expect(fs.readFileSync(path.join(dest, "2026-10-10", "globals.sql.age"), "utf8")).toBe("good-backup"); expect(output).toContain(`latest=${TODAY} files=1`); }, ); @@ -308,6 +316,47 @@ describe("rotation", () => { expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: true, files: 1 }); }); + // Nothing the VPS sends may delete a copy already here. A future-dated + // directory (VPS clock off, or a VPS gone rogue) would, as the newest name, + // push the rotation window forward over every real backup. + test("a future-dated directory from the VPS removes nothing and is not counted", () => { + const dest = path.join(tmp, "dest"); + for (const d of ["2026-09-20", "2026-10-01", "2026-10-10"]) { + fs.mkdirSync(path.join(dest, d), { recursive: true }); + fs.writeFileSync(path.join(dest, d, "a.age"), "x"); + } + fs.writeFileSync( + path.join(bin, "rsync"), + `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/2099-01-01"\necho z > "$last/2099-01-01/z.age"\n`, + ); + const { status, output } = run({ envFile: writeEnv() }); + expect(status).toBe(0); + expect(fs.readdirSync(dest).sort()).toEqual(["2026-09-20", "2026-10-01", "2026-10-10", "2099-01-01"]); + expect(output).toContain(`warning: ignoring 2099-01-01, after ${TODAY}`); + expect(output).toContain("latest=2026-10-10 files=1"); + expect(output).not.toContain("rotation: removed"); + }); + + // A name of the dated shape that is no date (2026-13-45; busybox would turn + // 2026-02-30 into 2026-03-02) is ignored: the pass goes on, the directory + // is neither counted nor removed. + test("an impossible dated name is ignored, never stops the pass, never removed", () => { + const dest = path.join(tmp, "dest"); + for (const d of ["2026-13-45", "2026-02-30", "2026-01-01"]) { + fs.mkdirSync(path.join(dest, d), { recursive: true }); + fs.writeFileSync(path.join(dest, d, "z.age"), "z"); + } + const { status, output } = run({ envFile: writeEnv() }); + expect(status).toBe(0); + expect(output).toContain("warning: ignoring 2026-13-45, not a valid date"); + expect(output).toContain("warning: ignoring 2026-02-30, not a valid date"); + expect(output).toContain(`latest=${TODAY} files=2`); + expect(output).toContain("rotation: removed 2026-01-01"); + expect(fs.existsSync(path.join(dest, "2026-13-45"))).toBe(true); + expect(fs.existsSync(path.join(dest, "2026-02-30"))).toBe(true); + expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: true, files: 2 }); + }); + // The failure comes from a fake rm, not from file modes: the CI runs as // root, which ignores a read-only directory. test("a directory that cannot be removed fails loudly and reports rotation_failed", () => { diff --git a/agent/README.md b/agent/README.md index 3fcc4b5..e1c9b9d 100644 --- a/agent/README.md +++ b/agent/README.md @@ -436,10 +436,21 @@ producing, or the pass ran before its 03:00 UTC run. A dated directory with no directory's count (partial files in `.rsync-partial/` not counted); freshness is judged on the VPS side by `GET /backups`. -**Rotation is anchored on the newest backup held here, not on today's date.** -It keeps that directory and the 29 days before it. If the VPS stops producing -(dead cron, failing `pg_dump`) or this machine's clock jumps ahead, the last -off-site copy stays put instead of aging out after 30 days. +**Nothing the VPS sends may delete or alter a copy already here.** + +- **No overwrite.** A file already pulled is never rewritten + (`--ignore-existing`): the VPS writes each dump to a `.tmp` and renames it, + so what lands is whole. +- **Directory names are checked.** A name is dated only if `date` gives it + back unchanged: `2026-13-45` or `2026-02-30` is logged and ignored, never + counted, never removed, and the pass goes on. A dated name after today + (VPS clock off, or a VPS gone rogue) is logged and ignored the same way. +- **Rotation is anchored on the newest valid backup not after today.** It + keeps that directory and the 29 days before it. Not on today alone: if the + VPS stops producing (dead cron, failing `pg_dump`) or this machine's clock + jumps ahead, the last off-site copy stays put instead of aging out after + 30 days. Not on the newest name alone: a future-dated directory would push + the window over every real backup. The script sticks to what busybox offers (no `find -printf`, no `date -d "... days"`): the CI runs its tests in `node:22-alpine`. diff --git a/agent/pg-backup-pull/run-pg-backup-pull.sh b/agent/pg-backup-pull/run-pg-backup-pull.sh index 8e06838..5b51abf 100755 --- a/agent/pg-backup-pull/run-pg-backup-pull.sh +++ b/agent/pg-backup-pull/run-pg-backup-pull.sh @@ -114,7 +114,10 @@ report() { # a clear dump here. No --delete: the VPS keeps 7 days, this machine keeps 30. # --prune-empty-dirs: a dated directory with no .age file (encryption failed on # the VPS) is not created here, so it cannot pass for today's backup. -if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --prune-empty-dirs --partial-dir=.rsync-partial \ +# --ignore-existing: a file already pulled is never rewritten, whatever the VPS +# serves under its name later. The VPS writes each dump to a .tmp and renames +# it (the .tmp never passes the filter), so what lands here is already whole. +if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --prune-empty-dirs --ignore-existing --partial-dir=.rsync-partial \ --include='*/' --include='*.age' --exclude='*' \ -e "ssh -p $SSH_PORT -i $SSH_KEY -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" \ "$SOURCE" "$DEST/"; then @@ -124,22 +127,43 @@ if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --prune-empty-dirs --partial-dir fi chmod -R go-rwx "$DEST" +# Nothing the VPS sends may delete or alter a copy already here: directory +# names come from the outbox, so they are checked before they steer anything. +# # Plain shell over the glob, no `find -printf` nor `date -d "... days"`: both # are GNU-only, and the CI runs this script under busybox (node:22-alpine). +# +# A name is dated only if `date` gives it back unchanged: GNU date refuses +# 2026-13-45, busybox turns 2026-02-30 into 2026-03-02. Anything else is never +# picked as latest, never removed, and must not stop the pass. is_dated() { case "$1" in - [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) return 0 ;; + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) return 1 ;; esac + [ "$(date -u -d "$1" +%F 2>/dev/null)" = "$1" ] } # YYYY-MM-DD as a YYYYMMDD integer: `[ a \< b ]` is not POSIX, and /bin/sh is dash. day_number() { echo "$1" | tr -d -; } +# latest is the newest dated directory not after today: a future name (VPS +# clock off, or a VPS gone rogue) is kept but steers neither the report nor the +# rotation below. latest="" for dir in "$DEST"/*/; do [ -d "$dir" ] || continue name=$(basename "$dir") - is_dated "$name" || continue + # Only names of the dated shape are logged: they hold digits and dashes, so a + # name from the VPS cannot forge a journal line. + case "$name" in [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) continue ;; esac + if ! is_dated "$name"; then + log "warning: ignoring $name, not a valid date" + continue + fi + if [ "$(day_number "$name")" -gt "$(day_number "$TODAY")" ]; then + log "warning: ignoring $name, after $TODAY" + continue + fi if [ -z "$latest" ] || [ "$(day_number "$name")" -gt "$(day_number "$latest")" ]; then latest="$name" fi @@ -161,10 +185,13 @@ if [ -n "$latest" ] && [ "$files" -eq 0 ]; then fi # --- Rotation (fails loudly) -------------------------------------------------- -# Anchored on the newest backup held here, never on today's date: when the VPS -# stops producing (dead cron, failing pg_dump), or this machine's clock jumps -# ahead, a TODAY-based cutoff would delete the last off-site copy. This keeps -# the 30 days before the newest backup, which itself is never removed. +# Anchored on latest, the newest backup held here that is not after today. Not +# on today alone: when the VPS stops producing (dead cron, failing pg_dump), or +# this machine's clock jumps ahead, a today-based cutoff would delete the last +# off-site copy. Not on the newest name alone: a future-dated directory from +# the VPS would push the window forward and delete every real backup. This +# keeps latest and the 29 days before it; a future or invalid name is newer +# than the cutoff or not dated, so it is never removed. # Epoch arithmetic: `date -d YYYY-MM-DD` and `date -d @N` are read by GNU date # and busybox alike. rotation_ok=true -- 2.45.2