vps-health-api/__tests__/agent-pg-backup-pull.test.js

const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const { spawnSync } = require("node:child_process");

// Coverage for the PostgreSQL backup pull agent (la-compagnie-maximus#220).
//
// The script runs under a systemd user timer and talks to the VPS over
// rsync/ssh, then reports to POST /backups/pull. Both are replaced here by fake
// binaries on PATH, which record their argv so the tests can check what would
// have crossed the wire — in particular that the ingest token never appears in
// a command line nor in the script's own output (journald keeps it forever).

const SCRIPT = path.join(__dirname, "..", "agent", "pg-backup-pull", "run-pg-backup-pull.sh");
const TOKEN = "ingest-token-0123456789abcdef";
const TODAY = "2026-10-11";

let tmp;
let bin;

function writeExecutable(file, content) {
  fs.writeFileSync(file, content, { mode: 0o755 });
}

beforeEach(() => {
  tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pg-backup-pull-"));
  bin = path.join(tmp, "bin");
  fs.mkdirSync(bin);
  fs.writeFileSync(path.join(tmp, "key"), "fake key", { mode: 0o600 });
  fs.writeFileSync(path.join(tmp, "known_hosts"), "[100.86.30.21]:2222 ssh-ed25519 AAAA");

  // Fake rsync: records argv, then lands two .age files in today's directory.
  writeExecutable(
    path.join(bin, "rsync"),
    `#!/bin/sh
printf '%s\\n' "$@" > "$FAKE_DIR/rsync.argv"
[ -n "\${FAKE_RSYNC_FAIL:-}" ] && exit 23
for last; do :; done
mkdir -p "$last/${TODAY}"
echo x > "$last/${TODAY}/maximus_api.dump.age"
echo x > "$last/${TODAY}/globals.sql.age"
`,
  );
  // Fake ssh: what rrsync does on the VPS, minus the restriction. Skips the
  // options, drops the host, and runs the remote rsync command inside
  // $FAKE_OUTBOX. Only reached by the real-rsync case; the fake rsync above
  // never calls it, but the script requires ssh on PATH (none in node:22-alpine).
  writeExecutable(
    path.join(bin, "ssh"),
    `#!/bin/sh
while [ $# -gt 0 ]; do
  case "$1" in
    -p|-i|-o|-l) shift 2 ;;
    -*) shift ;;
    *) shift; break ;;
  esac
done
cd "$FAKE_OUTBOX" && exec "$@"
`,
  );
  // No flock in the CI's busybox: a pass-through stand-in, only where the host
  // has none (no case here races two runs).
  if (!["/usr/bin/flock", "/bin/flock"].some((f) => fs.existsSync(f))) {
    writeExecutable(path.join(bin, "flock"), "#!/bin/sh\nexit 0\n");
  }
  // Fake curl: records argv, the header file it was handed and the body.
  writeExecutable(
    path.join(bin, "curl"),
    `#!/bin/sh
printf '%s\\n' "$@" > "$FAKE_DIR/curl.argv"
prev=""
for a; do
  case "$prev" in -H) case "$a" in @*) cp "\${a#@}" "$FAKE_DIR/curl.header" ;; esac ;; esac
  prev="$a"
done
cat > "$FAKE_DIR/curl.body"
printf '%s' "\${FAKE_CURL_STATUS:-204}"
`,
  );
});

afterEach(() => {
  fs.rmSync(tmp, { recursive: true, force: true });
});

function run({ envFile, extraEnv = {} } = {}) {
  const dest = path.join(tmp, "dest");
  const result = spawnSync("sh", [SCRIPT], {
    encoding: "utf8",
    env: {
      PATH: `${bin}:/usr/bin:/bin`,
      HOME: tmp,
      FAKE_DIR: tmp,
      PG_BACKUP_PULL_ENV_FILE: envFile ?? path.join(tmp, "absent.env"),
      PG_BACKUP_PULL_DEST: dest,
      PG_BACKUP_PULL_STATE_DIR: path.join(tmp, "state"),
      PG_BACKUP_PULL_SSH_KEY: path.join(tmp, "key"),
      PG_BACKUP_PULL_KNOWN_HOSTS: path.join(tmp, "known_hosts"),
      PG_BACKUP_PULL_TODAY: TODAY,
      ...extraEnv,
    },
  });
  return { ...result, output: result.stdout + result.stderr, dest };
}

function writeEnv() {
  const file = path.join(tmp, "agent.env");
  fs.writeFileSync(
    file,
    `BACKUPS_API_URL=https://health.example.test\nBACKUPS_INGEST_TOKEN=${TOKEN}\n`,
    { mode: 0o600 },
  );
  return file;
}

const read = (name) => fs.readFileSync(path.join(tmp, name), "utf8");

describe("pull", () => {
  test("pulls over port 2222 with a pinned host key, .age files only, never --delete", () => {
    const { status } = run();
    expect(status).toBe(0);
    const argv = read("rsync.argv").split("\n");
    expect(argv).toContain("--include=*.age");
    expect(argv).toContain("--exclude=*");
    // rsync keeps the first matching rule: without `--include=*/` first, the
    // dated directories hit `--exclude=*` and nothing crosses, silently.
    expect(argv.indexOf("--include=*/")).toBeGreaterThan(-1);
    expect(argv.indexOf("--include=*/")).toBeLessThan(argv.indexOf("--exclude=*"));
    expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*"));
    expect(argv).not.toContain("--delete");
    expect(argv).toContain("--prune-empty-dirs");
    expect(argv).toContain("--ignore-existing");
    expect(argv).toContain("pgbackup@100.86.30.21:");
    const ssh = argv.find((a) => a.startsWith("ssh "));
    expect(ssh).toContain("-p 2222");
    expect(ssh).toContain("StrictHostKeyChecking=yes");
    expect(ssh).toContain("BatchMode=yes");
    expect(ssh).toContain("IdentitiesOnly=yes");
  });

  test("without a token, still pulls and exits 0, saying the report was skipped", () => {
    const { status, output, dest } = run();
    expect(status).toBe(0);
    expect(output).toContain("report skipped");
    expect(fs.existsSync(path.join(dest, TODAY, "maximus_api.dump.age"))).toBe(true);
    expect(fs.existsSync(path.join(tmp, "curl.argv"))).toBe(false);
  });

  test("keeps the destination private", () => {
    const { dest } = run();
    expect(fs.statSync(dest).mode & 0o777).toBe(0o700);
    expect(fs.statSync(path.join(dest, TODAY, "globals.sql.age")).mode & 0o077).toBe(0);
  });

  test("a failed pull exits 2 and reports pull_failed", () => {
    const { status } = run({ envFile: writeEnv(), extraEnv: { FAKE_RSYNC_FAIL: "1" } });
    expect(status).toBe(2);
    expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, files: 0, error: "pull_failed" });
  });

  test("logs a warning when the newest directory is not today's", () => {
    fs.writeFileSync(
      path.join(bin, "rsync"),
      `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/2026-10-09"\necho x > "$last/2026-10-09/a.age"\n`,
    );
    const { status, output } = run();
    expect(status).toBe(0);
    expect(output).toContain("latest=2026-10-09 files=1");
    expect(output).toContain(`warning: no directory for ${TODAY}, latest is 2026-10-09`);
  });

  test("a dated directory without any .age file logs a warning", () => {
    fs.writeFileSync(path.join(bin, "rsync"), `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/${TODAY}"\n`);
    const { status, output } = run();
    expect(status).toBe(0);
    expect(output).toContain(`latest=${TODAY} files=0`);
    expect(output).toContain(`warning: no .age file in ${TODAY}`);
  });

  test("a partial file left by rsync is not counted", () => {
    fs.writeFileSync(
      path.join(bin, "rsync"),
      `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/${TODAY}/.rsync-partial"\n` +
        `echo x > "$last/${TODAY}/a.age"\necho x > "$last/${TODAY}/.rsync-partial/b.age"\n`,
    );
    const { output } = run();
    expect(output).toContain(`latest=${TODAY} files=1`);
  });

  test("no warning when today's directory is there", () => {
    const { output } = run();
    expect(output).toContain(`latest=${TODAY} files=2`);
    expect(output).not.toContain("warning");
  });

  // The real rsync with the script's own filter, against a local outbox through
  // the fake ssh: only .age files land, inside their dated directory, and a
  // dated directory holding no .age file is not created. Skipped
  // where rsync is absent (the CI image); the argv order test above runs there.
  test.skipIf(!["/usr/bin/rsync", "/bin/rsync"].some((f) => fs.existsSync(f)))(
    "with the real rsync, only .age files cross, inside their dated directory, never an empty one, never over a pulled file",
    () => {
      fs.rmSync(path.join(bin, "rsync"));
      const outbox = path.join(tmp, "outbox");
      fs.mkdirSync(path.join(outbox, TODAY), { recursive: true });
      fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump.age"), "cipher");
      fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump"), "clear");
      fs.writeFileSync(path.join(outbox, "stray.sql"), "clear");
      fs.mkdirSync(path.join(outbox, "2026-10-12"));
      fs.writeFileSync(path.join(outbox, "2026-10-12", "maximus_api.dump"), "clear");
      fs.mkdirSync(path.join(outbox, "2026-10-10"));
      fs.writeFileSync(path.join(outbox, "2026-10-10", "globals.sql.age"), "garbage");
      const dest = path.join(tmp, "dest");
      fs.mkdirSync(path.join(dest, "2026-10-10"), { recursive: true });
      fs.writeFileSync(path.join(dest, "2026-10-10", "globals.sql.age"), "good-backup");
      const { status, output } = run({ extraEnv: { FAKE_OUTBOX: outbox } });
      expect(status).toBe(0);
      expect(fs.readdirSync(dest).sort()).toEqual(["2026-10-10", TODAY]);
      expect(fs.readdirSync(path.join(dest, TODAY))).toEqual(["maximus_api.dump.age"]);
      // A file already pulled is never rewritten by what the VPS serves later.
      expect(fs.readFileSync(path.join(dest, "2026-10-10", "globals.sql.age"), "utf8")).toBe("good-backup");
      expect(output).toContain(`latest=${TODAY} files=1`);
    },
  );

  test("missing ssh key is an install problem: exit 1, nothing pulled", () => {
    fs.rmSync(path.join(tmp, "key"));
    const { status } = run();
    expect(status).toBe(1);
    expect(fs.existsSync(path.join(tmp, "rsync.argv"))).toBe(false);
  });
});

describe("report", () => {
  test("posts pulledAt, files and ok to /backups/pull", () => {
    const { status } = run({ envFile: writeEnv() });
    expect(status).toBe(0);
    const body = JSON.parse(read("curl.body"));
    expect(Object.keys(body).sort()).toEqual(["files", "ok", "pulledAt"]);
    expect(body).toMatchObject({ ok: true, files: 2 });
    expect(body.pulledAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/);
    expect(read("curl.argv").split("\n")).toContain("https://health.example.test/backups/pull");
  });

  test("the token travels in a header file, never in argv nor in the output", () => {
    const { output } = run({ envFile: writeEnv() });
    expect(read("curl.header")).toBe(`Authorization: Bearer ${TOKEN}\n`);
    expect(read("curl.argv")).not.toContain(TOKEN.slice(0, 12));
    expect(output).not.toContain(TOKEN.slice(0, 12));
    expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header");
  });

  // systemd stops a unit with SIGTERM. The fake curl sends it to the script
  // while the header file exists; the trap must still remove it.
  test("a TERM while the report is in flight still removes the header file", () => {
    writeExecutable(
      path.join(bin, "curl"),
      `#!/bin/sh
cat > /dev/null
# Climb from the parent (the script, or a subshell of it, same cmdline) while
# the parent is still the script: the last one is the script itself.
ppid_of() { sed 's/.*) //' "/proc/$1/stat" | cut -d' ' -f2; }
script=$PPID
while parent=$(ppid_of "$script") && tr '\\0' ' ' < "/proc/$parent/cmdline" | grep -q run-pg-backup-pull; do
  script=$parent
done
[ -f "$FAKE_DIR/state/auth.header" ] && echo present > "$FAKE_DIR/header.seen"
kill -TERM "$script"
sleep 1
printf 204
`,
    );
    const { status } = run({ envFile: writeEnv() });
    expect(read("header.seen")).toBe("present\n");
    expect(status).toBe(143);
    expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header");
  });

  test("a non-2xx answer exits 3", () => {
    const { status, output } = run({ envFile: writeEnv(), extraEnv: { FAKE_CURL_STATUS: "401" } });
    expect(status).toBe(3);
    expect(output).toContain("report failed status=401");
  });
});

describe("rotation", () => {
  test("keeps 30 dated directories, today included, and ignores other names", () => {
    const dest = path.join(tmp, "dest");
    for (const d of ["2026-09-11", "2026-09-12", "notes"]) fs.mkdirSync(path.join(dest, d), { recursive: true });
    const { status, output } = run();
    expect(status).toBe(0);
    // The newest of several dated directories is the one counted.
    expect(output).toContain(`latest=${TODAY} files=2`);
    expect(fs.existsSync(path.join(dest, "2026-09-11"))).toBe(false);
    expect(fs.existsSync(path.join(dest, "2026-09-12"))).toBe(true);
    expect(fs.existsSync(path.join(dest, "notes"))).toBe(true);
  });

  // The VPS stopped producing on 2026-09-01 (dead cron, failing pg_dump): its
  // outbox yields nothing new. Anchored on today, the rotation would delete the
  // last off-site copy; anchored on the newest backup, it keeps it and the 29
  // days before, and only removes what is older.
  test("never removes the newest backup, however old: the window ends at the newest directory", () => {
    const dest = path.join(tmp, "dest");
    for (const d of ["2026-08-02", "2026-08-03", "2026-09-01"]) {
      fs.mkdirSync(path.join(dest, d), { recursive: true });
      fs.writeFileSync(path.join(dest, d, "a.age"), "x");
    }
    fs.writeFileSync(path.join(bin, "rsync"), "#!/bin/sh\nexit 0\n");
    const { status, output } = run({ envFile: writeEnv() });
    expect(status).toBe(0);
    expect(output).toContain("latest=2026-09-01 files=1");
    expect(fs.existsSync(path.join(dest, "2026-09-01", "a.age"))).toBe(true);
    expect(fs.existsSync(path.join(dest, "2026-08-03"))).toBe(true);
    expect(fs.existsSync(path.join(dest, "2026-08-02"))).toBe(false);
    expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: true, files: 1 });
  });

  // Nothing the VPS sends may delete a copy already here. A future-dated
  // directory (VPS clock off, or a VPS gone rogue) would, as the newest name,
  // push the rotation window forward over every real backup.
  test("a future-dated directory from the VPS removes nothing and is not counted", () => {
    const dest = path.join(tmp, "dest");
    for (const d of ["2026-09-20", "2026-10-01", "2026-10-10"]) {
      fs.mkdirSync(path.join(dest, d), { recursive: true });
      fs.writeFileSync(path.join(dest, d, "a.age"), "x");
    }
    fs.writeFileSync(
      path.join(bin, "rsync"),
      `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/2099-01-01"\necho z > "$last/2099-01-01/z.age"\n`,
    );
    const { status, output } = run({ envFile: writeEnv() });
    expect(status).toBe(0);
    expect(fs.readdirSync(dest).sort()).toEqual(["2026-09-20", "2026-10-01", "2026-10-10", "2099-01-01"]);
    expect(output).toContain(`warning: ignoring 2099-01-01, after ${TODAY}`);
    expect(output).toContain("latest=2026-10-10 files=1");
    expect(output).not.toContain("rotation: removed");
  });

  // A name of the dated shape that is no date (2026-13-45; busybox would turn
  // 2026-02-30 into 2026-03-02) is ignored: the pass goes on, the directory
  // is neither counted nor removed.
  test("an impossible dated name is ignored, never stops the pass, never removed", () => {
    const dest = path.join(tmp, "dest");
    for (const d of ["2026-13-45", "2026-02-30", "2026-01-01"]) {
      fs.mkdirSync(path.join(dest, d), { recursive: true });
      fs.writeFileSync(path.join(dest, d, "z.age"), "z");
    }
    const { status, output } = run({ envFile: writeEnv() });
    expect(status).toBe(0);
    expect(output).toContain("warning: ignoring 2026-13-45, not a valid date");
    expect(output).toContain("warning: ignoring 2026-02-30, not a valid date");
    expect(output).toContain(`latest=${TODAY} files=2`);
    expect(output).toContain("rotation: removed 2026-01-01");
    expect(fs.existsSync(path.join(dest, "2026-13-45"))).toBe(true);
    expect(fs.existsSync(path.join(dest, "2026-02-30"))).toBe(true);
    expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: true, files: 2 });
  });

  // The failure comes from a fake rm, not from file modes: the CI runs as
  // root, which ignores a read-only directory.
  test("a directory that cannot be removed fails loudly and reports rotation_failed", () => {
    const dest = path.join(tmp, "dest");
    fs.mkdirSync(path.join(dest, "2026-09-01"), { recursive: true });
    writeExecutable(
      path.join(bin, "rm"),
      `#!/bin/sh\ncase "$*" in *2026-09-01*) exit 1 ;; esac\nexec /bin/rm "$@"\n`,
    );
    const { status, output } = run({ envFile: writeEnv() });
    expect(status).toBe(1);
    expect(output).toContain("rotation: could not remove 2026-09-01");
    expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" });
    expect(fs.existsSync(path.join(dest, "2026-09-01"))).toBe(true);
  });
});

describe("the script itself", () => {
  test("never enables shell tracing — `set -x` would echo the token into journald", () => {
    const traced = fs
      .readFileSync(SCRIPT, "utf8")
      .split("\n")
      .filter((line) => !line.trim().startsWith("#"))
      .filter((line) => /\bset\b[^#\n]*-[a-z]*x/.test(line));
    expect(traced).toEqual([]);
  });

  test("is executable", () => {
    expect(fs.statSync(SCRIPT).mode & 0o111).toBeGreaterThan(0);
  });
});