fix(agent): run the backup pull under busybox, catch an empty filter
All checks were successful
CI / verification (pull_request) Successful in 31s
All checks were successful
CI / verification (pull_request) Successful in 31s
The CI runs the tests in node:22-alpine, where 9 of the 12 cases failed: no ssh on PATH, and busybox has neither `find -printf` nor `date -d "... days"`. The newest dated directory is now found by a loop over the glob, and the rotation cutoff by epoch arithmetic, which GNU date and busybox both read. The tests add a fake ssh, and a fake flock where the host has none. The rotation_failed case failed the pass through a read-only directory, which root ignores: it now uses a fake rm. Review suggestions: the rsync argv must put --include=*/ before --exclude=* (without it nothing crosses and the pass still says ok), and a case runs the real rsync through the fake ssh when rsync is present. A pass whose newest directory is not today's logs a warning. Refs maximus/la-compagnie-maximus#220 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
59b963220d
commit
b87b3d7253
3 changed files with 120 additions and 21 deletions
|
|
@ -41,6 +41,28 @@ echo x > "$last/${TODAY}/maximus_api.dump.age"
|
||||||
echo x > "$last/${TODAY}/globals.sql.age"
|
echo x > "$last/${TODAY}/globals.sql.age"
|
||||||
`,
|
`,
|
||||||
);
|
);
|
||||||
|
// Fake ssh: what rrsync does on the VPS, minus the restriction. Skips the
|
||||||
|
// options, drops the host, and runs the remote rsync command inside
|
||||||
|
// $FAKE_OUTBOX. Only reached by the real-rsync case; the fake rsync above
|
||||||
|
// never calls it, but the script requires ssh on PATH (none in node:22-alpine).
|
||||||
|
writeExecutable(
|
||||||
|
path.join(bin, "ssh"),
|
||||||
|
`#!/bin/sh
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
-p|-i|-o|-l) shift 2 ;;
|
||||||
|
-*) shift ;;
|
||||||
|
*) shift; break ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
cd "$FAKE_OUTBOX" && exec "$@"
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
// No flock in the CI's busybox: a pass-through stand-in, only where the host
|
||||||
|
// has none (no case here races two runs).
|
||||||
|
if (!["/usr/bin/flock", "/bin/flock"].some((f) => fs.existsSync(f))) {
|
||||||
|
writeExecutable(path.join(bin, "flock"), "#!/bin/sh\nexit 0\n");
|
||||||
|
}
|
||||||
// Fake curl: records argv, the header file it was handed and the body.
|
// Fake curl: records argv, the header file it was handed and the body.
|
||||||
writeExecutable(
|
writeExecutable(
|
||||||
path.join(bin, "curl"),
|
path.join(bin, "curl"),
|
||||||
|
|
@ -100,6 +122,10 @@ describe("pull", () => {
|
||||||
const argv = read("rsync.argv").split("\n");
|
const argv = read("rsync.argv").split("\n");
|
||||||
expect(argv).toContain("--include=*.age");
|
expect(argv).toContain("--include=*.age");
|
||||||
expect(argv).toContain("--exclude=*");
|
expect(argv).toContain("--exclude=*");
|
||||||
|
// rsync keeps the first matching rule: without `--include=*/` first, the
|
||||||
|
// dated directories hit `--exclude=*` and nothing crosses, silently.
|
||||||
|
expect(argv.indexOf("--include=*/")).toBeGreaterThan(-1);
|
||||||
|
expect(argv.indexOf("--include=*/")).toBeLessThan(argv.indexOf("--exclude=*"));
|
||||||
expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*"));
|
expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*"));
|
||||||
expect(argv).not.toContain("--delete");
|
expect(argv).not.toContain("--delete");
|
||||||
expect(argv).toContain("pgbackup@100.86.30.21:");
|
expect(argv).toContain("pgbackup@100.86.30.21:");
|
||||||
|
|
@ -130,6 +156,43 @@ describe("pull", () => {
|
||||||
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, files: 0, error: "pull_failed" });
|
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, files: 0, error: "pull_failed" });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("logs a warning when the newest directory is not today's", () => {
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(bin, "rsync"),
|
||||||
|
`#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/2026-10-09"\necho x > "$last/2026-10-09/a.age"\n`,
|
||||||
|
);
|
||||||
|
const { status, output } = run();
|
||||||
|
expect(status).toBe(0);
|
||||||
|
expect(output).toContain("latest=2026-10-09 files=1");
|
||||||
|
expect(output).toContain(`warning: no directory for ${TODAY}, latest is 2026-10-09`);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("no warning when today's directory is there", () => {
|
||||||
|
const { output } = run();
|
||||||
|
expect(output).toContain(`latest=${TODAY} files=2`);
|
||||||
|
expect(output).not.toContain("warning");
|
||||||
|
});
|
||||||
|
|
||||||
|
// The real rsync with the script's own filter, against a local outbox through
|
||||||
|
// the fake ssh: only .age files land, inside their dated directory. Skipped
|
||||||
|
// where rsync is absent (the CI image); the argv order test above runs there.
|
||||||
|
test.skipIf(!["/usr/bin/rsync", "/bin/rsync"].some((f) => fs.existsSync(f)))(
|
||||||
|
"with the real rsync, only .age files cross, inside their dated directory",
|
||||||
|
() => {
|
||||||
|
fs.rmSync(path.join(bin, "rsync"));
|
||||||
|
const outbox = path.join(tmp, "outbox");
|
||||||
|
fs.mkdirSync(path.join(outbox, TODAY), { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump.age"), "cipher");
|
||||||
|
fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump"), "clear");
|
||||||
|
fs.writeFileSync(path.join(outbox, "stray.sql"), "clear");
|
||||||
|
const { status, output, dest } = run({ extraEnv: { FAKE_OUTBOX: outbox } });
|
||||||
|
expect(status).toBe(0);
|
||||||
|
expect(fs.readdirSync(dest)).toEqual([TODAY]);
|
||||||
|
expect(fs.readdirSync(path.join(dest, TODAY))).toEqual(["maximus_api.dump.age"]);
|
||||||
|
expect(output).toContain(`latest=${TODAY} files=1`);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
test("missing ssh key is an install problem: exit 1, nothing pulled", () => {
|
test("missing ssh key is an install problem: exit 1, nothing pulled", () => {
|
||||||
fs.rmSync(path.join(tmp, "key"));
|
fs.rmSync(path.join(tmp, "key"));
|
||||||
const { status } = run();
|
const { status } = run();
|
||||||
|
|
@ -168,26 +231,29 @@ describe("rotation", () => {
|
||||||
test("keeps 30 dated directories, today included, and ignores other names", () => {
|
test("keeps 30 dated directories, today included, and ignores other names", () => {
|
||||||
const dest = path.join(tmp, "dest");
|
const dest = path.join(tmp, "dest");
|
||||||
for (const d of ["2026-09-11", "2026-09-12", "notes"]) fs.mkdirSync(path.join(dest, d), { recursive: true });
|
for (const d of ["2026-09-11", "2026-09-12", "notes"]) fs.mkdirSync(path.join(dest, d), { recursive: true });
|
||||||
const { status } = run();
|
const { status, output } = run();
|
||||||
expect(status).toBe(0);
|
expect(status).toBe(0);
|
||||||
|
// The newest of several dated directories is the one counted.
|
||||||
|
expect(output).toContain(`latest=${TODAY} files=2`);
|
||||||
expect(fs.existsSync(path.join(dest, "2026-09-11"))).toBe(false);
|
expect(fs.existsSync(path.join(dest, "2026-09-11"))).toBe(false);
|
||||||
expect(fs.existsSync(path.join(dest, "2026-09-12"))).toBe(true);
|
expect(fs.existsSync(path.join(dest, "2026-09-12"))).toBe(true);
|
||||||
expect(fs.existsSync(path.join(dest, "notes"))).toBe(true);
|
expect(fs.existsSync(path.join(dest, "notes"))).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The failure comes from a fake rm, not from file modes: the CI runs as
|
||||||
|
// root, which ignores a read-only directory.
|
||||||
test("a directory that cannot be removed fails loudly and reports rotation_failed", () => {
|
test("a directory that cannot be removed fails loudly and reports rotation_failed", () => {
|
||||||
const dest = path.join(tmp, "dest");
|
const dest = path.join(tmp, "dest");
|
||||||
const old = path.join(dest, "2026-09-01");
|
fs.mkdirSync(path.join(dest, "2026-09-01"), { recursive: true });
|
||||||
fs.mkdirSync(path.join(old, "locked"), { recursive: true });
|
writeExecutable(
|
||||||
fs.writeFileSync(path.join(old, "locked", "f.age"), "x");
|
path.join(bin, "rm"),
|
||||||
fs.chmodSync(path.join(old, "locked"), 0o500);
|
`#!/bin/sh\ncase "$*" in *2026-09-01*) exit 1 ;; esac\nexec /bin/rm "$@"\n`,
|
||||||
try {
|
);
|
||||||
const { status } = run({ envFile: writeEnv() });
|
const { status, output } = run({ envFile: writeEnv() });
|
||||||
expect(status).toBe(1);
|
expect(status).toBe(1);
|
||||||
|
expect(output).toContain("rotation: could not remove 2026-09-01");
|
||||||
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" });
|
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" });
|
||||||
} finally {
|
expect(fs.existsSync(path.join(dest, "2026-09-01"))).toBe(true);
|
||||||
fs.chmodSync(path.join(old, "locked"), 0o700);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -427,6 +427,15 @@ journalctl --user -t pg-backup-pull --since -1h
|
||||||
systemctl --user list-timers maximus-pg-backup-pull.timer
|
systemctl --user list-timers maximus-pg-backup-pull.timer
|
||||||
```
|
```
|
||||||
|
|
||||||
|
A pass whose newest dated directory is not today's (UTC) logs
|
||||||
|
`warning: no directory for <today>, latest is <date>`: the VPS stopped
|
||||||
|
producing, or the pass ran before its 03:00 UTC run. The report still says
|
||||||
|
`ok` with the newest directory's count; freshness is judged on the VPS side
|
||||||
|
by `GET /backups`.
|
||||||
|
|
||||||
|
The script sticks to what busybox offers (no `find -printf`, no
|
||||||
|
`date -d "... days"`): the CI runs its tests in `node:22-alpine`.
|
||||||
|
|
||||||
### Exit codes
|
### Exit codes
|
||||||
|
|
||||||
| Code | Meaning |
|
| Code | Meaning |
|
||||||
|
|
|
||||||
|
|
@ -113,25 +113,49 @@ if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --partial-dir=.rsync-partial \
|
||||||
fi
|
fi
|
||||||
chmod -R go-rwx "$DEST"
|
chmod -R go-rwx "$DEST"
|
||||||
|
|
||||||
latest=$(find "$DEST" -mindepth 1 -maxdepth 1 -type d -name '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' -printf '%f\n' | sort | tail -1)
|
# Plain shell over the glob, no `find -printf` nor `date -d "... days"`: both
|
||||||
|
# are GNU-only, and the CI runs this script under busybox (node:22-alpine).
|
||||||
|
is_dated() {
|
||||||
|
case "$1" in
|
||||||
|
[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) return 0 ;;
|
||||||
|
*) return 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
# YYYY-MM-DD as a YYYYMMDD integer: `[ a \< b ]` is not POSIX, and /bin/sh is dash.
|
||||||
|
day_number() { echo "$1" | tr -d -; }
|
||||||
|
|
||||||
|
latest=""
|
||||||
|
for dir in "$DEST"/*/; do
|
||||||
|
[ -d "$dir" ] || continue
|
||||||
|
name=$(basename "$dir")
|
||||||
|
is_dated "$name" || continue
|
||||||
|
if [ -z "$latest" ] || [ "$(day_number "$name")" -gt "$(day_number "$latest")" ]; then
|
||||||
|
latest="$name"
|
||||||
|
fi
|
||||||
|
done
|
||||||
files=0
|
files=0
|
||||||
if [ -n "$latest" ]; then
|
if [ -n "$latest" ]; then
|
||||||
files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l)
|
files=$(find "$DEST/$latest" -type f -name '*.age' | wc -l | tr -d ' ')
|
||||||
fi
|
fi
|
||||||
log "pull ok latest=${latest:-none} files=$files"
|
log "pull ok latest=${latest:-none} files=$files"
|
||||||
|
# The report counts the newest local directory: if the VPS stops producing, a
|
||||||
|
# pass still says ok on an old one. GET /backups judges freshness on the VPS
|
||||||
|
# side; this line names the gap in the journal.
|
||||||
|
if [ "${latest:-}" != "$TODAY" ]; then
|
||||||
|
log "warning: no directory for $TODAY, latest is ${latest:-none}"
|
||||||
|
fi
|
||||||
|
|
||||||
# --- Rotation (fails loudly) --------------------------------------------------
|
# --- Rotation (fails loudly) --------------------------------------------------
|
||||||
cutoff=$(date -u -d "$TODAY -$((RETENTION_DAYS - 1)) days" +%F)
|
# Epoch arithmetic: `date -d YYYY-MM-DD` and `date -d @N` are read by GNU date
|
||||||
|
# and busybox alike.
|
||||||
|
today_epoch=$(date -u -d "$TODAY" +%s)
|
||||||
|
cutoff=$(date -u -d "@$((today_epoch - (RETENTION_DAYS - 1) * 86400))" +%F)
|
||||||
rotation_ok=true
|
rotation_ok=true
|
||||||
for dir in "$DEST"/*/; do
|
for dir in "$DEST"/*/; do
|
||||||
[ -d "$dir" ] || continue
|
[ -d "$dir" ] || continue
|
||||||
name=$(basename "$dir")
|
name=$(basename "$dir")
|
||||||
case "$name" in
|
is_dated "$name" || continue
|
||||||
[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;;
|
if [ "$(day_number "$name")" -lt "$(day_number "$cutoff")" ]; then
|
||||||
*) continue ;;
|
|
||||||
esac
|
|
||||||
# Compared as YYYYMMDD integers: `[ a \< b ]` is not POSIX, and /bin/sh is dash.
|
|
||||||
if [ "$(echo "$name" | tr -d -)" -lt "$(echo "$cutoff" | tr -d -)" ]; then
|
|
||||||
if rm -rf -- "$dir"; then
|
if rm -rf -- "$dir"; then
|
||||||
log "rotation: removed $name"
|
log "rotation: removed $name"
|
||||||
else
|
else
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue