Compare commits
No commits in common. "5208957ce91ca34e8947e84afbf4e0f81a1c33a5" and "c5ded9e6f858c07a2d5a00132f37ea5d7773ce2f" have entirely different histories.
5208957ce9
...
c5ded9e6f8
5 changed files with 0 additions and 745 deletions
|
|
@ -1,390 +0,0 @@
|
||||||
const fs = require("node:fs");
|
|
||||||
const os = require("node:os");
|
|
||||||
const path = require("node:path");
|
|
||||||
const { spawnSync } = require("node:child_process");
|
|
||||||
|
|
||||||
// Coverage for the PostgreSQL backup pull agent (la-compagnie-maximus#220).
|
|
||||||
//
|
|
||||||
// The script runs under a systemd user timer and talks to the VPS over
|
|
||||||
// rsync/ssh, then reports to POST /backups/pull. Both are replaced here by fake
|
|
||||||
// binaries on PATH, which record their argv so the tests can check what would
|
|
||||||
// have crossed the wire — in particular that the ingest token never appears in
|
|
||||||
// a command line nor in the script's own output (journald keeps it forever).
|
|
||||||
|
|
||||||
const SCRIPT = path.join(__dirname, "..", "agent", "pg-backup-pull", "run-pg-backup-pull.sh");
|
|
||||||
const TOKEN = "ingest-token-0123456789abcdef";
|
|
||||||
const TODAY = "2026-10-11";
|
|
||||||
|
|
||||||
let tmp;
|
|
||||||
let bin;
|
|
||||||
|
|
||||||
function writeExecutable(file, content) {
|
|
||||||
fs.writeFileSync(file, content, { mode: 0o755 });
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pg-backup-pull-"));
|
|
||||||
bin = path.join(tmp, "bin");
|
|
||||||
fs.mkdirSync(bin);
|
|
||||||
fs.writeFileSync(path.join(tmp, "key"), "fake key", { mode: 0o600 });
|
|
||||||
fs.writeFileSync(path.join(tmp, "known_hosts"), "[100.86.30.21]:2222 ssh-ed25519 AAAA");
|
|
||||||
|
|
||||||
// Fake rsync: records argv, then lands two .age files in today's directory.
|
|
||||||
writeExecutable(
|
|
||||||
path.join(bin, "rsync"),
|
|
||||||
`#!/bin/sh
|
|
||||||
printf '%s\\n' "$@" > "$FAKE_DIR/rsync.argv"
|
|
||||||
[ -n "\${FAKE_RSYNC_FAIL:-}" ] && exit 23
|
|
||||||
for last; do :; done
|
|
||||||
mkdir -p "$last/${TODAY}"
|
|
||||||
echo x > "$last/${TODAY}/maximus_api.dump.age"
|
|
||||||
echo x > "$last/${TODAY}/globals.sql.age"
|
|
||||||
`,
|
|
||||||
);
|
|
||||||
// Fake ssh: what rrsync does on the VPS, minus the restriction. Skips the
|
|
||||||
// options, drops the host, and runs the remote rsync command inside
|
|
||||||
// $FAKE_OUTBOX. Only reached by the real-rsync case; the fake rsync above
|
|
||||||
// never calls it, but the script requires ssh on PATH (none in node:22-alpine).
|
|
||||||
writeExecutable(
|
|
||||||
path.join(bin, "ssh"),
|
|
||||||
`#!/bin/sh
|
|
||||||
while [ $# -gt 0 ]; do
|
|
||||||
case "$1" in
|
|
||||||
-p|-i|-o|-l) shift 2 ;;
|
|
||||||
-*) shift ;;
|
|
||||||
*) shift; break ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
cd "$FAKE_OUTBOX" && exec "$@"
|
|
||||||
`,
|
|
||||||
);
|
|
||||||
// No flock in the CI's busybox: a pass-through stand-in, only where the host
|
|
||||||
// has none (no case here races two runs).
|
|
||||||
if (!["/usr/bin/flock", "/bin/flock"].some((f) => fs.existsSync(f))) {
|
|
||||||
writeExecutable(path.join(bin, "flock"), "#!/bin/sh\nexit 0\n");
|
|
||||||
}
|
|
||||||
// Fake curl: records argv, the header file it was handed and the body.
|
|
||||||
writeExecutable(
|
|
||||||
path.join(bin, "curl"),
|
|
||||||
`#!/bin/sh
|
|
||||||
printf '%s\\n' "$@" > "$FAKE_DIR/curl.argv"
|
|
||||||
prev=""
|
|
||||||
for a; do
|
|
||||||
case "$prev" in -H) case "$a" in @*) cp "\${a#@}" "$FAKE_DIR/curl.header" ;; esac ;; esac
|
|
||||||
prev="$a"
|
|
||||||
done
|
|
||||||
cat > "$FAKE_DIR/curl.body"
|
|
||||||
printf '%s' "\${FAKE_CURL_STATUS:-204}"
|
|
||||||
`,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
fs.rmSync(tmp, { recursive: true, force: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
function run({ envFile, extraEnv = {} } = {}) {
|
|
||||||
const dest = path.join(tmp, "dest");
|
|
||||||
const result = spawnSync("sh", [SCRIPT], {
|
|
||||||
encoding: "utf8",
|
|
||||||
env: {
|
|
||||||
PATH: `${bin}:/usr/bin:/bin`,
|
|
||||||
HOME: tmp,
|
|
||||||
FAKE_DIR: tmp,
|
|
||||||
PG_BACKUP_PULL_ENV_FILE: envFile ?? path.join(tmp, "absent.env"),
|
|
||||||
PG_BACKUP_PULL_DEST: dest,
|
|
||||||
PG_BACKUP_PULL_STATE_DIR: path.join(tmp, "state"),
|
|
||||||
PG_BACKUP_PULL_SSH_KEY: path.join(tmp, "key"),
|
|
||||||
PG_BACKUP_PULL_KNOWN_HOSTS: path.join(tmp, "known_hosts"),
|
|
||||||
PG_BACKUP_PULL_TODAY: TODAY,
|
|
||||||
...extraEnv,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return { ...result, output: result.stdout + result.stderr, dest };
|
|
||||||
}
|
|
||||||
|
|
||||||
function writeEnv() {
|
|
||||||
const file = path.join(tmp, "agent.env");
|
|
||||||
fs.writeFileSync(
|
|
||||||
file,
|
|
||||||
`BACKUPS_API_URL=https://health.example.test\nBACKUPS_INGEST_TOKEN=${TOKEN}\n`,
|
|
||||||
{ mode: 0o600 },
|
|
||||||
);
|
|
||||||
return file;
|
|
||||||
}
|
|
||||||
|
|
||||||
const read = (name) => fs.readFileSync(path.join(tmp, name), "utf8");
|
|
||||||
|
|
||||||
describe("pull", () => {
|
|
||||||
test("pulls over port 2222 with a pinned host key, .age files only, never --delete", () => {
|
|
||||||
const { status } = run();
|
|
||||||
expect(status).toBe(0);
|
|
||||||
const argv = read("rsync.argv").split("\n");
|
|
||||||
expect(argv).toContain("--include=*.age");
|
|
||||||
expect(argv).toContain("--exclude=*");
|
|
||||||
// rsync keeps the first matching rule: without `--include=*/` first, the
|
|
||||||
// dated directories hit `--exclude=*` and nothing crosses, silently.
|
|
||||||
expect(argv.indexOf("--include=*/")).toBeGreaterThan(-1);
|
|
||||||
expect(argv.indexOf("--include=*/")).toBeLessThan(argv.indexOf("--exclude=*"));
|
|
||||||
expect(argv.indexOf("--include=*.age")).toBeLessThan(argv.indexOf("--exclude=*"));
|
|
||||||
expect(argv).not.toContain("--delete");
|
|
||||||
expect(argv).toContain("--prune-empty-dirs");
|
|
||||||
expect(argv).toContain("--ignore-existing");
|
|
||||||
expect(argv).toContain("pgbackup@100.86.30.21:");
|
|
||||||
const ssh = argv.find((a) => a.startsWith("ssh "));
|
|
||||||
expect(ssh).toContain("-p 2222");
|
|
||||||
expect(ssh).toContain("StrictHostKeyChecking=yes");
|
|
||||||
expect(ssh).toContain("BatchMode=yes");
|
|
||||||
expect(ssh).toContain("IdentitiesOnly=yes");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("without a token, still pulls and exits 0, saying the report was skipped", () => {
|
|
||||||
const { status, output, dest } = run();
|
|
||||||
expect(status).toBe(0);
|
|
||||||
expect(output).toContain("report skipped");
|
|
||||||
expect(fs.existsSync(path.join(dest, TODAY, "maximus_api.dump.age"))).toBe(true);
|
|
||||||
expect(fs.existsSync(path.join(tmp, "curl.argv"))).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("keeps the destination private", () => {
|
|
||||||
const { dest } = run();
|
|
||||||
expect(fs.statSync(dest).mode & 0o777).toBe(0o700);
|
|
||||||
expect(fs.statSync(path.join(dest, TODAY, "globals.sql.age")).mode & 0o077).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a failed pull exits 2 and reports pull_failed", () => {
|
|
||||||
const { status } = run({ envFile: writeEnv(), extraEnv: { FAKE_RSYNC_FAIL: "1" } });
|
|
||||||
expect(status).toBe(2);
|
|
||||||
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, files: 0, error: "pull_failed" });
|
|
||||||
});
|
|
||||||
|
|
||||||
test("logs a warning when the newest directory is not today's", () => {
|
|
||||||
fs.writeFileSync(
|
|
||||||
path.join(bin, "rsync"),
|
|
||||||
`#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/2026-10-09"\necho x > "$last/2026-10-09/a.age"\n`,
|
|
||||||
);
|
|
||||||
const { status, output } = run();
|
|
||||||
expect(status).toBe(0);
|
|
||||||
expect(output).toContain("latest=2026-10-09 files=1");
|
|
||||||
expect(output).toContain(`warning: no directory for ${TODAY}, latest is 2026-10-09`);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a dated directory without any .age file logs a warning", () => {
|
|
||||||
fs.writeFileSync(path.join(bin, "rsync"), `#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/${TODAY}"\n`);
|
|
||||||
const { status, output } = run();
|
|
||||||
expect(status).toBe(0);
|
|
||||||
expect(output).toContain(`latest=${TODAY} files=0`);
|
|
||||||
expect(output).toContain(`warning: no .age file in ${TODAY}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a partial file left by rsync is not counted", () => {
|
|
||||||
fs.writeFileSync(
|
|
||||||
path.join(bin, "rsync"),
|
|
||||||
`#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/${TODAY}/.rsync-partial"\n` +
|
|
||||||
`echo x > "$last/${TODAY}/a.age"\necho x > "$last/${TODAY}/.rsync-partial/b.age"\n`,
|
|
||||||
);
|
|
||||||
const { output } = run();
|
|
||||||
expect(output).toContain(`latest=${TODAY} files=1`);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("no warning when today's directory is there", () => {
|
|
||||||
const { output } = run();
|
|
||||||
expect(output).toContain(`latest=${TODAY} files=2`);
|
|
||||||
expect(output).not.toContain("warning");
|
|
||||||
});
|
|
||||||
|
|
||||||
// The real rsync with the script's own filter, against a local outbox through
|
|
||||||
// the fake ssh: only .age files land, inside their dated directory, and a
|
|
||||||
// dated directory holding no .age file is not created. Skipped
|
|
||||||
// where rsync is absent (the CI image); the argv order test above runs there.
|
|
||||||
test.skipIf(!["/usr/bin/rsync", "/bin/rsync"].some((f) => fs.existsSync(f)))(
|
|
||||||
"with the real rsync, only .age files cross, inside their dated directory, never an empty one, never over a pulled file",
|
|
||||||
() => {
|
|
||||||
fs.rmSync(path.join(bin, "rsync"));
|
|
||||||
const outbox = path.join(tmp, "outbox");
|
|
||||||
fs.mkdirSync(path.join(outbox, TODAY), { recursive: true });
|
|
||||||
fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump.age"), "cipher");
|
|
||||||
fs.writeFileSync(path.join(outbox, TODAY, "maximus_api.dump"), "clear");
|
|
||||||
fs.writeFileSync(path.join(outbox, "stray.sql"), "clear");
|
|
||||||
fs.mkdirSync(path.join(outbox, "2026-10-12"));
|
|
||||||
fs.writeFileSync(path.join(outbox, "2026-10-12", "maximus_api.dump"), "clear");
|
|
||||||
fs.mkdirSync(path.join(outbox, "2026-10-10"));
|
|
||||||
fs.writeFileSync(path.join(outbox, "2026-10-10", "globals.sql.age"), "garbage");
|
|
||||||
const dest = path.join(tmp, "dest");
|
|
||||||
fs.mkdirSync(path.join(dest, "2026-10-10"), { recursive: true });
|
|
||||||
fs.writeFileSync(path.join(dest, "2026-10-10", "globals.sql.age"), "good-backup");
|
|
||||||
const { status, output } = run({ extraEnv: { FAKE_OUTBOX: outbox } });
|
|
||||||
expect(status).toBe(0);
|
|
||||||
expect(fs.readdirSync(dest).sort()).toEqual(["2026-10-10", TODAY]);
|
|
||||||
expect(fs.readdirSync(path.join(dest, TODAY))).toEqual(["maximus_api.dump.age"]);
|
|
||||||
// A file already pulled is never rewritten by what the VPS serves later.
|
|
||||||
expect(fs.readFileSync(path.join(dest, "2026-10-10", "globals.sql.age"), "utf8")).toBe("good-backup");
|
|
||||||
expect(output).toContain(`latest=${TODAY} files=1`);
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
test("missing ssh key is an install problem: exit 1, nothing pulled", () => {
|
|
||||||
fs.rmSync(path.join(tmp, "key"));
|
|
||||||
const { status } = run();
|
|
||||||
expect(status).toBe(1);
|
|
||||||
expect(fs.existsSync(path.join(tmp, "rsync.argv"))).toBe(false);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("report", () => {
|
|
||||||
test("posts pulledAt, files and ok to /backups/pull", () => {
|
|
||||||
const { status } = run({ envFile: writeEnv() });
|
|
||||||
expect(status).toBe(0);
|
|
||||||
const body = JSON.parse(read("curl.body"));
|
|
||||||
expect(Object.keys(body).sort()).toEqual(["files", "ok", "pulledAt"]);
|
|
||||||
expect(body).toMatchObject({ ok: true, files: 2 });
|
|
||||||
expect(body.pulledAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/);
|
|
||||||
expect(read("curl.argv").split("\n")).toContain("https://health.example.test/backups/pull");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("the token travels in a header file, never in argv nor in the output", () => {
|
|
||||||
const { output } = run({ envFile: writeEnv() });
|
|
||||||
expect(read("curl.header")).toBe(`Authorization: Bearer ${TOKEN}\n`);
|
|
||||||
expect(read("curl.argv")).not.toContain(TOKEN.slice(0, 12));
|
|
||||||
expect(output).not.toContain(TOKEN.slice(0, 12));
|
|
||||||
expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header");
|
|
||||||
});
|
|
||||||
|
|
||||||
// systemd stops a unit with SIGTERM. The fake curl sends it to the script
|
|
||||||
// while the header file exists; the trap must still remove it.
|
|
||||||
test("a TERM while the report is in flight still removes the header file", () => {
|
|
||||||
writeExecutable(
|
|
||||||
path.join(bin, "curl"),
|
|
||||||
`#!/bin/sh
|
|
||||||
cat > /dev/null
|
|
||||||
# Climb from the parent (the script, or a subshell of it, same cmdline) while
|
|
||||||
# the parent is still the script: the last one is the script itself.
|
|
||||||
ppid_of() { sed 's/.*) //' "/proc/$1/stat" | cut -d' ' -f2; }
|
|
||||||
script=$PPID
|
|
||||||
while parent=$(ppid_of "$script") && tr '\\0' ' ' < "/proc/$parent/cmdline" | grep -q run-pg-backup-pull; do
|
|
||||||
script=$parent
|
|
||||||
done
|
|
||||||
[ -f "$FAKE_DIR/state/auth.header" ] && echo present > "$FAKE_DIR/header.seen"
|
|
||||||
kill -TERM "$script"
|
|
||||||
sleep 1
|
|
||||||
printf 204
|
|
||||||
`,
|
|
||||||
);
|
|
||||||
const { status } = run({ envFile: writeEnv() });
|
|
||||||
expect(read("header.seen")).toBe("present\n");
|
|
||||||
expect(status).toBe(143);
|
|
||||||
expect(fs.readdirSync(path.join(tmp, "state"))).not.toContain("auth.header");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a non-2xx answer exits 3", () => {
|
|
||||||
const { status, output } = run({ envFile: writeEnv(), extraEnv: { FAKE_CURL_STATUS: "401" } });
|
|
||||||
expect(status).toBe(3);
|
|
||||||
expect(output).toContain("report failed status=401");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("rotation", () => {
|
|
||||||
test("keeps 30 dated directories, today included, and ignores other names", () => {
|
|
||||||
const dest = path.join(tmp, "dest");
|
|
||||||
for (const d of ["2026-09-11", "2026-09-12", "notes"]) fs.mkdirSync(path.join(dest, d), { recursive: true });
|
|
||||||
const { status, output } = run();
|
|
||||||
expect(status).toBe(0);
|
|
||||||
// The newest of several dated directories is the one counted.
|
|
||||||
expect(output).toContain(`latest=${TODAY} files=2`);
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-09-11"))).toBe(false);
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-09-12"))).toBe(true);
|
|
||||||
expect(fs.existsSync(path.join(dest, "notes"))).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
// The VPS stopped producing on 2026-09-01 (dead cron, failing pg_dump): its
|
|
||||||
// outbox yields nothing new. Anchored on today, the rotation would delete the
|
|
||||||
// last off-site copy; anchored on the newest backup, it keeps it and the 29
|
|
||||||
// days before, and only removes what is older.
|
|
||||||
test("never removes the newest backup, however old: the window ends at the newest directory", () => {
|
|
||||||
const dest = path.join(tmp, "dest");
|
|
||||||
for (const d of ["2026-08-02", "2026-08-03", "2026-09-01"]) {
|
|
||||||
fs.mkdirSync(path.join(dest, d), { recursive: true });
|
|
||||||
fs.writeFileSync(path.join(dest, d, "a.age"), "x");
|
|
||||||
}
|
|
||||||
fs.writeFileSync(path.join(bin, "rsync"), "#!/bin/sh\nexit 0\n");
|
|
||||||
const { status, output } = run({ envFile: writeEnv() });
|
|
||||||
expect(status).toBe(0);
|
|
||||||
expect(output).toContain("latest=2026-09-01 files=1");
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-09-01", "a.age"))).toBe(true);
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-08-03"))).toBe(true);
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-08-02"))).toBe(false);
|
|
||||||
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: true, files: 1 });
|
|
||||||
});
|
|
||||||
|
|
||||||
// Nothing the VPS sends may delete a copy already here. A future-dated
|
|
||||||
// directory (VPS clock off, or a VPS gone rogue) would, as the newest name,
|
|
||||||
// push the rotation window forward over every real backup.
|
|
||||||
test("a future-dated directory from the VPS removes nothing and is not counted", () => {
|
|
||||||
const dest = path.join(tmp, "dest");
|
|
||||||
for (const d of ["2026-09-20", "2026-10-01", "2026-10-10"]) {
|
|
||||||
fs.mkdirSync(path.join(dest, d), { recursive: true });
|
|
||||||
fs.writeFileSync(path.join(dest, d, "a.age"), "x");
|
|
||||||
}
|
|
||||||
fs.writeFileSync(
|
|
||||||
path.join(bin, "rsync"),
|
|
||||||
`#!/bin/sh\nfor last; do :; done\nmkdir -p "$last/2099-01-01"\necho z > "$last/2099-01-01/z.age"\n`,
|
|
||||||
);
|
|
||||||
const { status, output } = run({ envFile: writeEnv() });
|
|
||||||
expect(status).toBe(0);
|
|
||||||
expect(fs.readdirSync(dest).sort()).toEqual(["2026-09-20", "2026-10-01", "2026-10-10", "2099-01-01"]);
|
|
||||||
expect(output).toContain(`warning: ignoring 2099-01-01, after ${TODAY}`);
|
|
||||||
expect(output).toContain("latest=2026-10-10 files=1");
|
|
||||||
expect(output).not.toContain("rotation: removed");
|
|
||||||
});
|
|
||||||
|
|
||||||
// A name of the dated shape that is no date (2026-13-45; busybox would turn
|
|
||||||
// 2026-02-30 into 2026-03-02) is ignored: the pass goes on, the directory
|
|
||||||
// is neither counted nor removed.
|
|
||||||
test("an impossible dated name is ignored, never stops the pass, never removed", () => {
|
|
||||||
const dest = path.join(tmp, "dest");
|
|
||||||
for (const d of ["2026-13-45", "2026-02-30", "2026-01-01"]) {
|
|
||||||
fs.mkdirSync(path.join(dest, d), { recursive: true });
|
|
||||||
fs.writeFileSync(path.join(dest, d, "z.age"), "z");
|
|
||||||
}
|
|
||||||
const { status, output } = run({ envFile: writeEnv() });
|
|
||||||
expect(status).toBe(0);
|
|
||||||
expect(output).toContain("warning: ignoring 2026-13-45, not a valid date");
|
|
||||||
expect(output).toContain("warning: ignoring 2026-02-30, not a valid date");
|
|
||||||
expect(output).toContain(`latest=${TODAY} files=2`);
|
|
||||||
expect(output).toContain("rotation: removed 2026-01-01");
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-13-45"))).toBe(true);
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-02-30"))).toBe(true);
|
|
||||||
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: true, files: 2 });
|
|
||||||
});
|
|
||||||
|
|
||||||
// The failure comes from a fake rm, not from file modes: the CI runs as
|
|
||||||
// root, which ignores a read-only directory.
|
|
||||||
test("a directory that cannot be removed fails loudly and reports rotation_failed", () => {
|
|
||||||
const dest = path.join(tmp, "dest");
|
|
||||||
fs.mkdirSync(path.join(dest, "2026-09-01"), { recursive: true });
|
|
||||||
writeExecutable(
|
|
||||||
path.join(bin, "rm"),
|
|
||||||
`#!/bin/sh\ncase "$*" in *2026-09-01*) exit 1 ;; esac\nexec /bin/rm "$@"\n`,
|
|
||||||
);
|
|
||||||
const { status, output } = run({ envFile: writeEnv() });
|
|
||||||
expect(status).toBe(1);
|
|
||||||
expect(output).toContain("rotation: could not remove 2026-09-01");
|
|
||||||
expect(JSON.parse(read("curl.body"))).toMatchObject({ ok: false, error: "rotation_failed" });
|
|
||||||
expect(fs.existsSync(path.join(dest, "2026-09-01"))).toBe(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("the script itself", () => {
|
|
||||||
test("never enables shell tracing — `set -x` would echo the token into journald", () => {
|
|
||||||
const traced = fs
|
|
||||||
.readFileSync(SCRIPT, "utf8")
|
|
||||||
.split("\n")
|
|
||||||
.filter((line) => !line.trim().startsWith("#"))
|
|
||||||
.filter((line) => /\bset\b[^#\n]*-[a-z]*x/.test(line));
|
|
||||||
expect(traced).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("is executable", () => {
|
|
||||||
expect(fs.statSync(SCRIPT).mode & 0o111).toBeGreaterThan(0);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
102
agent/README.md
102
agent/README.md
|
|
@ -361,105 +361,3 @@ agent itself does not change.
|
||||||
`GET /claude-usage/history?from=<first month>&to=<last month>`.
|
`GET /claude-usage/history?from=<first month>&to=<last month>`.
|
||||||
|
|
||||||
Then delete `~/claude-usage-backfill.json` on the workstation.
|
Then delete `~/claude-usage-backfill.json` on the workstation.
|
||||||
|
|
||||||
## PostgreSQL backup pull (`agent/pg-backup-pull/`)
|
|
||||||
|
|
||||||
A third, independent agent on the Lenovo. Once a day it pulls the
|
|
||||||
age-encrypted dumps of the shared PostgreSQL cluster from the VPS, keeps 30
|
|
||||||
days of them in `~/backups/postgres/`, and reports the pull to
|
|
||||||
`POST /backups/pull`. The VPS side (`backup-postgres.sh`, 03:00 UTC), the
|
|
||||||
`status.json` contract and the restore procedure live in
|
|
||||||
`la-compagnie-maximus/docs/postgres-ops.md`, section Backup
|
|
||||||
(la-compagnie-maximus#220).
|
|
||||||
|
|
||||||
| File | Role |
|
|
||||||
|------|------|
|
|
||||||
| `run-pg-backup-pull.sh` | rsync over ssh, rotation, report; POSIX `sh`, no node |
|
|
||||||
| `maximus-pg-backup-pull.service` | systemd user unit, oneshot, retries on failure |
|
|
||||||
| `maximus-pg-backup-pull.timer` | `OnCalendar=daily`, `Persistent=true` |
|
|
||||||
|
|
||||||
**The only departure from the cron convention of the other agents is the
|
|
||||||
timer.** A cron line missed while the laptop is off is simply lost;
|
|
||||||
`Persistent=true` runs the missed pass at the next boot. That catch-up run can
|
|
||||||
start before Tailscale is up, hence `Restart=on-failure` every 15 minutes (8
|
|
||||||
tries at most in 4 hours).
|
|
||||||
|
|
||||||
**What it never does.** It never decrypts: the age private key is not on this
|
|
||||||
machine (Vaultwarden plus a paper copy). It never deletes on the VPS side, and
|
|
||||||
rsync only lets directories and `*.age` files through, so a misconfigured
|
|
||||||
outbox cannot land a clear dump here. The key it uses
|
|
||||||
(`~/.ssh/pgbackup_pull_ed25519`) can only run `rrsync -ro` on the outbox, from
|
|
||||||
the Lenovo's Tailscale IP, on the classic sshd at `100.86.30.21:2222`; the VPS
|
|
||||||
host key is pinned in `~/.ssh/known_hosts_pgbackup` (no trust on first use).
|
|
||||||
|
|
||||||
### Install
|
|
||||||
|
|
||||||
```
|
|
||||||
mkdir -p ~/.local/share/maximus-pg-backup-pull ~/.config/systemd/user
|
|
||||||
cp agent/pg-backup-pull/run-pg-backup-pull.sh ~/.local/share/maximus-pg-backup-pull/
|
|
||||||
chmod +x ~/.local/share/maximus-pg-backup-pull/run-pg-backup-pull.sh
|
|
||||||
cp agent/pg-backup-pull/maximus-pg-backup-pull.{service,timer} ~/.config/systemd/user/
|
|
||||||
systemctl --user daemon-reload
|
|
||||||
systemctl --user enable --now maximus-pg-backup-pull.timer
|
|
||||||
```
|
|
||||||
|
|
||||||
As for the other agents, the copy step *is* the deployment: never point the
|
|
||||||
unit at the git working tree. The user manager must run without an open
|
|
||||||
session: `loginctl show-user $USER -p Linger` must say `yes`.
|
|
||||||
|
|
||||||
The env file `~/.config/maximus-pg-backup-pull.env` (mode `600`) is optional
|
|
||||||
until the ingest route is deployed. Without it the pull runs and the report is
|
|
||||||
skipped, with one log line. Then, written with `printf`:
|
|
||||||
|
|
||||||
```
|
|
||||||
BACKUPS_API_URL=https://health.lacompagniemaximus.com
|
|
||||||
BACKUPS_INGEST_TOKEN=<the backups ingest token>
|
|
||||||
```
|
|
||||||
|
|
||||||
The token reaches curl through a header file created `0600` and removed right
|
|
||||||
after, never through argv. An `EXIT` trap removes it too when the run is
|
|
||||||
stopped mid-report (systemd's `SIGTERM` is turned into an exit first).
|
|
||||||
|
|
||||||
### Check
|
|
||||||
|
|
||||||
```
|
|
||||||
systemctl --user start maximus-pg-backup-pull.service
|
|
||||||
journalctl --user -t pg-backup-pull --since -1h
|
|
||||||
systemctl --user list-timers maximus-pg-backup-pull.timer
|
|
||||||
```
|
|
||||||
|
|
||||||
A pass whose newest dated directory is not today's (UTC) logs
|
|
||||||
`warning: no directory for <today>, latest is <date>`: the VPS stopped
|
|
||||||
producing, or the pass ran before its 03:00 UTC run. A dated directory with no
|
|
||||||
`.age` file is never created (`--prune-empty-dirs`), and one found empty logs
|
|
||||||
`warning: no .age file in <date>`. The report still says `ok` with the newest
|
|
||||||
directory's count (partial files in `.rsync-partial/` not counted); freshness
|
|
||||||
is judged on the VPS side by `GET /backups`.
|
|
||||||
|
|
||||||
**Nothing the VPS sends may delete or alter a copy already here.**
|
|
||||||
|
|
||||||
- **No overwrite.** A file already pulled is never rewritten
|
|
||||||
(`--ignore-existing`): the VPS writes each dump to a `.tmp` and renames it,
|
|
||||||
so what lands is whole.
|
|
||||||
- **Directory names are checked.** A name is dated only if `date` gives it
|
|
||||||
back unchanged: `2026-13-45` or `2026-02-30` is logged and ignored, never
|
|
||||||
counted, never removed, and the pass goes on. A dated name after today
|
|
||||||
(VPS clock off, or a VPS gone rogue) is logged and ignored the same way.
|
|
||||||
- **Rotation is anchored on the newest valid backup not after today.** It
|
|
||||||
keeps that directory and the 29 days before it. Not on today alone: if the
|
|
||||||
VPS stops producing (dead cron, failing `pg_dump`) or this machine's clock
|
|
||||||
jumps ahead, the last off-site copy stays put instead of aging out after
|
|
||||||
30 days. Not on the newest name alone: a future-dated directory would push
|
|
||||||
the window over every real backup.
|
|
||||||
|
|
||||||
The script sticks to what busybox offers (no `find -printf`, no
|
|
||||||
`date -d "... days"`): the CI runs its tests in `node:22-alpine`.
|
|
||||||
|
|
||||||
### Exit codes
|
|
||||||
|
|
||||||
| Code | Meaning |
|
|
||||||
|------|---------|
|
|
||||||
| 0 | Pulled, rotated, reported (or report skipped without a token) |
|
|
||||||
| 1 | Install problem (missing key, known_hosts, binary) or rotation failure |
|
|
||||||
| 2 | The pull failed (`pull_failed` reported) |
|
|
||||||
| 3 | Pulled and rotated, but the report did not land |
|
|
||||||
|
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
# Daily pull of the encrypted PostgreSQL backups (la-compagnie-maximus#220).
|
|
||||||
# Installed as a systemd *user* unit, see agent/README.md.
|
|
||||||
[Unit]
|
|
||||||
Description=Pull the encrypted PostgreSQL backups from the VPS
|
|
||||||
# A run caught up at boot may start before Tailscale is up: retry a few times.
|
|
||||||
StartLimitIntervalSec=4h
|
|
||||||
StartLimitBurst=8
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=oneshot
|
|
||||||
ExecStart=%h/.local/share/maximus-pg-backup-pull/run-pg-backup-pull.sh
|
|
||||||
SyslogIdentifier=pg-backup-pull
|
|
||||||
Nice=10
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=15min
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
# Daily, after the VPS run (03:00 UTC). Persistent=true catches up a run missed
|
|
||||||
# while the workstation was off, as soon as it is back (needs `loginctl
|
|
||||||
# enable-linger` for the user manager to run without an open session).
|
|
||||||
[Unit]
|
|
||||||
Description=Daily pull of the encrypted PostgreSQL backups
|
|
||||||
|
|
||||||
[Timer]
|
|
||||||
OnCalendar=daily
|
|
||||||
Persistent=true
|
|
||||||
RandomizedDelaySec=15min
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=timers.target
|
|
||||||
|
|
@ -1,225 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
#
|
|
||||||
# Daily pull of the encrypted PostgreSQL backups from the VPS onto this
|
|
||||||
# workstation (la-compagnie-maximus#220). Run by the systemd user timer
|
|
||||||
# maximus-pg-backup-pull.timer, unattended.
|
|
||||||
#
|
|
||||||
# The VPS (backup-postgres.sh, 03:00 UTC) age-encrypts one copy of every dump
|
|
||||||
# into /home/pgbackup/outbox/<date>/. This script fetches that outbox over the
|
|
||||||
# classic sshd on 100.86.30.21:2222, where the key can only run `rrsync -ro`.
|
|
||||||
# It never decrypts anything: the age private key is not on this machine.
|
|
||||||
#
|
|
||||||
# NEVER add `set -x`. The output goes to journald, and the shell would echo the
|
|
||||||
# `. "$ENV_FILE"` expansion — the ingest token — into it for good. Nothing below
|
|
||||||
# ever echoes a variable that holds a secret.
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
# Overridable for testing only; the timer uses the defaults.
|
|
||||||
ENV_FILE="${PG_BACKUP_PULL_ENV_FILE:-$HOME/.config/maximus-pg-backup-pull.env}"
|
|
||||||
DEST="${PG_BACKUP_PULL_DEST:-$HOME/backups/postgres}"
|
|
||||||
STATE_DIR="${PG_BACKUP_PULL_STATE_DIR:-$HOME/.local/state/maximus-pg-backup-pull}"
|
|
||||||
SSH_KEY="${PG_BACKUP_PULL_SSH_KEY:-$HOME/.ssh/pgbackup_pull_ed25519}"
|
|
||||||
KNOWN_HOSTS="${PG_BACKUP_PULL_KNOWN_HOSTS:-$HOME/.ssh/known_hosts_pgbackup}"
|
|
||||||
TODAY="${PG_BACKUP_PULL_TODAY:-$(date -u +%F)}"
|
|
||||||
|
|
||||||
SOURCE="pgbackup@100.86.30.21:"
|
|
||||||
SSH_PORT=2222
|
|
||||||
RETENTION_DAYS=30
|
|
||||||
RSYNC_TIMEOUT_SECONDS=900
|
|
||||||
POST_TIMEOUT_SECONDS=20
|
|
||||||
|
|
||||||
EXIT_INSTALL=1
|
|
||||||
EXIT_PULL=2
|
|
||||||
EXIT_REPORT=3
|
|
||||||
|
|
||||||
log() { echo "pg-backup-pull: $*"; }
|
|
||||||
|
|
||||||
for bin in rsync ssh flock timeout; do
|
|
||||||
if ! command -v "$bin" >/dev/null 2>&1; then
|
|
||||||
log "$bin not found"
|
|
||||||
exit "$EXIT_INSTALL"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
for file in "$SSH_KEY" "$KNOWN_HOSTS"; do
|
|
||||||
if [ ! -f "$file" ]; then
|
|
||||||
log "missing $file"
|
|
||||||
exit "$EXIT_INSTALL"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
mkdir -p "$STATE_DIR" "$DEST"
|
|
||||||
chmod 700 "$STATE_DIR" "$DEST"
|
|
||||||
|
|
||||||
exec 9>"$STATE_DIR/run.lock"
|
|
||||||
if ! flock -n 9; then
|
|
||||||
log "a run is already in progress, skipping"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# The ingest token sits in this file for the length of one curl call. The trap
|
|
||||||
# removes it on any exit, a kill by systemd included: the signal traps turn
|
|
||||||
# TERM/INT/HUP into a plain exit, which runs the EXIT trap (dash does not run
|
|
||||||
# it on a signal by itself).
|
|
||||||
header_file="$STATE_DIR/auth.header"
|
|
||||||
trap 'rm -f "$header_file"' EXIT
|
|
||||||
trap 'exit 129' HUP
|
|
||||||
trap 'exit 130' INT
|
|
||||||
trap 'exit 143' TERM
|
|
||||||
|
|
||||||
# The env file is optional until the ingest route is deployed: without a token,
|
|
||||||
# the pull still runs and only the report to the health API is skipped.
|
|
||||||
BACKUPS_API_URL=""
|
|
||||||
BACKUPS_INGEST_TOKEN=""
|
|
||||||
if [ -f "$ENV_FILE" ]; then
|
|
||||||
FILE_MODE=$(stat -c "%a" "$ENV_FILE" 2>/dev/null || echo "")
|
|
||||||
if [ -n "$FILE_MODE" ] && [ "$FILE_MODE" != "600" ]; then
|
|
||||||
log "warning: $ENV_FILE is mode $FILE_MODE, expected 600"
|
|
||||||
fi
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
. "$ENV_FILE"
|
|
||||||
fi
|
|
||||||
|
|
||||||
report() {
|
|
||||||
# report <ok:true|false> <files> [error-code]
|
|
||||||
if [ -z "$BACKUPS_INGEST_TOKEN" ] || [ -z "$BACKUPS_API_URL" ]; then
|
|
||||||
log "no BACKUPS_INGEST_TOKEN/BACKUPS_API_URL in $ENV_FILE, report skipped"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if ! command -v curl >/dev/null 2>&1; then
|
|
||||||
log "curl not found, report skipped"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
pulled_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
||||||
if [ -n "${3:-}" ]; then
|
|
||||||
body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s,"error":"%s"}' "$pulled_at" "$2" "$1" "$3")
|
|
||||||
else
|
|
||||||
body=$(printf '{"pulledAt":"%s","files":%s,"ok":%s}' "$pulled_at" "$2" "$1")
|
|
||||||
fi
|
|
||||||
# The token goes through a header file, never through argv (visible in ps).
|
|
||||||
(umask 077 && printf 'Authorization: Bearer %s\n' "$BACKUPS_INGEST_TOKEN" > "$header_file")
|
|
||||||
status=$(printf '%s' "$body" | curl -sS -o /dev/null -w '%{http_code}' \
|
|
||||||
--max-time "$POST_TIMEOUT_SECONDS" \
|
|
||||||
-H @"$header_file" -H 'Content-Type: application/json' \
|
|
||||||
--data-binary @- "$BACKUPS_API_URL/backups/pull" 2>/dev/null) || status="000"
|
|
||||||
rm -f "$header_file"
|
|
||||||
case "$status" in
|
|
||||||
2??) log "report ok status=$status"; return 0 ;;
|
|
||||||
*) log "report failed status=$status"; return 1 ;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
# --- Pull ---------------------------------------------------------------------
|
|
||||||
# Only directories and .age files cross: a misconfigured outbox can never land
|
|
||||||
# a clear dump here. No --delete: the VPS keeps 7 days, this machine keeps 30.
|
|
||||||
# --prune-empty-dirs: a dated directory with no .age file (encryption failed on
|
|
||||||
# the VPS) is not created here, so it cannot pass for today's backup.
|
|
||||||
# --ignore-existing: a file already pulled is never rewritten, whatever the VPS
|
|
||||||
# serves under its name later. The VPS writes each dump to a .tmp and renames
|
|
||||||
# it (the .tmp never passes the filter), so what lands here is already whole.
|
|
||||||
if ! timeout "$RSYNC_TIMEOUT_SECONDS" rsync -rt --prune-empty-dirs --ignore-existing --partial-dir=.rsync-partial \
|
|
||||||
--include='*/' --include='*.age' --exclude='*' \
|
|
||||||
-e "ssh -p $SSH_PORT -i $SSH_KEY -o IdentitiesOnly=yes -o UserKnownHostsFile=$KNOWN_HOSTS -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" \
|
|
||||||
"$SOURCE" "$DEST/"; then
|
|
||||||
log "pull failed"
|
|
||||||
report false 0 pull_failed || true
|
|
||||||
exit "$EXIT_PULL"
|
|
||||||
fi
|
|
||||||
chmod -R go-rwx "$DEST"
|
|
||||||
|
|
||||||
# Nothing the VPS sends may delete or alter a copy already here: directory
|
|
||||||
# names come from the outbox, so they are checked before they steer anything.
|
|
||||||
#
|
|
||||||
# Plain shell over the glob, no `find -printf` nor `date -d "... days"`: both
|
|
||||||
# are GNU-only, and the CI runs this script under busybox (node:22-alpine).
|
|
||||||
#
|
|
||||||
# A name is dated only if `date` gives it back unchanged: GNU date refuses
|
|
||||||
# 2026-13-45, busybox turns 2026-02-30 into 2026-03-02. Anything else is never
|
|
||||||
# picked as latest, never removed, and must not stop the pass.
|
|
||||||
is_dated() {
|
|
||||||
case "$1" in
|
|
||||||
[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;;
|
|
||||||
*) return 1 ;;
|
|
||||||
esac
|
|
||||||
[ "$(date -u -d "$1" +%F 2>/dev/null)" = "$1" ]
|
|
||||||
}
|
|
||||||
# YYYY-MM-DD as a YYYYMMDD integer: `[ a \< b ]` is not POSIX, and /bin/sh is dash.
|
|
||||||
day_number() { echo "$1" | tr -d -; }
|
|
||||||
|
|
||||||
# latest is the newest dated directory not after today: a future name (VPS
|
|
||||||
# clock off, or a VPS gone rogue) is kept but steers neither the report nor the
|
|
||||||
# rotation below.
|
|
||||||
latest=""
|
|
||||||
for dir in "$DEST"/*/; do
|
|
||||||
[ -d "$dir" ] || continue
|
|
||||||
name=$(basename "$dir")
|
|
||||||
# Only names of the dated shape are logged: they hold digits and dashes, so a
|
|
||||||
# name from the VPS cannot forge a journal line.
|
|
||||||
case "$name" in [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) continue ;; esac
|
|
||||||
if ! is_dated "$name"; then
|
|
||||||
log "warning: ignoring $name, not a valid date"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if [ "$(day_number "$name")" -gt "$(day_number "$TODAY")" ]; then
|
|
||||||
log "warning: ignoring $name, after $TODAY"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if [ -z "$latest" ] || [ "$(day_number "$name")" -gt "$(day_number "$latest")" ]; then
|
|
||||||
latest="$name"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
files=0
|
|
||||||
if [ -n "$latest" ]; then
|
|
||||||
# -maxdepth 1: a partial file left in .rsync-partial/ is not a backup.
|
|
||||||
files=$(find "$DEST/$latest" -maxdepth 1 -type f -name '*.age' | wc -l | tr -d ' ')
|
|
||||||
fi
|
|
||||||
log "pull ok latest=${latest:-none} files=$files"
|
|
||||||
# The report counts the newest local directory: if the VPS stops producing, a
|
|
||||||
# pass still says ok on an old one. GET /backups judges freshness on the VPS
|
|
||||||
# side; this line names the gap in the journal.
|
|
||||||
if [ "${latest:-}" != "$TODAY" ]; then
|
|
||||||
log "warning: no directory for $TODAY, latest is ${latest:-none}"
|
|
||||||
fi
|
|
||||||
if [ -n "$latest" ] && [ "$files" -eq 0 ]; then
|
|
||||||
log "warning: no .age file in $latest"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# --- Rotation (fails loudly) --------------------------------------------------
|
|
||||||
# Anchored on latest, the newest backup held here that is not after today. Not
|
|
||||||
# on today alone: when the VPS stops producing (dead cron, failing pg_dump), or
|
|
||||||
# this machine's clock jumps ahead, a today-based cutoff would delete the last
|
|
||||||
# off-site copy. Not on the newest name alone: a future-dated directory from
|
|
||||||
# the VPS would push the window forward and delete every real backup. This
|
|
||||||
# keeps latest and the 29 days before it; a future or invalid name is newer
|
|
||||||
# than the cutoff or not dated, so it is never removed.
|
|
||||||
# Epoch arithmetic: `date -d YYYY-MM-DD` and `date -d @N` are read by GNU date
|
|
||||||
# and busybox alike.
|
|
||||||
rotation_ok=true
|
|
||||||
cutoff=""
|
|
||||||
if [ -n "$latest" ]; then
|
|
||||||
latest_epoch=$(date -u -d "$latest" +%s)
|
|
||||||
cutoff=$(date -u -d "@$((latest_epoch - (RETENTION_DAYS - 1) * 86400))" +%F)
|
|
||||||
fi
|
|
||||||
# No latest means no dated directory: the loop below then compares nothing.
|
|
||||||
for dir in "$DEST"/*/; do
|
|
||||||
[ -d "$dir" ] || continue
|
|
||||||
name=$(basename "$dir")
|
|
||||||
is_dated "$name" || continue
|
|
||||||
if [ "$(day_number "$name")" -lt "$(day_number "$cutoff")" ]; then
|
|
||||||
if rm -rf -- "$dir"; then
|
|
||||||
log "rotation: removed $name"
|
|
||||||
else
|
|
||||||
log "rotation: could not remove $name"
|
|
||||||
rotation_ok=false
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "$rotation_ok" != true ]; then
|
|
||||||
report false "$files" rotation_failed || true
|
|
||||||
exit "$EXIT_INSTALL"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! report true "$files"; then
|
|
||||||
exit "$EXIT_REPORT"
|
|
||||||
fi
|
|
||||||
Loading…
Reference in a new issue