feat(claude-usage): archive closed days by month, history route, one-shot backfill #57
3 changed files with 345 additions and 0 deletions
|
|
@ -1953,3 +1953,165 @@ describe("run-claude-usage.sh", () => {
|
|||
expect(script).toMatch(/flock -n 9/);
|
||||
});
|
||||
});
|
||||
|
||||
// --- One-shot history backfill (#55) ----------------------------------------
|
||||
|
||||
describe("backfill-history.js", () => {
|
||||
const { backfill, main } = require("../agent/claude-usage/backfill-history.js");
|
||||
const MTIME = "2026-10-07T12:00:00Z";
|
||||
|
||||
// Toronto noon of a day (EDT in this range): one line per call.
|
||||
const noon = (date) => `${date}T16:00:00Z`;
|
||||
|
||||
function outputOn(result, date) {
|
||||
const found = result.days.find((d) => d.date === date);
|
||||
return found ? Object.values(found.models).reduce((s, m) => s + m.output, 0) : undefined;
|
||||
}
|
||||
|
||||
function writeSpread() {
|
||||
writeTranscript(
|
||||
"a/spread.jsonl",
|
||||
[
|
||||
line({ ts: noon("2026-08-01"), output: 1 }),
|
||||
line({ ts: noon("2026-09-08"), output: 2 }),
|
||||
line({ ts: noon("2026-09-09"), output: 4 }),
|
||||
line({ ts: noon("2026-10-06"), output: 8 }),
|
||||
line({ ts: noon("2026-10-07"), output: 16 }), // D-1: may still be open
|
||||
line({ ts: noon("2026-10-08"), output: 32 }), // today
|
||||
// One request straddling the 09-08/09-09 midnight, the boundary between
|
||||
// the first two windows, both lines with the same output: counted once,
|
||||
// on the day of the earliest line.
|
||||
line({ ts: "2026-09-09T03:59:00Z", id: "msg_straddle", req: "req_straddle", output: 64 }),
|
||||
line({ ts: "2026-09-09T04:01:00Z", id: "msg_straddle", req: "req_straddle", output: 64 }),
|
||||
// Same boundary, best line after midnight: counted once, on 09-09. A
|
||||
// window ending on 09-08 without its one-day margin would see only the
|
||||
// earlier line and count it a second time on 09-08.
|
||||
line({ ts: "2026-09-09T03:58:00Z", id: "msg_late", req: "req_late", output: 256 }),
|
||||
line({ ts: "2026-09-09T04:02:00Z", id: "msg_late", req: "req_late", output: 512 }),
|
||||
],
|
||||
MTIME,
|
||||
);
|
||||
}
|
||||
|
||||
it("covers contiguous past windows up to D-2, the straddling request once, and nothing else of the snapshot", async () => {
|
||||
writeSpread();
|
||||
const result = await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 3 });
|
||||
expect(result.windows).toBe(3);
|
||||
expect(result.readErrors).toBe(0);
|
||||
|
||||
const dates = result.days.map((d) => d.date);
|
||||
expect(dates[0]).toBe("2026-08-01"); // leading empty days trimmed
|
||||
expect(dates.at(-1)).toBe("2026-10-06"); // D-2
|
||||
for (let i = 1; i < dates.length; i++) expect(dates[i]).toBe(shiftDay(dates[i - 1], 1));
|
||||
|
||||
expect(outputOn(result, "2026-08-01")).toBe(1);
|
||||
expect(outputOn(result, "2026-09-08")).toBe(2 + 64);
|
||||
expect(outputOn(result, "2026-09-09")).toBe(4 + 512);
|
||||
expect(outputOn(result, "2026-10-06")).toBe(8);
|
||||
expect(outputOn(result, "2026-10-07")).toBeUndefined();
|
||||
for (const d of result.days) expect(Object.keys(d).sort()).toEqual(["date", "models"]);
|
||||
expect(result).not.toHaveProperty("projects");
|
||||
});
|
||||
|
||||
it("stops at --max-windows: an older day is left out", async () => {
|
||||
writeSpread();
|
||||
writeTranscript("b/old.jsonl", [line({ ts: noon("2026-06-01"), output: 128 })], MTIME);
|
||||
expect(outputOn(await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 3 }), "2026-06-01")).toBeUndefined();
|
||||
const wider = await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 5 });
|
||||
expect(wider.days[0].date).toBe("2026-06-01");
|
||||
expect(outputOn(wider, "2026-06-01")).toBe(128);
|
||||
});
|
||||
|
||||
it("no transcript at all: no day", async () => {
|
||||
expect((await backfill({ root: path.join(tmpDir, "absent"), today: "2026-10-08", maxWindows: 2 })).days).toEqual([]);
|
||||
});
|
||||
|
||||
it("counts read errors across windows", async () => {
|
||||
writeSpread();
|
||||
const { Readable } = require("node:stream");
|
||||
const result = await backfill({
|
||||
root: tmpDir,
|
||||
today: "2026-10-08",
|
||||
maxWindows: 2,
|
||||
openFile: () =>
|
||||
new Readable({
|
||||
read() {
|
||||
this.destroy(Object.assign(new Error("boom"), { code: "EIO" }));
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect(result.readErrors).toBe(2);
|
||||
});
|
||||
|
||||
describe("main", () => {
|
||||
let home;
|
||||
let io;
|
||||
|
||||
beforeEach(() => {
|
||||
home = path.join(tmpDir, "home");
|
||||
fs.mkdirSync(path.join(home, ".claude", "projects", "p"), { recursive: true });
|
||||
fs.writeFileSync(path.join(home, ".claude", ".credentials.json"), '{"claudeAiOauth":{"accessToken":"sk-ant-oat-SECRET"}}');
|
||||
const recent = path.join(home, ".claude", "projects", "p", "t.jsonl");
|
||||
const twoDaysAgo = shiftDay(torontoDay(Date.now()), -2);
|
||||
fs.writeFileSync(recent, `${JSON.stringify(line({ ts: `${twoDaysAgo}T16:00:00Z`, output: 5 }))}\n`);
|
||||
io = { lines: [], out: (l) => io.lines.push(["out", l]), err: (l) => io.lines.push(["err", l]) };
|
||||
});
|
||||
|
||||
it("writes { days } to --out only, mode 600, no network, credentials never opened", async () => {
|
||||
const out = path.join(tmpDir, "backfill.json");
|
||||
const fetchSpy = vi.spyOn(globalThis, "fetch");
|
||||
const touched = [];
|
||||
for (const fn of ["readFileSync", "openSync", "createReadStream", "writeFileSync", "renameSync", "mkdirSync", "appendFileSync"]) {
|
||||
const realFn = fs[fn];
|
||||
vi.spyOn(fs, fn).mockImplementation(function (target, ...rest) {
|
||||
touched.push([fn, String(target)]);
|
||||
return realFn.call(this, target, ...rest);
|
||||
});
|
||||
}
|
||||
for (const fn of ["readFile", "open", "writeFile"]) {
|
||||
const realFn = fs.promises[fn];
|
||||
vi.spyOn(fs.promises, fn).mockImplementation(function (target, ...rest) {
|
||||
touched.push([fn, String(target)]);
|
||||
return realFn.call(this, target, ...rest);
|
||||
});
|
||||
}
|
||||
|
||||
const code = await main(["--out", out, "--max-windows", "2"], { HOME: home }, io);
|
||||
expect(io.lines.filter(([kind]) => kind === "err")).toEqual([]);
|
||||
expect(code).toBe(0);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(touched.filter(([, target]) => target.includes(".credentials"))).toEqual([]);
|
||||
const writes = touched.filter(([fn]) => /write|rename|mkdir|append/i.test(fn));
|
||||
expect(writes).toEqual([["writeFileSync", out]]);
|
||||
|
||||
expect(fs.statSync(out).mode & 0o777).toBe(0o600);
|
||||
const written = JSON.parse(fs.readFileSync(out, "utf-8"));
|
||||
expect(Object.keys(written)).toEqual(["days"]);
|
||||
expect(written.days).toHaveLength(1);
|
||||
expect(Object.values(written.days[0].models)[0].output).toBe(5);
|
||||
// The written file passes the very import parser of the server.
|
||||
const { parseImport } = require("../claude-usage-history.js");
|
||||
expect(parseImport(fs.readFileSync(out, "utf-8"))).toEqual(written.days);
|
||||
// Counts only on the way out: no path, no token, no day content.
|
||||
expect(io.lines).toEqual([["out", expect.stringMatching(/^claude-usage backfill: 1 day\(s\) from \d{4}-\d{2}-\d{2} to \d{4}-\d{2}-\d{2}, 2 window\(s\), read_errors=0$/)]]);
|
||||
});
|
||||
|
||||
it("refuses to overwrite an existing --out file", async () => {
|
||||
const out = path.join(tmpDir, "backfill.json");
|
||||
fs.writeFileSync(out, "keep me");
|
||||
expect(await main(["--out", out, "--max-windows", "1"], { HOME: home }, io)).toBe(1);
|
||||
expect(fs.readFileSync(out, "utf-8")).toBe("keep me");
|
||||
expect(io.lines).toEqual([["err", "claude-usage backfill: output not written (code=EEXIST)"]]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[[]],
|
||||
[["--out"]],
|
||||
[["--out", "x", "--max-windows", "0"]],
|
||||
[["--out", "x", "--nope", "1"]],
|
||||
])("usage error on %j: exit 2, nothing written", async (argv) => {
|
||||
expect(await main(argv, { HOME: home }, io)).toBe(2);
|
||||
expect(fs.existsSync(path.join(process.cwd(), "x"))).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -316,3 +316,48 @@ unreadable (left at `chmod 000`, for instance) blocks freezing on every beat:
|
|||
each run then rescans the whole 30-day window, slower but correct, and this
|
||||
log line is the only sign. Fix the file's permissions. To force a full
|
||||
recount after the fact, `rm ~/.local/state/maximus-claude-usage/state.json`.
|
||||
|
||||
### Backfilling the history (one-shot, #55)
|
||||
|
||||
The server keeps every closed day in `history/YYYY-MM.json` from the first push
|
||||
after the deploy of #55. The days that already left the 30-day window then only
|
||||
live in the transcripts, which Claude Code deletes as they age: this backfill
|
||||
rebuilds them once. It runs from the repo, never from the cron install, and the
|
||||
agent itself does not change.
|
||||
|
||||
1. On the workstation, from the repo:
|
||||
|
||||
```bash
|
||||
node agent/claude-usage/backfill-history.js --out ~/claude-usage-backfill.json
|
||||
# claude-usage backfill: N day(s) from YYYY-MM-DD to YYYY-MM-DD, 24 window(s), read_errors=0
|
||||
```
|
||||
|
||||
It reads `~/.claude/projects` only (never `~/.claude/.credentials.json`),
|
||||
opens no connection, and writes only the `--out` file, in mode 600, refusing
|
||||
to overwrite it. It goes back `--max-windows` windows of 28 days (default 24,
|
||||
about 22 months; each window takes a few seconds), and stops at D-2: the
|
||||
last closed days are archived by the cron's next push anyway. Exit 1 with
|
||||
`read_errors=N`: some transcript could not be read, rerun before importing.
|
||||
|
||||
2. Copy it to the VPS and merge it inside the container, as `node` (the
|
||||
container's user, owner of the mount):
|
||||
|
||||
```bash
|
||||
scp ~/claude-usage-backfill.json ubuntu@vps-b0826277.tail3c811f.ts.net:/tmp/
|
||||
ssh ubuntu@vps-b0826277.tail3c811f.ts.net
|
||||
CTR=$(docker ps --format '{{.Names}}' | grep u8000gsg044wsk0oo0w884ok | head -1)
|
||||
docker exec -i "$CTR" node claude-usage-history.js --import - < /tmp/claude-usage-backfill.json
|
||||
# import: N day(s) read, A added, K already archived, M month(s) written
|
||||
rm /tmp/claude-usage-backfill.json
|
||||
```
|
||||
|
||||
**An already archived date always wins**: the import only adds the dates
|
||||
the archive lacks, so running it twice, or after the live archive has grown,
|
||||
changes nothing that is there. An import file with one invalid entry or a
|
||||
repeated date is refused whole (exit 1, nothing written). A month file the
|
||||
server cannot read back is skipped and left as is (exit 1, named on stderr).
|
||||
|
||||
3. Check through the API, with the read token:
|
||||
`GET /claude-usage/history?from=<first month>&to=<last month>`.
|
||||
|
||||
Then delete `~/claude-usage-backfill.json` on the workstation.
|
||||
|
|
|
|||
138
agent/claude-usage/backfill-history.js
Normal file
138
agent/claude-usage/backfill-history.js
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
#!/usr/bin/env node
|
||||
// One-shot backfill of the Claude usage history (issue #55).
|
||||
//
|
||||
// Run by hand from the repo, on the workstation that holds the transcripts.
|
||||
// It is NOT copied into the cron install: the agent itself never changes.
|
||||
//
|
||||
// node agent/claude-usage/backfill-history.js --out <file> [--root <dir>] [--max-windows <n>]
|
||||
//
|
||||
// Rebuilds the closed days from ~/.claude/projects with the agent's own
|
||||
// aggregate(), on past windows, and writes { days: [...] } to <file> in the
|
||||
// archive's day format. The file then goes to the server and is merged with
|
||||
// `node claude-usage-history.js --import`, where an already archived date
|
||||
// always wins (see agent/README.md, "Backfilling the history").
|
||||
//
|
||||
// What it never does: read ~/.claude/.credentials.json, open a network
|
||||
// connection, or write anything but <file>, which it refuses to overwrite.
|
||||
// Output: counts only, never a path or a day's content.
|
||||
//
|
||||
// WINDOWS. aggregate() reads a 30-day window ending on `today`. Each window
|
||||
// keeps only its 28 inner days and the next one steps back 28 days: every
|
||||
// kept day D has D-1 and D+1 in the same window, so a request whose lines
|
||||
// straddle a midnight is deduplicated by the window that keeps it, exactly as
|
||||
// the agent does inside its own window, and is never counted on both sides of
|
||||
// a window boundary. The first window keeps nothing after D-2, the last day
|
||||
// the agent may have frozen: an open day never enters the archive this way.
|
||||
//
|
||||
// STOP. A fixed number of windows, --max-windows (default 24, about 22
|
||||
// months), then the leading empty days are trimmed. No guess from file
|
||||
// mtimes: an mtime is a file's LAST write, and a resumed session keeps lines
|
||||
// far older than it, so "past the oldest mtime" is no proof of "past the
|
||||
// oldest line". A full scan of ~600 MB takes a few seconds per window.
|
||||
//
|
||||
// Each window calls aggregate() with no frozen days, and only { date, models }
|
||||
// of its days is kept: no project, no limit, nothing else of the snapshot.
|
||||
|
||||
const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { aggregate, torontoDay, shiftDay } = require("./aggregate.js");
|
||||
|
||||
const PREFIX = "claude-usage backfill:";
|
||||
const KEPT_DAYS = 28;
|
||||
const DEFAULT_MAX_WINDOWS = 24;
|
||||
|
||||
function isEmptyDay(day) {
|
||||
return Object.keys(day.models).length === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} opts
|
||||
* @param {string} opts.root transcripts root
|
||||
* @param {string} [opts.today] Toronto date "YYYY-MM-DD" (default: now)
|
||||
* @param {number} [opts.maxWindows] hard cap on the number of windows
|
||||
* @param {Function} [opts.openFile] passed through to aggregate(), for tests
|
||||
* @returns {Promise<{ days: Array<{date, models}>, windows: number, readErrors: number }>}
|
||||
*/
|
||||
async function backfill({ root, today = torontoDay(Date.now()), maxWindows = DEFAULT_MAX_WINDOWS, openFile } = {}) {
|
||||
const collected = [];
|
||||
let readErrors = 0;
|
||||
let windows = 0;
|
||||
let keptEnd = shiftDay(today, -2);
|
||||
|
||||
while (windows < maxWindows) {
|
||||
const result = await aggregate({
|
||||
root,
|
||||
frozenDays: {},
|
||||
today: shiftDay(keptEnd, 1),
|
||||
...(openFile ? { openFile } : {}),
|
||||
});
|
||||
windows += 1;
|
||||
readErrors += result.readErrors;
|
||||
// result.days: keptEnd-28 .. keptEnd+1; the inner 28 are kept.
|
||||
const kept = result.days.slice(1, 1 + KEPT_DAYS);
|
||||
collected.unshift(...kept);
|
||||
keptEnd = shiftDay(kept[0].date, -1);
|
||||
}
|
||||
|
||||
const first = collected.findIndex((day) => !isEmptyDay(day));
|
||||
const days = first === -1 ? [] : collected.slice(first).map(({ date, models }) => ({ date, models }));
|
||||
return { days, windows, readErrors };
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = { out: null, root: null, maxWindows: DEFAULT_MAX_WINDOWS };
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
const [flag, value] = [argv[i], argv[i + 1]];
|
||||
if (value === undefined) return { error: `missing value after ${flag}` };
|
||||
if (flag === "--out") opts.out = value;
|
||||
else if (flag === "--root") opts.root = value;
|
||||
else if (flag === "--max-windows") {
|
||||
if (!/^[1-9]\d{0,2}$/.test(value)) return { error: "--max-windows must be a number from 1 to 999" };
|
||||
opts.maxWindows = Number(value);
|
||||
} else return { error: `unknown option ${flag}` };
|
||||
}
|
||||
if (!opts.out) return { error: "--out <file> is required" };
|
||||
return { opts };
|
||||
}
|
||||
|
||||
async function main(argv = process.argv.slice(2), env = process.env, io = { out: console.log, err: console.error }) {
|
||||
const parsed = parseArgs(argv);
|
||||
if (parsed.error) {
|
||||
io.err(`${PREFIX} ${parsed.error}`);
|
||||
io.err("usage: node agent/claude-usage/backfill-history.js --out <file> [--root <dir>] [--max-windows <n>]");
|
||||
return 2;
|
||||
}
|
||||
const { out, maxWindows } = parsed.opts;
|
||||
const root = parsed.opts.root || path.join(env.HOME || os.homedir(), ".claude", "projects");
|
||||
|
||||
const result = await backfill({ root, maxWindows });
|
||||
try {
|
||||
// wx: never overwrite, a second run must not clobber a file already sent.
|
||||
fs.writeFileSync(out, `${JSON.stringify({ days: result.days })}\n`, { flag: "wx", mode: 0o600 });
|
||||
} catch (err) {
|
||||
io.err(`${PREFIX} output not written (code=${err.code || "write_error"})`);
|
||||
return 1;
|
||||
}
|
||||
const span = result.days.length ? ` from ${result.days[0].date} to ${result.days.at(-1).date}` : "";
|
||||
io.out(`${PREFIX} ${result.days.length} day(s)${span}, ${result.windows} window(s), read_errors=${result.readErrors}`);
|
||||
if (result.readErrors > 0) {
|
||||
io.err(`${PREFIX} warning: read_errors=${result.readErrors}, some days may be incomplete; rerun before importing`);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().then(
|
||||
(code) => {
|
||||
process.exitCode = code;
|
||||
},
|
||||
(err) => {
|
||||
console.error(`${PREFIX} unexpected failure (code=${(err && err.code) || "error"})`);
|
||||
process.exitCode = 1;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = { backfill, main, parseArgs, KEPT_DAYS, DEFAULT_MAX_WINDOWS };
|
||||
Loading…
Reference in a new issue