diff --git a/__tests__/agent-claude-usage.test.js b/__tests__/agent-claude-usage.test.js index 419fc92..3821e11 100644 --- a/__tests__/agent-claude-usage.test.js +++ b/__tests__/agent-claude-usage.test.js @@ -1953,3 +1953,165 @@ describe("run-claude-usage.sh", () => { expect(script).toMatch(/flock -n 9/); }); }); + +// --- One-shot history backfill (#55) ---------------------------------------- + +describe("backfill-history.js", () => { + const { backfill, main } = require("../agent/claude-usage/backfill-history.js"); + const MTIME = "2026-10-07T12:00:00Z"; + + // Toronto noon of a day (EDT in this range): one line per call. + const noon = (date) => `${date}T16:00:00Z`; + + function outputOn(result, date) { + const found = result.days.find((d) => d.date === date); + return found ? Object.values(found.models).reduce((s, m) => s + m.output, 0) : undefined; + } + + function writeSpread() { + writeTranscript( + "a/spread.jsonl", + [ + line({ ts: noon("2026-08-01"), output: 1 }), + line({ ts: noon("2026-09-08"), output: 2 }), + line({ ts: noon("2026-09-09"), output: 4 }), + line({ ts: noon("2026-10-06"), output: 8 }), + line({ ts: noon("2026-10-07"), output: 16 }), // D-1: may still be open + line({ ts: noon("2026-10-08"), output: 32 }), // today + // One request straddling the 09-08/09-09 midnight, the boundary between + // the first two windows, both lines with the same output: counted once, + // on the day of the earliest line. + line({ ts: "2026-09-09T03:59:00Z", id: "msg_straddle", req: "req_straddle", output: 64 }), + line({ ts: "2026-09-09T04:01:00Z", id: "msg_straddle", req: "req_straddle", output: 64 }), + // Same boundary, best line after midnight: counted once, on 09-09. A + // window ending on 09-08 without its one-day margin would see only the + // earlier line and count it a second time on 09-08. + line({ ts: "2026-09-09T03:58:00Z", id: "msg_late", req: "req_late", output: 256 }), + line({ ts: "2026-09-09T04:02:00Z", id: "msg_late", req: "req_late", output: 512 }), + ], + MTIME, + ); + } + + it("covers contiguous past windows up to D-2, the straddling request once, and nothing else of the snapshot", async () => { + writeSpread(); + const result = await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 3 }); + expect(result.windows).toBe(3); + expect(result.readErrors).toBe(0); + + const dates = result.days.map((d) => d.date); + expect(dates[0]).toBe("2026-08-01"); // leading empty days trimmed + expect(dates.at(-1)).toBe("2026-10-06"); // D-2 + for (let i = 1; i < dates.length; i++) expect(dates[i]).toBe(shiftDay(dates[i - 1], 1)); + + expect(outputOn(result, "2026-08-01")).toBe(1); + expect(outputOn(result, "2026-09-08")).toBe(2 + 64); + expect(outputOn(result, "2026-09-09")).toBe(4 + 512); + expect(outputOn(result, "2026-10-06")).toBe(8); + expect(outputOn(result, "2026-10-07")).toBeUndefined(); + for (const d of result.days) expect(Object.keys(d).sort()).toEqual(["date", "models"]); + expect(result).not.toHaveProperty("projects"); + }); + + it("stops at --max-windows: an older day is left out", async () => { + writeSpread(); + writeTranscript("b/old.jsonl", [line({ ts: noon("2026-06-01"), output: 128 })], MTIME); + expect(outputOn(await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 3 }), "2026-06-01")).toBeUndefined(); + const wider = await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 5 }); + expect(wider.days[0].date).toBe("2026-06-01"); + expect(outputOn(wider, "2026-06-01")).toBe(128); + }); + + it("no transcript at all: no day", async () => { + expect((await backfill({ root: path.join(tmpDir, "absent"), today: "2026-10-08", maxWindows: 2 })).days).toEqual([]); + }); + + it("counts read errors across windows", async () => { + writeSpread(); + const { Readable } = require("node:stream"); + const result = await backfill({ + root: tmpDir, + today: "2026-10-08", + maxWindows: 2, + openFile: () => + new Readable({ + read() { + this.destroy(Object.assign(new Error("boom"), { code: "EIO" })); + }, + }), + }); + expect(result.readErrors).toBe(2); + }); + + describe("main", () => { + let home; + let io; + + beforeEach(() => { + home = path.join(tmpDir, "home"); + fs.mkdirSync(path.join(home, ".claude", "projects", "p"), { recursive: true }); + fs.writeFileSync(path.join(home, ".claude", ".credentials.json"), '{"claudeAiOauth":{"accessToken":"sk-ant-oat-SECRET"}}'); + const recent = path.join(home, ".claude", "projects", "p", "t.jsonl"); + const twoDaysAgo = shiftDay(torontoDay(Date.now()), -2); + fs.writeFileSync(recent, `${JSON.stringify(line({ ts: `${twoDaysAgo}T16:00:00Z`, output: 5 }))}\n`); + io = { lines: [], out: (l) => io.lines.push(["out", l]), err: (l) => io.lines.push(["err", l]) }; + }); + + it("writes { days } to --out only, mode 600, no network, credentials never opened", async () => { + const out = path.join(tmpDir, "backfill.json"); + const fetchSpy = vi.spyOn(globalThis, "fetch"); + const touched = []; + for (const fn of ["readFileSync", "openSync", "createReadStream", "writeFileSync", "renameSync", "mkdirSync", "appendFileSync"]) { + const realFn = fs[fn]; + vi.spyOn(fs, fn).mockImplementation(function (target, ...rest) { + touched.push([fn, String(target)]); + return realFn.call(this, target, ...rest); + }); + } + for (const fn of ["readFile", "open", "writeFile"]) { + const realFn = fs.promises[fn]; + vi.spyOn(fs.promises, fn).mockImplementation(function (target, ...rest) { + touched.push([fn, String(target)]); + return realFn.call(this, target, ...rest); + }); + } + + const code = await main(["--out", out, "--max-windows", "2"], { HOME: home }, io); + expect(io.lines.filter(([kind]) => kind === "err")).toEqual([]); + expect(code).toBe(0); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(touched.filter(([, target]) => target.includes(".credentials"))).toEqual([]); + const writes = touched.filter(([fn]) => /write|rename|mkdir|append/i.test(fn)); + expect(writes).toEqual([["writeFileSync", out]]); + + expect(fs.statSync(out).mode & 0o777).toBe(0o600); + const written = JSON.parse(fs.readFileSync(out, "utf-8")); + expect(Object.keys(written)).toEqual(["days"]); + expect(written.days).toHaveLength(1); + expect(Object.values(written.days[0].models)[0].output).toBe(5); + // The written file passes the very import parser of the server. + const { parseImport } = require("../claude-usage-history.js"); + expect(parseImport(fs.readFileSync(out, "utf-8"))).toEqual(written.days); + // Counts only on the way out: no path, no token, no day content. + expect(io.lines).toEqual([["out", expect.stringMatching(/^claude-usage backfill: 1 day\(s\) from \d{4}-\d{2}-\d{2} to \d{4}-\d{2}-\d{2}, 2 window\(s\), read_errors=0$/)]]); + }); + + it("refuses to overwrite an existing --out file", async () => { + const out = path.join(tmpDir, "backfill.json"); + fs.writeFileSync(out, "keep me"); + expect(await main(["--out", out, "--max-windows", "1"], { HOME: home }, io)).toBe(1); + expect(fs.readFileSync(out, "utf-8")).toBe("keep me"); + expect(io.lines).toEqual([["err", "claude-usage backfill: output not written (code=EEXIST)"]]); + }); + + it.each([ + [[]], + [["--out"]], + [["--out", "x", "--max-windows", "0"]], + [["--out", "x", "--nope", "1"]], + ])("usage error on %j: exit 2, nothing written", async (argv) => { + expect(await main(argv, { HOME: home }, io)).toBe(2); + expect(fs.existsSync(path.join(process.cwd(), "x"))).toBe(false); + }); + }); +}); diff --git a/agent/README.md b/agent/README.md index 8e2084d..821b2ac 100644 --- a/agent/README.md +++ b/agent/README.md @@ -316,3 +316,48 @@ unreadable (left at `chmod 000`, for instance) blocks freezing on every beat: each run then rescans the whole 30-day window, slower but correct, and this log line is the only sign. Fix the file's permissions. To force a full recount after the fact, `rm ~/.local/state/maximus-claude-usage/state.json`. + +### Backfilling the history (one-shot, #55) + +The server keeps every closed day in `history/YYYY-MM.json` from the first push +after the deploy of #55. The days that already left the 30-day window then only +live in the transcripts, which Claude Code deletes as they age: this backfill +rebuilds them once. It runs from the repo, never from the cron install, and the +agent itself does not change. + +1. On the workstation, from the repo: + + ```bash + node agent/claude-usage/backfill-history.js --out ~/claude-usage-backfill.json + # claude-usage backfill: N day(s) from YYYY-MM-DD to YYYY-MM-DD, 24 window(s), read_errors=0 + ``` + + It reads `~/.claude/projects` only (never `~/.claude/.credentials.json`), + opens no connection, and writes only the `--out` file, in mode 600, refusing + to overwrite it. It goes back `--max-windows` windows of 28 days (default 24, + about 22 months; each window takes a few seconds), and stops at D-2: the + last closed days are archived by the cron's next push anyway. Exit 1 with + `read_errors=N`: some transcript could not be read, rerun before importing. + +2. Copy it to the VPS and merge it inside the container, as `node` (the + container's user, owner of the mount): + + ```bash + scp ~/claude-usage-backfill.json ubuntu@vps-b0826277.tail3c811f.ts.net:/tmp/ + ssh ubuntu@vps-b0826277.tail3c811f.ts.net + CTR=$(docker ps --format '{{.Names}}' | grep u8000gsg044wsk0oo0w884ok | head -1) + docker exec -i "$CTR" node claude-usage-history.js --import - < /tmp/claude-usage-backfill.json + # import: N day(s) read, A added, K already archived, M month(s) written + rm /tmp/claude-usage-backfill.json + ``` + + **An already archived date always wins**: the import only adds the dates + the archive lacks, so running it twice, or after the live archive has grown, + changes nothing that is there. An import file with one invalid entry or a + repeated date is refused whole (exit 1, nothing written). A month file the + server cannot read back is skipped and left as is (exit 1, named on stderr). + +3. Check through the API, with the read token: + `GET /claude-usage/history?from=&to=`. + +Then delete `~/claude-usage-backfill.json` on the workstation. diff --git a/agent/claude-usage/backfill-history.js b/agent/claude-usage/backfill-history.js new file mode 100644 index 0000000..e4eabcc --- /dev/null +++ b/agent/claude-usage/backfill-history.js @@ -0,0 +1,138 @@ +#!/usr/bin/env node +// One-shot backfill of the Claude usage history (issue #55). +// +// Run by hand from the repo, on the workstation that holds the transcripts. +// It is NOT copied into the cron install: the agent itself never changes. +// +// node agent/claude-usage/backfill-history.js --out [--root ] [--max-windows ] +// +// Rebuilds the closed days from ~/.claude/projects with the agent's own +// aggregate(), on past windows, and writes { days: [...] } to in the +// archive's day format. The file then goes to the server and is merged with +// `node claude-usage-history.js --import`, where an already archived date +// always wins (see agent/README.md, "Backfilling the history"). +// +// What it never does: read ~/.claude/.credentials.json, open a network +// connection, or write anything but , which it refuses to overwrite. +// Output: counts only, never a path or a day's content. +// +// WINDOWS. aggregate() reads a 30-day window ending on `today`. Each window +// keeps only its 28 inner days and the next one steps back 28 days: every +// kept day D has D-1 and D+1 in the same window, so a request whose lines +// straddle a midnight is deduplicated by the window that keeps it, exactly as +// the agent does inside its own window, and is never counted on both sides of +// a window boundary. The first window keeps nothing after D-2, the last day +// the agent may have frozen: an open day never enters the archive this way. +// +// STOP. A fixed number of windows, --max-windows (default 24, about 22 +// months), then the leading empty days are trimmed. No guess from file +// mtimes: an mtime is a file's LAST write, and a resumed session keeps lines +// far older than it, so "past the oldest mtime" is no proof of "past the +// oldest line". A full scan of ~600 MB takes a few seconds per window. +// +// Each window calls aggregate() with no frozen days, and only { date, models } +// of its days is kept: no project, no limit, nothing else of the snapshot. + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { aggregate, torontoDay, shiftDay } = require("./aggregate.js"); + +const PREFIX = "claude-usage backfill:"; +const KEPT_DAYS = 28; +const DEFAULT_MAX_WINDOWS = 24; + +function isEmptyDay(day) { + return Object.keys(day.models).length === 0; +} + +/** + * @param {object} opts + * @param {string} opts.root transcripts root + * @param {string} [opts.today] Toronto date "YYYY-MM-DD" (default: now) + * @param {number} [opts.maxWindows] hard cap on the number of windows + * @param {Function} [opts.openFile] passed through to aggregate(), for tests + * @returns {Promise<{ days: Array<{date, models}>, windows: number, readErrors: number }>} + */ +async function backfill({ root, today = torontoDay(Date.now()), maxWindows = DEFAULT_MAX_WINDOWS, openFile } = {}) { + const collected = []; + let readErrors = 0; + let windows = 0; + let keptEnd = shiftDay(today, -2); + + while (windows < maxWindows) { + const result = await aggregate({ + root, + frozenDays: {}, + today: shiftDay(keptEnd, 1), + ...(openFile ? { openFile } : {}), + }); + windows += 1; + readErrors += result.readErrors; + // result.days: keptEnd-28 .. keptEnd+1; the inner 28 are kept. + const kept = result.days.slice(1, 1 + KEPT_DAYS); + collected.unshift(...kept); + keptEnd = shiftDay(kept[0].date, -1); + } + + const first = collected.findIndex((day) => !isEmptyDay(day)); + const days = first === -1 ? [] : collected.slice(first).map(({ date, models }) => ({ date, models })); + return { days, windows, readErrors }; +} + +function parseArgs(argv) { + const opts = { out: null, root: null, maxWindows: DEFAULT_MAX_WINDOWS }; + for (let i = 0; i < argv.length; i += 2) { + const [flag, value] = [argv[i], argv[i + 1]]; + if (value === undefined) return { error: `missing value after ${flag}` }; + if (flag === "--out") opts.out = value; + else if (flag === "--root") opts.root = value; + else if (flag === "--max-windows") { + if (!/^[1-9]\d{0,2}$/.test(value)) return { error: "--max-windows must be a number from 1 to 999" }; + opts.maxWindows = Number(value); + } else return { error: `unknown option ${flag}` }; + } + if (!opts.out) return { error: "--out is required" }; + return { opts }; +} + +async function main(argv = process.argv.slice(2), env = process.env, io = { out: console.log, err: console.error }) { + const parsed = parseArgs(argv); + if (parsed.error) { + io.err(`${PREFIX} ${parsed.error}`); + io.err("usage: node agent/claude-usage/backfill-history.js --out [--root ] [--max-windows ]"); + return 2; + } + const { out, maxWindows } = parsed.opts; + const root = parsed.opts.root || path.join(env.HOME || os.homedir(), ".claude", "projects"); + + const result = await backfill({ root, maxWindows }); + try { + // wx: never overwrite, a second run must not clobber a file already sent. + fs.writeFileSync(out, `${JSON.stringify({ days: result.days })}\n`, { flag: "wx", mode: 0o600 }); + } catch (err) { + io.err(`${PREFIX} output not written (code=${err.code || "write_error"})`); + return 1; + } + const span = result.days.length ? ` from ${result.days[0].date} to ${result.days.at(-1).date}` : ""; + io.out(`${PREFIX} ${result.days.length} day(s)${span}, ${result.windows} window(s), read_errors=${result.readErrors}`); + if (result.readErrors > 0) { + io.err(`${PREFIX} warning: read_errors=${result.readErrors}, some days may be incomplete; rerun before importing`); + return 1; + } + return 0; +} + +if (require.main === module) { + main().then( + (code) => { + process.exitCode = code; + }, + (err) => { + console.error(`${PREFIX} unexpected failure (code=${(err && err.code) || "error"})`); + process.exitCode = 1; + }, + ); +} + +module.exports = { backfill, main, parseArgs, KEPT_DAYS, DEFAULT_MAX_WINDOWS };