__tests__/agent-claude-usage.test.js

@@ -1953,3 +1953,165 @@ describe("run-claude-usage.sh", () => {
     expect(script).toMatch(/flock -n 9/);
   });
 });
+
+// --- One-shot history backfill (#55) ----------------------------------------
+
+describe("backfill-history.js", () => {
+  const { backfill, main } = require("../agent/claude-usage/backfill-history.js");
+  const MTIME = "2026-10-07T12:00:00Z";
+
+  // Toronto noon of a day (EDT in this range): one line per call.
+  const noon = (date) => `${date}T16:00:00Z`;
+
+  function outputOn(result, date) {
+    const found = result.days.find((d) => d.date === date);
+    return found ? Object.values(found.models).reduce((s, m) => s + m.output, 0) : undefined;
+  }
+
+  function writeSpread() {
+    writeTranscript(
+      "a/spread.jsonl",
+      [
+        line({ ts: noon("2026-08-01"), output: 1 }),
+        line({ ts: noon("2026-09-08"), output: 2 }),
+        line({ ts: noon("2026-09-09"), output: 4 }),
+        line({ ts: noon("2026-10-06"), output: 8 }),
+        line({ ts: noon("2026-10-07"), output: 16 }), // D-1: may still be open
+        line({ ts: noon("2026-10-08"), output: 32 }), // today
+        // One request straddling the 09-08/09-09 midnight, the boundary between
+        // the first two windows, both lines with the same output: counted once,
+        // on the day of the earliest line.
+        line({ ts: "2026-09-09T03:59:00Z", id: "msg_straddle", req: "req_straddle", output: 64 }),
+        line({ ts: "2026-09-09T04:01:00Z", id: "msg_straddle", req: "req_straddle", output: 64 }),
+        // Same boundary, best line after midnight: counted once, on 09-09. A
+        // window ending on 09-08 without its one-day margin would see only the
+        // earlier line and count it a second time on 09-08.
+        line({ ts: "2026-09-09T03:58:00Z", id: "msg_late", req: "req_late", output: 256 }),
+        line({ ts: "2026-09-09T04:02:00Z", id: "msg_late", req: "req_late", output: 512 }),
+      ],
+      MTIME,
+    );
+  }
+
+  it("covers contiguous past windows up to D-2, the straddling request once, and nothing else of the snapshot", async () => {
+    writeSpread();
+    const result = await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 3 });
+    expect(result.windows).toBe(3);
+    expect(result.readErrors).toBe(0);
+
+    const dates = result.days.map((d) => d.date);
+    expect(dates[0]).toBe("2026-08-01"); // leading empty days trimmed
+    expect(dates.at(-1)).toBe("2026-10-06"); // D-2
+    for (let i = 1; i < dates.length; i++) expect(dates[i]).toBe(shiftDay(dates[i - 1], 1));
+
+    expect(outputOn(result, "2026-08-01")).toBe(1);
+    expect(outputOn(result, "2026-09-08")).toBe(2 + 64);
+    expect(outputOn(result, "2026-09-09")).toBe(4 + 512);
+    expect(outputOn(result, "2026-10-06")).toBe(8);
+    expect(outputOn(result, "2026-10-07")).toBeUndefined();
+    for (const d of result.days) expect(Object.keys(d).sort()).toEqual(["date", "models"]);
+    expect(result).not.toHaveProperty("projects");
+  });
+
+  it("stops at --max-windows: an older day is left out", async () => {
+    writeSpread();
+    writeTranscript("b/old.jsonl", [line({ ts: noon("2026-06-01"), output: 128 })], MTIME);
+    expect(outputOn(await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 3 }), "2026-06-01")).toBeUndefined();
+    const wider = await backfill({ root: tmpDir, today: "2026-10-08", maxWindows: 5 });
+    expect(wider.days[0].date).toBe("2026-06-01");
+    expect(outputOn(wider, "2026-06-01")).toBe(128);
+  });
+
+  it("no transcript at all: no day", async () => {
+    expect((await backfill({ root: path.join(tmpDir, "absent"), today: "2026-10-08", maxWindows: 2 })).days).toEqual([]);
+  });
+
+  it("counts read errors across windows", async () => {
+    writeSpread();
+    const { Readable } = require("node:stream");
+    const result = await backfill({
+      root: tmpDir,
+      today: "2026-10-08",
+      maxWindows: 2,
+      openFile: () =>
+        new Readable({
+          read() {
+            this.destroy(Object.assign(new Error("boom"), { code: "EIO" }));
+          },
+        }),
+    });
+    expect(result.readErrors).toBe(2);
+  });
+
+  describe("main", () => {
+    let home;
+    let io;
+
+    beforeEach(() => {
+      home = path.join(tmpDir, "home");
+      fs.mkdirSync(path.join(home, ".claude", "projects", "p"), { recursive: true });
+      fs.writeFileSync(path.join(home, ".claude", ".credentials.json"), '{"claudeAiOauth":{"accessToken":"sk-ant-oat-SECRET"}}');
+      const recent = path.join(home, ".claude", "projects", "p", "t.jsonl");
+      const twoDaysAgo = shiftDay(torontoDay(Date.now()), -2);
+      fs.writeFileSync(recent, `${JSON.stringify(line({ ts: `${twoDaysAgo}T16:00:00Z`, output: 5 }))}\n`);
+      io = { lines: [], out: (l) => io.lines.push(["out", l]), err: (l) => io.lines.push(["err", l]) };
+    });
+
+    it("writes { days } to --out only, mode 600, no network, credentials never opened", async () => {
+      const out = path.join(tmpDir, "backfill.json");
+      const fetchSpy = vi.spyOn(globalThis, "fetch");
+      const touched = [];
+      for (const fn of ["readFileSync", "openSync", "createReadStream", "writeFileSync", "renameSync", "mkdirSync", "appendFileSync"]) {
+        const realFn = fs[fn];
+        vi.spyOn(fs, fn).mockImplementation(function (target, ...rest) {
+          touched.push([fn, String(target)]);
+          return realFn.call(this, target, ...rest);
+        });
+      }
+      for (const fn of ["readFile", "open", "writeFile"]) {
+        const realFn = fs.promises[fn];
+        vi.spyOn(fs.promises, fn).mockImplementation(function (target, ...rest) {
+          touched.push([fn, String(target)]);
+          return realFn.call(this, target, ...rest);
+        });
+      }
+
+      const code = await main(["--out", out, "--max-windows", "2"], { HOME: home }, io);
+      expect(io.lines.filter(([kind]) => kind === "err")).toEqual([]);
+      expect(code).toBe(0);
+      expect(fetchSpy).not.toHaveBeenCalled();
+      expect(touched.filter(([, target]) => target.includes(".credentials"))).toEqual([]);
+      const writes = touched.filter(([fn]) => /write|rename|mkdir|append/i.test(fn));
+      expect(writes).toEqual([["writeFileSync", out]]);
+
+      expect(fs.statSync(out).mode & 0o777).toBe(0o600);
+      const written = JSON.parse(fs.readFileSync(out, "utf-8"));
+      expect(Object.keys(written)).toEqual(["days"]);
+      expect(written.days).toHaveLength(1);
+      expect(Object.values(written.days[0].models)[0].output).toBe(5);
+      // The written file passes the very import parser of the server.
+      const { parseImport } = require("../claude-usage-history.js");
+      expect(parseImport(fs.readFileSync(out, "utf-8"))).toEqual(written.days);
+      // Counts only on the way out: no path, no token, no day content.
+      expect(io.lines).toEqual([["out", expect.stringMatching(/^claude-usage backfill: 1 day\(s\) from \d{4}-\d{2}-\d{2} to \d{4}-\d{2}-\d{2}, 2 window\(s\), read_errors=0$/)]]);
+    });
+
+    it("refuses to overwrite an existing --out file", async () => {
+      const out = path.join(tmpDir, "backfill.json");
+      fs.writeFileSync(out, "keep me");
+      expect(await main(["--out", out, "--max-windows", "1"], { HOME: home }, io)).toBe(1);
+      expect(fs.readFileSync(out, "utf-8")).toBe("keep me");
+      expect(io.lines).toEqual([["err", "claude-usage backfill: output not written (code=EEXIST)"]]);
+    });
+
+    it.each([
+      [[]],
+      [["--out"]],
+      [["--out", "x", "--max-windows", "0"]],
+      [["--out", "x", "--nope", "1"]],
+    ])("usage error on %j: exit 2, nothing written", async (argv) => {
+      expect(await main(argv, { HOME: home }, io)).toBe(2);
+      expect(fs.existsSync(path.join(process.cwd(), "x"))).toBe(false);
+    });
+  });
+});

agent/README.md

@@ -316,3 +316,48 @@ unreadable (left at `chmod 000`, for instance) blocks freezing on every beat:
 each run then rescans the whole 30-day window, slower but correct, and this
 log line is the only sign. Fix the file's permissions. To force a full
 recount after the fact, `rm ~/.local/state/maximus-claude-usage/state.json`.
+
+### Backfilling the history (one-shot, #55)
+
+The server keeps every closed day in `history/YYYY-MM.json` from the first push
+after the deploy of #55. The days that already left the 30-day window then only
+live in the transcripts, which Claude Code deletes as they age: this backfill
+rebuilds them once. It runs from the repo, never from the cron install, and the
+agent itself does not change.
+
+1. On the workstation, from the repo:
+
+   ```bash
+   node agent/claude-usage/backfill-history.js --out ~/claude-usage-backfill.json
+   # claude-usage backfill: N day(s) from YYYY-MM-DD to YYYY-MM-DD, 24 window(s), read_errors=0
+   ```
+
+   It reads `~/.claude/projects` only (never `~/.claude/.credentials.json`),
+   opens no connection, and writes only the `--out` file, in mode 600, refusing
+   to overwrite it. It goes back `--max-windows` windows of 28 days (default 24,
+   about 22 months; each window takes a few seconds), and stops at D-2: the
+   last closed days are archived by the cron's next push anyway. Exit 1 with
+   `read_errors=N`: some transcript could not be read, rerun before importing.
+
+2. Copy it to the VPS and merge it inside the container, as `node` (the
+   container's user, owner of the mount):
+
+   ```bash
+   scp ~/claude-usage-backfill.json ubuntu@vps-b0826277.tail3c811f.ts.net:/tmp/
+   ssh ubuntu@vps-b0826277.tail3c811f.ts.net
+   CTR=$(docker ps --format '{{.Names}}' | grep u8000gsg044wsk0oo0w884ok | head -1)
+   docker exec -i "$CTR" node claude-usage-history.js --import - < /tmp/claude-usage-backfill.json
+   # import: N day(s) read, A added, K already archived, M month(s) written
+   rm /tmp/claude-usage-backfill.json
+   ```
+
+   **An already archived date always wins**: the import only adds the dates
+   the archive lacks, so running it twice, or after the live archive has grown,
+   changes nothing that is there. An import file with one invalid entry or a
+   repeated date is refused whole (exit 1, nothing written). A month file the
+   server cannot read back is skipped and left as is (exit 1, named on stderr).
+
+3. Check through the API, with the read token:
+   `GET /claude-usage/history?from=<first month>&to=<last month>`.
+
+Then delete `~/claude-usage-backfill.json` on the workstation.

agent/claude-usage/backfill-history.js

@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+// One-shot backfill of the Claude usage history (issue #55).
+//
+// Run by hand from the repo, on the workstation that holds the transcripts.
+// It is NOT copied into the cron install: the agent itself never changes.
+//
+//   node agent/claude-usage/backfill-history.js --out <file> [--root <dir>] [--max-windows <n>]
+//
+// Rebuilds the closed days from ~/.claude/projects with the agent's own
+// aggregate(), on past windows, and writes { days: [...] } to <file> in the
+// archive's day format. The file then goes to the server and is merged with
+// `node claude-usage-history.js --import`, where an already archived date
+// always wins (see agent/README.md, "Backfilling the history").
+//
+// What it never does: read ~/.claude/.credentials.json, open a network
+// connection, or write anything but <file>, which it refuses to overwrite.
+// Output: counts only, never a path or a day's content.
+//
+// WINDOWS. aggregate() reads a 30-day window ending on `today`. Each window
+// keeps only its 28 inner days and the next one steps back 28 days: every
+// kept day D has D-1 and D+1 in the same window, so a request whose lines
+// straddle a midnight is deduplicated by the window that keeps it, exactly as
+// the agent does inside its own window, and is never counted on both sides of
+// a window boundary. The first window keeps nothing after D-2, the last day
+// the agent may have frozen: an open day never enters the archive this way.
+//
+// STOP. A fixed number of windows, --max-windows (default 24, about 22
+// months), then the leading empty days are trimmed. No guess from file
+// mtimes: an mtime is a file's LAST write, and a resumed session keeps lines
+// far older than it, so "past the oldest mtime" is no proof of "past the
+// oldest line". A full scan of ~600 MB takes a few seconds per window.
+//
+// Each window calls aggregate() with no frozen days, and only { date, models }
+// of its days is kept: no project, no limit, nothing else of the snapshot.
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const { aggregate, torontoDay, shiftDay } = require("./aggregate.js");
+
+const PREFIX = "claude-usage backfill:";
+const KEPT_DAYS = 28;
+const DEFAULT_MAX_WINDOWS = 24;
+
+function isEmptyDay(day) {
+  return Object.keys(day.models).length === 0;
+}
+
+/**
+ * @param {object} opts
+ * @param {string} opts.root          transcripts root
+ * @param {string} [opts.today]       Toronto date "YYYY-MM-DD" (default: now)
+ * @param {number} [opts.maxWindows]  hard cap on the number of windows
+ * @param {Function} [opts.openFile]  passed through to aggregate(), for tests
+ * @returns {Promise<{ days: Array<{date, models}>, windows: number, readErrors: number }>}
+ */
+async function backfill({ root, today = torontoDay(Date.now()), maxWindows = DEFAULT_MAX_WINDOWS, openFile } = {}) {
+  const collected = [];
+  let readErrors = 0;
+  let windows = 0;
+  let keptEnd = shiftDay(today, -2);
+
+  while (windows < maxWindows) {
+    const result = await aggregate({
+      root,
+      frozenDays: {},
+      today: shiftDay(keptEnd, 1),
+      ...(openFile ? { openFile } : {}),
+    });
+    windows += 1;
+    readErrors += result.readErrors;
+    // result.days: keptEnd-28 .. keptEnd+1; the inner 28 are kept.
+    const kept = result.days.slice(1, 1 + KEPT_DAYS);
+    collected.unshift(...kept);
+    keptEnd = shiftDay(kept[0].date, -1);
+  }
+
+  const first = collected.findIndex((day) => !isEmptyDay(day));
+  const days = first === -1 ? [] : collected.slice(first).map(({ date, models }) => ({ date, models }));
+  return { days, windows, readErrors };
+}
+
+function parseArgs(argv) {
+  const opts = { out: null, root: null, maxWindows: DEFAULT_MAX_WINDOWS };
+  for (let i = 0; i < argv.length; i += 2) {
+    const [flag, value] = [argv[i], argv[i + 1]];
+    if (value === undefined) return { error: `missing value after ${flag}` };
+    if (flag === "--out") opts.out = value;
+    else if (flag === "--root") opts.root = value;
+    else if (flag === "--max-windows") {
+      if (!/^[1-9]\d{0,2}$/.test(value)) return { error: "--max-windows must be a number from 1 to 999" };
+      opts.maxWindows = Number(value);
+    } else return { error: `unknown option ${flag}` };
+  }
+  if (!opts.out) return { error: "--out <file> is required" };
+  return { opts };
+}
+
+async function main(argv = process.argv.slice(2), env = process.env, io = { out: console.log, err: console.error }) {
+  const parsed = parseArgs(argv);
+  if (parsed.error) {
+    io.err(`${PREFIX} ${parsed.error}`);
+    io.err("usage: node agent/claude-usage/backfill-history.js --out <file> [--root <dir>] [--max-windows <n>]");
+    return 2;
+  }
+  const { out, maxWindows } = parsed.opts;
+  const root = parsed.opts.root || path.join(env.HOME || os.homedir(), ".claude", "projects");
+
+  const result = await backfill({ root, maxWindows });
+  try {
+    // wx: never overwrite, a second run must not clobber a file already sent.
+    fs.writeFileSync(out, `${JSON.stringify({ days: result.days })}\n`, { flag: "wx", mode: 0o600 });
+  } catch (err) {
+    io.err(`${PREFIX} output not written (code=${err.code || "write_error"})`);
+    return 1;
+  }
+  const span = result.days.length ? ` from ${result.days[0].date} to ${result.days.at(-1).date}` : "";
+  io.out(`${PREFIX} ${result.days.length} day(s)${span}, ${result.windows} window(s), read_errors=${result.readErrors}`);
+  if (result.readErrors > 0) {
+    io.err(`${PREFIX} warning: read_errors=${result.readErrors}, some days may be incomplete; rerun before importing`);
+    return 1;
+  }
+  return 0;
+}
+
+if (require.main === module) {
+  main().then(
+    (code) => {
+      process.exitCode = code;
+    },
+    (err) => {
+      console.error(`${PREFIX} unexpected failure (code=${(err && err.code) || "error"})`);
+      process.exitCode = 1;
+    },
+  );
+}
+
+module.exports = { backfill, main, parseArgs, KEPT_DAYS, DEFAULT_MAX_WINDOWS };