vps-health-api/agent/claude-usage/backfill-history.js

#!/usr/bin/env node
// One-shot backfill of the Claude usage history (issue #55).
//
// Run by hand from the repo, on the workstation that holds the transcripts.
// It is NOT copied into the cron install: the agent itself never changes.
//
//   node agent/claude-usage/backfill-history.js --out <file> [--root <dir>] [--max-windows <n>]
//
// Rebuilds the closed days from ~/.claude/projects with the agent's own
// aggregate(), on past windows, and writes { days: [...] } to <file> in the
// archive's day format. The file then goes to the server and is merged with
// `node claude-usage-history.js --import`, where an already archived date
// always wins (see agent/README.md, "Backfilling the history").
//
// What it never does: read ~/.claude/.credentials.json, open a network
// connection, or write anything but <file>, which it refuses to overwrite.
// Output: counts only, never a path or a day's content.
//
// WINDOWS. aggregate() reads a 30-day window ending on `today`. Each window
// keeps only its 28 inner days and the next one steps back 28 days: every
// kept day D has D-1 and D+1 in the same window, so a request whose lines
// straddle a midnight is deduplicated by the window that keeps it, exactly as
// the agent does inside its own window, and is never counted on both sides of
// a window boundary. The first window keeps nothing after D-2, the last day
// the agent may have frozen: an open day never enters the archive this way.
//
// STOP. A fixed number of windows, --max-windows (default 24, about 22
// months), then the leading empty days are trimmed. No guess from file
// mtimes: an mtime is a file's LAST write, and a resumed session keeps lines
// far older than it, so "past the oldest mtime" is no proof of "past the
// oldest line". A full scan of ~600 MB takes a few seconds per window.
//
// Each window calls aggregate() with no frozen days, and only { date, models }
// of its days is kept: no project, no limit, nothing else of the snapshot.

const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const { aggregate, torontoDay, shiftDay } = require("./aggregate.js");

const PREFIX = "claude-usage backfill:";
const KEPT_DAYS = 28;
const DEFAULT_MAX_WINDOWS = 24;

function isEmptyDay(day) {
  return Object.keys(day.models).length === 0;
}

/**
 * @param {object} opts
 * @param {string} opts.root          transcripts root
 * @param {string} [opts.today]       Toronto date "YYYY-MM-DD" (default: now)
 * @param {number} [opts.maxWindows]  hard cap on the number of windows
 * @param {Function} [opts.openFile]  passed through to aggregate(), for tests
 * @returns {Promise<{ days: Array<{date, models}>, windows: number, readErrors: number }>}
 */
async function backfill({ root, today = torontoDay(Date.now()), maxWindows = DEFAULT_MAX_WINDOWS, openFile } = {}) {
  const collected = [];
  let readErrors = 0;
  let windows = 0;
  let keptEnd = shiftDay(today, -2);

  while (windows < maxWindows) {
    const result = await aggregate({
      root,
      frozenDays: {},
      today: shiftDay(keptEnd, 1),
      ...(openFile ? { openFile } : {}),
    });
    windows += 1;
    readErrors += result.readErrors;
    // result.days: keptEnd-28 .. keptEnd+1; the inner 28 are kept.
    const kept = result.days.slice(1, 1 + KEPT_DAYS);
    collected.unshift(...kept);
    keptEnd = shiftDay(kept[0].date, -1);
  }

  const first = collected.findIndex((day) => !isEmptyDay(day));
  const days = first === -1 ? [] : collected.slice(first).map(({ date, models }) => ({ date, models }));
  return { days, windows, readErrors };
}

function parseArgs(argv) {
  const opts = { out: null, root: null, maxWindows: DEFAULT_MAX_WINDOWS };
  for (let i = 0; i < argv.length; i += 2) {
    const [flag, value] = [argv[i], argv[i + 1]];
    if (value === undefined) return { error: `missing value after ${flag}` };
    if (flag === "--out") opts.out = value;
    else if (flag === "--root") opts.root = value;
    else if (flag === "--max-windows") {
      if (!/^[1-9]\d{0,2}$/.test(value)) return { error: "--max-windows must be a number from 1 to 999" };
      opts.maxWindows = Number(value);
    } else return { error: `unknown option ${flag}` };
  }
  if (!opts.out) return { error: "--out <file> is required" };
  return { opts };
}

async function main(argv = process.argv.slice(2), env = process.env, io = { out: console.log, err: console.error }) {
  const parsed = parseArgs(argv);
  if (parsed.error) {
    io.err(`${PREFIX} ${parsed.error}`);
    io.err("usage: node agent/claude-usage/backfill-history.js --out <file> [--root <dir>] [--max-windows <n>]");
    return 2;
  }
  const { out, maxWindows } = parsed.opts;
  const root = parsed.opts.root || path.join(env.HOME || os.homedir(), ".claude", "projects");

  const result = await backfill({ root, maxWindows });
  try {
    // wx: never overwrite, a second run must not clobber a file already sent.
    fs.writeFileSync(out, `${JSON.stringify({ days: result.days })}\n`, { flag: "wx", mode: 0o600 });
  } catch (err) {
    io.err(`${PREFIX} output not written (code=${err.code || "write_error"})`);
    return 1;
  }
  const span = result.days.length ? ` from ${result.days[0].date} to ${result.days.at(-1).date}` : "";
  io.out(`${PREFIX} ${result.days.length} day(s)${span}, ${result.windows} window(s), read_errors=${result.readErrors}`);
  if (result.readErrors > 0) {
    io.err(`${PREFIX} warning: read_errors=${result.readErrors}, some days may be incomplete; rerun before importing`);
    return 1;
  }
  return 0;
}

if (require.main === module) {
  main().then(
    (code) => {
      process.exitCode = code;
    },
    (err) => {
      console.error(`${PREFIX} unexpected failure (code=${(err && err.code) || "error"})`);
      process.exitCode = 1;
    },
  );
}

module.exports = { backfill, main, parseArgs, KEPT_DAYS, DEFAULT_MAX_WINDOWS };