/** * Bank signatures and header-signature drift (#330). * * TWO THINGS LIVE HERE, and they are the same thing seen twice: a header row * reduced to its normalized labels. * * - A BANK SIGNATURE is that label list written down in advance, per bank, so * a known export is recognised by name instead of being guessed at. * - A STORED SIGNATURE is that same label list recorded on the source at the * last successful import (`import_sources.header_signature`), so the next * file from the same bank can be compared against it column by column. * * WHY SIGNATURES RUN BEFORE THE GENERIC DICTIONARY. `headerDictionary.ts` * matches keywords as SUBSTRINGS, one role at a time, and has no notion of a * best match — the first column containing the keyword wins. That is the right * rule for an unknown file and the wrong one for a known layout: * - Tangerine writes `Date,Transaction,Name,Memo,Amount`. The dictionary reads * `Transaction` as the description (it is a description keyword) and maps * the transaction TYPE column — `DEBIT`/`CREDIT` — as the label of every * row. The signature names `Name`. * - RBC writes its amount column `CAD$`, which contains neither `montant` nor * `amount`, so the dictionary finds no amount column at all and the shape * heuristic pairs `CAD$` with the (usually empty) `USD$` as a debit/credit * pair. The signature names `CAD$` as a single signed amount. * * WHY A FAILING SIGNATURE CANNOT BREAK ANYTHING. These signatures are written * from documented export layouts, WITHOUT real statements — the app is * privacy-first and no real statement lands in this repository. So a signature * is only ever allowed to do what the generic dictionary already does: hand * `detectImportFormat` a set of PREFERENCES. Every one of them is dropped the * moment the data contradicts it (a date column nothing parses as a date, an * amount column the shape scan never proposed). A file no signature recognises * falls through to the dictionary, unchanged. Failing degrades; it never breaks. * * WHY THE STORED SIGNATURE IS A LABEL LIST AND NOT A HASH. The drift panel has * to name the columns that moved ("Montant : 3 → 4"), which a hash forbids. */ import { normalizeHeaderCell, type LexicalHeaderMap, } from "./headerDictionary"; /** Every bank the table knows, and the i18n-free proper noun it is shown as. */ export type BankSignatureId = "desjardins" | "rbc" | "bnc" | "tangerine"; /** The transaction roles a signature can name — `LexicalHeaderMap` minus its count. */ export type HeaderRole = keyof Omit; /** * One documented layout of one bank. * * `labels` is the FINGERPRINT: every one of them must appear in the header row, * compared on the whole normalized cell (not as a substring). `roles` is a * projection of that fingerprint onto the columns detection cares about — * labels carrying no role (`Catégorie`, `Memo`, `N° de chèque`) stay in * `labels` precisely because they are what makes the fingerprint distinctive. */ export interface BankHeaderVariant { readonly labels: readonly string[]; readonly roles: Readonly>>; } export interface BankSignature { readonly id: BankSignatureId; /** Proper noun. Displayed as-is in every language, never translated. */ readonly label: string; /** Delimiters this bank's exports use. A precondition of the match, never an override. */ readonly delimiters: readonly string[]; /** Preamble lines this export prints before its header row. */ readonly maxPreambleLines: number; /** The export wraps whole lines in quotes — see `preprocessQuotedCSV`. */ readonly wholeLineQuoted: boolean; readonly variants: readonly BankHeaderVariant[]; } /** * Labels a variant must carry before it is allowed to claim a file. * * Three is not enough: `Date;Description;Montant` is the shape of half the * synthetic corpus and of any hand-made export, so a three-label Desjardins * variant would put "Format Desjardins reconnu" over files no one can attribute * to Desjardins. Announcing the wrong bank is worse than announcing none — the * generic path reads those files correctly already. */ export const MIN_SIGNATURE_LABELS = 4; /** * Labels any Canadian bank could emit. A variant built only from these * identifies no bank in particular, so the count alone does not deliver the * property `MIN_SIGNATURE_LABELS` promises — such a variant has to match the * header exactly (see `matchBankSignature`). */ const GENERIC_LABELS: ReadonlySet = new Set([ "date", "description", "libelle", "detail", "transaction", "montant", "amount", "solde", "balance", "debit", "credit", "retrait", "depot", "withdrawal", "deposit", ]); /** * The table. Written from the banks' documented export layouts; none of it has * been verified against a real statement (see the file header). Adding a bank * is adding an entry — there is no code to write. */ export const BANK_SIGNATURES: readonly BankSignature[] = [ { // AccèsD exports Date / Description / Montant / Solde, semicolon-separated, // with comma decimals, in French or in English. Some credit-card exports // wrap every line in quotes (handled upstream by `preprocessQuotedCSV`, // which leaves a comma-separated file behind) and some carry NO header row // at all — those cannot be signed, by construction. id: "desjardins", label: "Desjardins", delimiters: [";", ","], maxPreambleLines: 0, wholeLineQuoted: true, variants: [ { labels: ["date", "description", "montant", "solde"], roles: { date: "date", description: "description", amount: "montant", balance: "solde", }, }, { labels: ["date", "description", "amount", "balance"], roles: { date: "date", description: "description", amount: "amount", balance: "balance", }, }, ], }, { // RBC exports one comma-separated file for every account, amounts in a // currency-named column. `CAD$` normalizes to `cad`, which no generic // amount keyword matches — this variant is the only thing that maps it. id: "rbc", label: "RBC", delimiters: [","], maxPreambleLines: 0, wholeLineQuoted: false, variants: [ { labels: [ "accounttype", "accountnumber", "transactiondate", "chequenumber", "description1", "cad", ], roles: { date: "transactiondate", description: "description1", amount: "cad", }, }, ], }, { // Banque Nationale exports a debit/credit pair next to a category column, // semicolon-separated in French. id: "bnc", label: "Banque Nationale", delimiters: [";", ","], maxPreambleLines: 0, wholeLineQuoted: false, variants: [ { labels: ["date", "description", "categorie", "debit", "credit", "solde"], roles: { date: "date", description: "description", debit: "debit", credit: "credit", balance: "solde", }, }, { labels: ["date", "description", "category", "debit", "credit", "balance"], roles: { date: "date", description: "description", debit: "debit", credit: "credit", balance: "balance", }, }, ], }, { // Tangerine exports Date,Transaction,Name,Memo,Amount — comma-separated, // signed amounts. `Transaction` holds the direction word, NOT the label of // the operation; reading it as the description makes every row read // `DEBIT` or `CREDIT`. id: "tangerine", label: "Tangerine", delimiters: [","], maxPreambleLines: 0, wholeLineQuoted: false, variants: [ { labels: ["date", "transaction", "name", "memo", "amount"], roles: { date: "date", description: "name", amount: "amount" }, }, ], }, ]; /** What the caller needs to know before a signature may claim a file. */ export interface BankSignatureInput { /** The header row as parsed, raw cells. */ readonly headerRow: readonly string[]; /** Delimiter detection settled on. */ readonly delimiter: string; /** Preamble lines detection had to skip. */ readonly skipLines: number; /** True when `preprocessQuotedCSV` actually unwrapped the file. */ readonly wholeLineQuoted: boolean; } export interface BankSignatureMatch { readonly signature: BankSignature; readonly variant: BankHeaderVariant; /** The variant's roles, resolved to column indices of THIS file. */ readonly roles: LexicalHeaderMap; } /** * First-occurrence index of every normalized label of a header row. * * First occurrence, not last: a file repeating a label (`Montant;…;Montant`) * is degenerate either way, and taking the leftmost keeps the result stable * between the matcher and the drift diff, which both read this map. */ function labelIndex(headerRow: readonly string[]): Map { const index = new Map(); headerRow.forEach((cell, i) => { const label = normalizeHeaderCell(cell); if (label && !index.has(label)) index.set(label, i); }); return index; } function rolesOf( variant: BankHeaderVariant, index: ReadonlyMap ): LexicalHeaderMap { const columnOf = (role: HeaderRole): number | null => { const label = variant.roles[role]; if (label === undefined) return null; return index.get(label) ?? null; }; const date = columnOf("date"); const description = columnOf("description"); const amount = columnOf("amount"); const debit = columnOf("debit"); const credit = columnOf("credit"); const balance = columnOf("balance"); return { date, description, amount, debit, credit, balance, roleCount: [date, description, amount, debit, credit, balance].filter( (c) => c !== null ).length, }; } /** * Find the bank whose documented layout this header row is. * * Three preconditions guard every match, each one a way the file says it is not * this export: a delimiter the bank does not use, more preamble than the bank * prints, or a whole-line-quoted file claiming to be a bank that does not quote. * They can only ever turn a match into a fall-back to the generic dictionary. * * Returns null for an unknown file — which is the common case and not an error. */ export function matchBankSignature( input: BankSignatureInput ): BankSignatureMatch | null { const index = labelIndex(input.headerRow); if (index.size === 0) return null; for (const signature of BANK_SIGNATURES) { if (!signature.delimiters.includes(input.delimiter)) continue; if (input.skipLines > signature.maxPreambleLines) continue; if (input.wholeLineQuoted && !signature.wholeLineQuoted) continue; for (const variant of signature.variants) { if (!variant.labels.every((label) => index.has(label))) continue; // A variant made only of generic labels must describe the header EXACTLY, // not merely be contained in it. Desjardins is // `date;description;montant;solde` — four labels any Canadian bank could // emit — so accepting them as a SUBSET let a // `Date;Description;Débit;Crédit;Montant;Solde` file claim to be // Desjardins. A variant carrying a discriminating label (`chequenumber`, // `categorie`, `memo`, …) keeps subset matching: extra columns are fine // once something actually identifies the bank. if ( variant.labels.every((label) => GENERIC_LABELS.has(label)) && index.size !== variant.labels.length ) { continue; } return { signature, variant, roles: rolesOf(variant, index) }; } } return null; } /** The bank behind an id, for the banner. Null for an id the table lost. */ export function bankSignatureById( id: BankSignatureId | null | undefined ): BankSignature | null { if (!id) return null; return BANK_SIGNATURES.find((s) => s.id === id) ?? null; } // --------------------------------------------------------------------------- // The stored signature and the drift it detects // --------------------------------------------------------------------------- /** * The header row of a successful import, as `import_sources.header_signature` * stores it: a JSON array of normalized labels. * * Returns null for a file with NO header row — and that null is written to the * column as-is. A headerless source has no signature to compare against and * drift detection is inoperative on it, deliberately: a signature invented from * the data would fire on every change of content, which is every import. */ export function buildHeaderSignature( headers: readonly string[] | null | undefined ): string | null { if (!headers || headers.length === 0) return null; return JSON.stringify(headers.map(normalizeHeaderCell)); } /** * Read back a stored signature. Anything that is not an array of strings — a * hash written by an older build, a truncated row, hand-edited SQL — comes back * null, which switches drift detection off rather than throwing inside an * import. Same rule as everywhere else here: degrade, never break. */ export function parseHeaderSignature( stored: string | null | undefined ): string[] | null { if (!stored) return null; try { const parsed: unknown = JSON.parse(stored); if (!Array.isArray(parsed)) return null; if (!parsed.every((label) => typeof label === "string")) return null; return parsed as string[]; } catch { return null; } } export type HeaderDriftKind = "moved" | "added" | "removed"; export interface HeaderDriftEntry { readonly kind: HeaderDriftKind; /** The normalized label — the key the two signatures are compared on. */ readonly normalizedLabel: string; /** * What to show. The file's own spelling for a column that still exists * (`Montant`), the normalized label for one that disappeared — the raw * spelling of a removed column is not recorded anywhere. */ readonly label: string; /** Column index at the last successful import; null for a new column. */ readonly previousIndex: number | null; /** Column index in the file being imported; null for a column that vanished. */ readonly currentIndex: number | null; } /** * Compare the header row of the file being imported against the one the last * successful import recorded. * * Returns null when there is nothing to say — no stored signature, a headerless * file, an unreadable stored value, or a header that normalizes to exactly the * same labels in the same order. `Montant` becoming `MONTANT ($)` is NOT drift: * it normalizes identically and changes nothing about how the file is read. * * A non-null result is always a non-empty list, so the caller can treat it as * "show the panel". */ export function detectHeaderDrift( stored: string | null | undefined, currentHeaders: readonly string[] | null | undefined ): HeaderDriftEntry[] | null { const previous = parseHeaderSignature(stored); if (!previous || previous.length === 0) return null; if (!currentHeaders || currentHeaders.length === 0) return null; const current = currentHeaders.map(normalizeHeaderCell); if ( previous.length === current.length && previous.every((label, i) => label === current[i]) ) { return null; } const previousIndex = new Map(); previous.forEach((label, i) => { if (label && !previousIndex.has(label)) previousIndex.set(label, i); }); const currentIndex = labelIndex(currentHeaders); const entries: HeaderDriftEntry[] = []; // Walk the file being imported first, so the panel reads in its column order. current.forEach((label, i) => { if (!label) return; if (currentIndex.get(label) !== i) return; // duplicate label, already handled const before = previousIndex.get(label); if (before === undefined) { entries.push({ kind: "added", normalizedLabel: label, label: (currentHeaders[i] ?? "").trim() || label, previousIndex: null, currentIndex: i, }); } else if (before !== i) { entries.push({ kind: "moved", normalizedLabel: label, label: (currentHeaders[i] ?? "").trim() || label, previousIndex: before, currentIndex: i, }); } }); // Then the columns that are simply gone, in the order they used to be in. previous.forEach((label, i) => { if (!label) return; if (previousIndex.get(label) !== i) return; if (currentIndex.has(label)) return; entries.push({ kind: "removed", normalizedLabel: label, label, previousIndex: i, currentIndex: null, }); }); return entries.length > 0 ? entries : null; }