diff --git a/src/hooks/useImportWizard.ts b/src/hooks/useImportWizard.ts index 79a207f..9029bab 100644 --- a/src/hooks/useImportWizard.ts +++ b/src/hooks/useImportWizard.ts @@ -41,7 +41,7 @@ import { } from "../services/importConfigTemplateService"; import { preprocessQuotedCSV, - autoDetectConfig as runAutoDetect, + detectImportFormat as runAutoDetect, } from "../utils/csvAutoDetect"; import { detectAmountSeparators, @@ -868,9 +868,10 @@ export function useImportWizard() { encoding: state.sourceConfig.encoding, }); - const result = runAutoDetect(content); + const outcome = runAutoDetect(content); - if (result) { + if (outcome.status === "ok") { + const result = outcome.config; const newConfig = { ...state.sourceConfig, delimiter: result.delimiter, @@ -893,9 +894,16 @@ export function useImportWizard() { newConfig.hasHeader ); } else { + // A refused format states WHY (#327): "I cannot read this file" and + // "this file is a shape this version would import backwards" are not + // the same news, and only the second one is the app's own limitation. + // `SET_ERROR` clears the loading flag on its own. dispatch({ type: "SET_ERROR", - payload: "Auto-detection failed. Please configure manually.", + payload: + outcome.status === "rejected" + ? outcome.reason + : "import.errors.autoDetectFailed", }); } } catch (e) { diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index cc74728..3a55f8f 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -189,7 +189,9 @@ "errors": { "unsupportedAmountMode": "The amount mode saved for this source is not recognized. Reconfigure the source before importing.", "unsupportedSignConvention": "The sign convention saved for this source is not recognized. Reconfigure the source before importing.", - "invalidColumnMapping": "The column mapping saved for this source cannot be read. Reconfigure the source before importing." + "invalidColumnMapping": "The column mapping saved for this source cannot be read. Reconfigure the source before importing.", + "absoluteIndicatorFormat": "This file uses positive amounts with a separate column giving the direction (D for debit, C for credit). That format is not supported yet: importing it now would record every deposit as an expense. Export the statement with signed amounts, or with separate debit and credit columns.", + "autoDetectFailed": "Auto-detection could not read this file. Configure the format manually." }, "rowErrors": { "invalidDate": "Unreadable date", diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index 4aa1c97..370a021 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -189,7 +189,9 @@ "errors": { "unsupportedAmountMode": "Le mode de montant enregistré pour cette source n'est pas reconnu. Reconfigurez la source avant d'importer.", "unsupportedSignConvention": "La convention de signe enregistrée pour cette source n'est pas reconnue. Reconfigurez la source avant d'importer.", - "invalidColumnMapping": "Le mapping de colonnes enregistré pour cette source est illisible. Reconfigurez la source avant d'importer." + "invalidColumnMapping": "Le mapping de colonnes enregistré pour cette source est illisible. Reconfigurez la source avant d'importer.", + "absoluteIndicatorFormat": "Ce fichier utilise des montants positifs accompagnés d'une colonne indiquant le sens (D pour débit, C pour crédit). Ce format n'est pas encore pris en charge : l'importer maintenant enregistrerait chaque dépôt comme une dépense. Exportez le relevé avec des montants signés ou avec deux colonnes débit et crédit.", + "autoDetectFailed": "La détection automatique n'a pas pu lire ce fichier. Configurez le format manuellement." }, "rowErrors": { "invalidDate": "Date illisible", diff --git a/src/utils/csvAutoDetect.test.ts b/src/utils/csvAutoDetect.test.ts index 328ef90..821ef40 100644 --- a/src/utils/csvAutoDetect.test.ts +++ b/src/utils/csvAutoDetect.test.ts @@ -18,6 +18,7 @@ import { autoDetectHoldingColumns, analyzeHoldingsCsv, autoDetectConfig, + detectImportFormat, preprocessQuotedCSV, } from "./csvAutoDetect"; import type { DecimalSeparator } from "./amountParser"; @@ -27,6 +28,8 @@ import { readCsvFixture, type CsvFixtureName, } from "../__fixtures__/csv"; +import fr from "../i18n/locales/fr.json"; +import en from "../i18n/locales/en.json"; describe("autoDetectHoldingColumns (#245)", () => { it("detects symbol/quantity/price/book_cost from EN headers", () => { @@ -220,12 +223,23 @@ describe("corpus integrity (#326)", () => { } }); - it("detects a configuration for every fixture (none returns null)", () => { + it("detects a configuration for every fixture but the refused one", () => { + // `absolute-indicator` left this list in #327: it is the one shape + // detection now REFUSES instead of configuring, and it says so — the case + // below asserts the reason rather than a bare null. for (const name of CSV_FIXTURE_NAMES) { + if (name === "absolute-indicator") continue; expect(autoDetectConfig(readCsvFixture(name)), name).not.toBeNull(); } }); + it("never fails silently: every fixture is a config or a stated reason", () => { + for (const name of CSV_FIXTURE_NAMES) { + const outcome = detectImportFormat(readCsvFixture(name)); + expect(outcome.status, name).not.toBe("failed"); + } + }); + it("runs the production rule, with no mirror left to drift (#325)", () => { // The guard that used to live here pinned five expressions of // `parseFilesInternal` so the hand-written mirror could not drift. Both are @@ -301,36 +315,42 @@ describe("autoDetectConfig — debit/credit pair (#326)", () => { }); }); -describe("autoDetectConfig — KNOWN DEFECT: debit/credit order guessed by position (#326, fixed by #328)", () => { - // `detectAmountMode` assigns the LEFTMOST column of a complementary pair to - // `debitAmount` and the rightmost to `creditAmount` (csvAutoDetect.ts:461-470). - // It never looks at the header labels. A file laid out `Credit;Debit` is - // therefore mapped backwards, and every sign in the import is inverted. +describe("autoDetectConfig — debit/credit order read from the labels (#326, fixed by #327)", () => { + // FIXED. `detectAmountMode` used to assign the LEFTMOST column of a + // complementary pair to `debitAmount` and the rightmost to `creditAmount` + // (the loop enumerates `a < b`), without ever looking at the header labels. + // A file laid out `Credit;Debit` was mapped backwards and every sign of the + // import came out inverted. // - // #328 resolves the order from the header dictionary instead of the position. + // The pair is still found by shape — sparse and complementary — but which + // half is the debit now comes from the dictionary (`orderDebitCredit`). - it("maps Credit to debitAmount and Debit to creditAmount", () => { + it("maps Debit to debitAmount and Credit to creditAmount", () => { const cfg = autoDetectConfig(readCsvFixture("debit-credit-reversed"))!; // Column 2 is labelled "Credit" and column 3 "Debit" in the fixture. - expect(cfg.columnMapping.debitAmount).toBe(2); // DEFECT — should be 3 - expect(cfg.columnMapping.creditAmount).toBe(3); // DEFECT — should be 2 + expect(cfg.columnMapping.debitAmount).toBe(3); + expect(cfg.columnMapping.creditAmount).toBe(2); expect(cfg.amountMode).toBe("debit_credit"); }); - it("inverts every sign end to end — groceries become income", () => { - expect(amountsOf("debit-credit-reversed")).toEqual( - // DEFECT — should be REFERENCE_AMOUNTS - [84.32, -1250, 142.18, 56.75, -300, 6.95] - ); + it("parses end to end to the reference amounts", () => { + expect(amountsOf("debit-credit-reversed")).toEqual(REFERENCE_AMOUNTS); }); - it("still nets to the same total, which is why the bug hides", () => { - // The signs are wrong one by one, but the sum is merely negated — no - // aggregate check catches it. Only a per-row review does. + it("reads the reversed file identically to its Debit-first twin", () => { + // The same six transactions, columns swapped. Nothing but the labels tells + // them apart, which is the whole point of the lexical layer. + expect(amountsOf("debit-credit-reversed")).toEqual(amountsOf("debit-credit")); + }); + + it("nets to the reference total, no longer to its negation", () => { + // While the bug was live the signs were wrong one by one but the sum was + // merely negated, so no aggregate check could catch it. That is why the + // per-row equality above is the assertion that matters. const sum = (xs: (number | string)[]) => xs.reduce((a, b) => a + (typeof b === "number" ? b : 0), 0); expect(sum(amountsOf("debit-credit-reversed"))).toBeCloseTo( - -sum(REFERENCE_AMOUNTS), + sum(REFERENCE_AMOUNTS), 6 ); }); @@ -419,11 +439,12 @@ describe("autoDetectConfig — header cell starting with digits (#326, fixed by // normalizes to "2025Montant"; `parseFloat` returned the 2025 prefix, // `hasNumber` flipped true, and the header row was taken for data. // - // Link 1's note: "#328 adds a lexical signal to `detectHeader`, and #325 + // Link 1's note: "#327 adds a lexical signal to `detectHeader`, and #325 // anchors the parser so a numeric prefix no longer reads as a number. Either // fix closes this." #325 landed first — the anchored parser returns NaN for // that cell, so `detectHeader` sees a row with no number and no date. - // #328 keeps its lexical signal for header cells that carry a BARE number. + // #327 keeps its lexical signal for header cells that carry a BARE number, + // which the anchored parser reads as a number by design (see below). it("recognises the header row", () => { const cfg = autoDetectConfig(readCsvFixture("header-numeric-label"))!; @@ -530,41 +551,265 @@ describe("preprocessQuotedCSV + autoDetectConfig — whole-line-quoted file (#32 }); }); -describe("autoDetectConfig — KNOWN DEFECT: absolute amount + D/C indicator (#326, fixed by #328)", () => { - // The third amount mode: one column of unsigned magnitudes plus a - // single-letter D/C column carrying the direction. Nothing reads that - // column. Detection sees only positive numbers, picks `positive_expense`, - // and every credit is imported as an expense. +describe("autoDetectConfig — absolute amount + D/C indicator, refused (#326, fixed by #327)", () => { + // FIXED, by refusing rather than by reading. The third amount mode is one + // column of unsigned magnitudes plus a single-letter D/C column carrying the + // direction. Nothing reads that column: detection saw only positive numbers, + // picked `positive_expense`, and every credit was imported as an expense — + // the file was indistinguishable from `all-positive`, byte for byte of + // config. // - // #328 must DETECT this shape and REFUSE the file with a dedicated message, - // rather than import it backwards. The spec decision is explicit: refused, - // not silently mis-imported. + // The spec decision is explicit — such a file is refused, not silently + // mis-imported — and `amount_mode` carries no CHECK constraint, so reading + // this shape properly can ship later without a migration. - it("ignores the indicator column entirely", () => { - const cfg = autoDetectConfig(readCsvFixture("absolute-indicator"))!; + it("refuses the file with a dedicated reason", () => { + const outcome = detectImportFormat(readCsvFixture("absolute-indicator")); + expect(outcome.status).toBe("rejected"); + expect(outcome).toEqual({ + status: "rejected", + reason: "import.errors.absoluteIndicatorFormat", + }); + }); + + it("produces no configuration, so no row can be imported backwards", () => { + expect(autoDetectConfig(readCsvFixture("absolute-indicator"))).toBeNull(); + // Rows 2 and 5 are flagged "C" in the fixture; they used to import as + // -1250 and -300. + expect(() => amountsOf("absolute-indicator")).toThrow(); + }); + + it("is no longer indistinguishable from the all-positive fixture", () => { + // Same amount column, same absence of negatives — the indicator column is + // the only discriminator, and it is now read. + const without = autoDetectConfig(readCsvFixture("all-positive"))!; + expect(without.signConvention).toBe("positive_expense"); + expect(autoDetectConfig(readCsvFixture("absolute-indicator"))).toBeNull(); + }); + + it("refuses on the values alone, with no header row to help", () => { + const headerless = readCsvFixture("absolute-indicator") + .split("\n") + .slice(1) + .join("\n"); + expect(detectImportFormat(headerless).status).toBe("rejected"); + }); + + it("carries a translated message in both languages", () => { + // The reason is an i18n key, resolved by the error banner + // (`ImportPage.tsx`) like every other key the wizard reports. + expect(fr.import.errors.absoluteIndicatorFormat.length).toBeGreaterThan(0); + expect(en.import.errors.absoluteIndicatorFormat.length).toBeGreaterThan(0); + }); + + it("is the reason the wizard reports, in place of an English literal", () => { + // The hook lives in a `useCallback` and the repository has no jsdom, so + // the wiring is asserted on the source — same technique as the guard above. + const SRC = readFileSync( + resolve(import.meta.dirname, "..", "hooks", "useImportWizard.ts"), + "utf-8" + ); + expect(SRC).toContain("detectImportFormat as runAutoDetect"); + expect(SRC).toContain("outcome.reason"); + // The banner used to render this literal, in English, whatever the + // interface language. + expect(SRC).not.toContain("Auto-detection failed. Please configure"); + }); +}); + +// ============================================================================= +// Issue #327 — the lexical layer, beyond what the corpus covers +// ============================================================================= +// +// The corpus fixtures are frozen files with FR labels. These cases exercise the +// clauses they cannot: EN labels, a header row the shape test alone cannot +// classify, and the fall-back to shape when the labels say nothing. + +/** Build a semicolon CSV from a header line and the standard six rows. */ +function withHeader(header: string, rows: string[]): string { + return [header, ...rows].join("\n") + "\n"; +} + +const SIGNED_ROWS = [ + "05/01/2025;EPICERIE METRO SAINTE-FOY;-84,32", + "15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00", + "18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,18", + "22/01/2025;RESTAURANT LE BISTRO;-56,75", + "27/01/2025;VIREMENT RECU;300,00", + "31/01/2025;FRAIS MENSUELS;-6,95", +]; + +const PAIR_ROWS = [ + "05/01/2025;EPICERIE METRO SAINTE-FOY;;84,32", + "15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;", + "18/01/2025;HYDRO QUEBEC PREAUTORISE;;142,18", + "22/01/2025;RESTAURANT LE BISTRO;;56,75", + "27/01/2025;VIREMENT RECU;300,00;", + "31/01/2025;FRAIS MENSUELS;;6,95", +]; + +describe("detectHeader — lexical signal on a bare-number header cell (#327)", () => { + // The residual case link 1 named. `2025 Montant` is closed by #325's anchored + // parser, but a column titled `2025` outright IS a number by every rule the + // parser has, and the shape test reads `hasNumber` as "this row is data". + const csv = withHeader("Date;Description;2025", SIGNED_ROWS); + + it("recognises the header row from its labels", () => { + const cfg = autoDetectConfig(csv)!; + expect(cfg.hasHeader).toBe(true); + expect(cfg.columnMapping).toEqual({ date: 0, description: 1, amount: 2 }); + }); + + it("keeps the label row out of the data", () => { + const cfg = autoDetectConfig(csv)!; + const rows = Papa.parse(csv, { + delimiter: cfg.delimiter, + skipEmptyLines: true, + }).data as string[][]; + const parsed = rows + .slice(cfg.hasHeader ? 1 : 0) + .map((raw) => mapCorpusRow(raw, cfg)); + expect(parsed).toHaveLength(6); + expect(parsed.every((r) => !("error" in r))).toBe(true); + }); + + it("needs two named roles, not one", () => { + // A single keyword hit proves nothing: a description cell like `DEPOT PAIE + // EMPLOYEUR` contains `depot`. One label plus one number is left to shape. + const oneRole = withHeader("Compte;Description;2025", SIGNED_ROWS); + expect(autoDetectConfig(oneRole)!.hasHeader).toBe(false); + }); + + it("leaves a row carrying a real date to the shape test", () => { + // The date test stays absolute — a data row is data however its other + // cells read, which is what stops a transaction from being eaten as a + // header. + expect(autoDetectConfig(readCsvFixture("no-header"))!.hasHeader).toBe(false); + }); +}); + +describe("autoDetectConfig — English labels (#327)", () => { + it("orders a Withdrawal/Deposit pair from the labels", () => { + const cfg = autoDetectConfig( + withHeader("Date;Description;Deposit;Withdrawal", PAIR_ROWS) + )!; + expect(cfg.amountMode).toBe("debit_credit"); + expect(cfg.columnMapping.debitAmount).toBe(3); + expect(cfg.columnMapping.creditAmount).toBe(2); + }); + + it("reads an Amount/Balance file without mapping the balance", () => { + const cfg = autoDetectConfig( + withHeader("Date;Description;Amount;Balance", [ + "05/01/2025;EPICERIE METRO SAINTE-FOY;-84,32;1415,68", + "15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;2665,68", + "18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,18;2523,50", + ]) + )!; expect(cfg.amountMode).toBe("single"); expect(cfg.columnMapping.amount).toBe(2); - // Column 3 ("Sens", values D/C) appears nowhere in the mapping. - expect(Object.values(cfg.columnMapping)).not.toContain(3); - expect(cfg.signConvention).toBe("positive_expense"); }); - it("imports both credit rows as expenses", () => { - const amounts = amountsOf("absolute-indicator"); - // Rows 2 and 5 are flagged "C" in the fixture. - expect(amounts[1]).toBe(-1250); // DEFECT — should be +1250 - expect(amounts[4]).toBe(-300); // DEFECT — should be +300 - expect(amounts.every((a) => typeof a === "number" && a < 0)).toBe(true); + it("excludes a balance column the arithmetic cannot recognise", () => { + // The running-balance test needs the column to reconcile row to row. These + // rows do not — a mid-period export whose opening balance is missing — and + // the balance is the only column carrying cents, which is what + // `pickBestAmountColumn` goes by. + const rows = [ + "05/01/2025;EPICERIE METRO SAINTE-FOY;-84,00;1415,68", + "15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;9002,11", + "18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,00;4477,03", + ]; + + // Labelled with a word the dictionary does not know, shape wins and maps + // the balance as the amount. + expect( + autoDetectConfig(withHeader("Date;Description;Sortie;Cumul", rows))! + .columnMapping.amount + ).toBe(3); + + // Labelled `Solde`, it is excluded and the real amount column is mapped. + expect( + autoDetectConfig(withHeader("Date;Description;Sortie;Solde", rows))! + .columnMapping.amount + ).toBe(2); + }); +}); + +describe("autoDetectConfig — one labelled half is enough (#327)", () => { + it("infers the credit column from the labelled debit", () => { + // `Sorties` is not in the dictionary; `Débit` is. Knowing one half names + // the other, because the pair has exactly two members. + const cfg = autoDetectConfig( + withHeader("Date;Description;Sorties;Debit", PAIR_ROWS) + )!; + expect(cfg.columnMapping.debitAmount).toBe(3); + expect(cfg.columnMapping.creditAmount).toBe(2); + }); +}); + +describe("autoDetectConfig — falling back to shape when labels are mute (#327)", () => { + it("keeps the positional order when neither half is labelled", () => { + const cfg = autoDetectConfig( + withHeader("Date;Description;Sorties;Entrees", PAIR_ROWS) + )!; + // Unknown labels: the leftmost column of the pair stays the debit, exactly + // as before #327. The file reads backwards — and that is the honest + // outcome, since nothing in it says otherwise. + expect(cfg.columnMapping.debitAmount).toBe(2); + expect(cfg.columnMapping.creditAmount).toBe(3); }); - it("is currently indistinguishable from the all-positive fixture", () => { - // Same detected config, same wrong outcome — which is exactly why #328 - // needs the indicator column as its discriminator. - const withIndicator = autoDetectConfig(readCsvFixture("absolute-indicator"))!; - const without = autoDetectConfig(readCsvFixture("all-positive"))!; - expect(withIndicator.amountMode).toBe(without.amountMode); - expect(withIndicator.signConvention).toBe(without.signConvention); - expect(withIndicator.columnMapping).toEqual(without.columnMapping); + it("keeps mapping a headerless file positionally", () => { + expect(autoDetectConfig(readCsvFixture("no-header"))!.hasHeader).toBe(false); + expect(amountsOf("no-header")).toEqual(REFERENCE_AMOUNTS); + }); + + it("ignores a label the data contradicts", () => { + // The column labelled `Date` holds no date; the one that does is mapped. + const cfg = autoDetectConfig( + withHeader("Date de production;Description;Date;Montant", [ + "N/A;EPICERIE METRO SAINTE-FOY;05/01/2025;-84,32", + "N/A;DEPOT PAIE EMPLOYEUR;15/01/2025;1250,00", + "N/A;HYDRO QUEBEC PREAUTORISE;18/01/2025;-142,18", + ]) + )!; + expect(cfg.columnMapping.date).toBe(2); + expect(cfg.columnMapping.amount).toBe(3); + }); + + it("maps a Solde column rather than nothing when it is all there is", () => { + // The lexical exclusion of a balance column must never empty the candidate + // list: a mapping the user can fix beats a file that cannot be configured. + const cfg = autoDetectConfig( + withHeader("Date;Description;Solde", SIGNED_ROWS) + )!; + expect(cfg.amountMode).toBe("single"); + expect(cfg.columnMapping.amount).toBe(2); + }); +}); + +describe("autoDetectConfig — description column read from the labels (#327)", () => { + it("prefers the labelled column over the longest one", () => { + const cfg = autoDetectConfig( + withHeader("Date;Libelle;Note;Montant", [ + "05/01/2025;EPICERIE METRO;une note nettement plus longue que le libelle;-84,32", + "15/01/2025;DEPOT PAIE;une autre note nettement plus longue encore ici;1250,00", + "18/01/2025;HYDRO QUEBEC;troisieme note nettement plus longue que tout;-142,18", + ]) + )!; + expect(cfg.columnMapping.description).toBe(1); + }); + + it("falls back to the longest column when no label names one", () => { + const cfg = autoDetectConfig( + withHeader("Date;Ref;Note;Montant", [ + "05/01/2025;A1;une note nettement plus longue que la reference;-84,32", + "15/01/2025;B2;une autre note nettement plus longue encore ici;1250,00", + "18/01/2025;C3;troisieme note nettement plus longue que tout;-142,18", + ]) + )!; + expect(cfg.columnMapping.description).toBe(2); }); }); diff --git a/src/utils/csvAutoDetect.ts b/src/utils/csvAutoDetect.ts index e6bd16a..79a5c5f 100644 --- a/src/utils/csvAutoDetect.ts +++ b/src/utils/csvAutoDetect.ts @@ -1,6 +1,15 @@ import Papa from "papaparse"; import { parseDate } from "./dateParser"; import { parseFrenchAmount } from "./amountParser"; +import { + CREDIT_INDICATOR_TOKENS, + DEBIT_INDICATOR_TOKENS, + matchHeaderColumn, + matchTransactionHeaders, + MIN_HEADER_ROLE_MATCHES, + normalizeHeaderCell, + type LexicalHeaderMap, +} from "./headerDictionary"; import type { ColumnMapping, AmountMode, SignConvention } from "../shared/types"; export interface AutoDetectResult { @@ -13,6 +22,26 @@ export interface AutoDetectResult { signConvention: SignConvention; } +/** + * i18n key of a file detection recognised but REFUSES to configure. Refusing is + * a feature: the alternative for this shape is a config that imports every + * debit as income (see `findDirectionIndicatorColumn`). + */ +export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat"; + +/** + * The full-fidelity outcome of detection. + * + * `autoDetectConfig` keeps its `AutoDetectResult | null` shape for the callers + * that only need the configuration, but null cannot say WHY: "I could not read + * this file" and "I read this file and it is a format this version imports + * backwards" call for different messages. `detectImportFormat` separates them. + */ +export type AutoDetectOutcome = + | { status: "ok"; config: AutoDetectResult } + | { status: "rejected"; reason: AutoDetectRejectionKey } + | { status: "failed" }; + const DATE_FORMATS = [ "DD/MM/YYYY", "MM/DD/YYYY", @@ -25,6 +54,12 @@ const DATE_FORMATS = [ const DELIMITERS = [",", ";", "\t"]; +/** Every token a direction-indicator column may hold, both directions. */ +const DIRECTION_INDICATOR_TOKENS = new Set([ + ...DEBIT_INDICATOR_TOKENS, + ...CREDIT_INDICATOR_TOKENS, +]); + /** * Detect and unwrap Desjardins-style CSVs where each entire line is * wrapped in quotes with "" escaping inside. @@ -52,20 +87,41 @@ export function preprocessQuotedCSV(content: string): string { /** * Analyze raw CSV content and return a suggested configuration, - * or null if detection fails. + * or null if detection fails OR the file is a refused format. + * + * Callers that must tell those two apart — the wizard, which owes the user a + * message — use `detectImportFormat` instead. */ export function autoDetectConfig(rawContent: string): AutoDetectResult | null { + const outcome = detectImportFormat(rawContent); + return outcome.status === "ok" ? outcome.config : null; +} + +/** + * Analyze raw CSV content and return a suggested configuration, the reason the + * file is refused, or a plain failure. + * + * The pipeline is unchanged in its bones — delimiter, preamble, header, date, + * numeric columns, amount mode — with a LEXICAL layer in front of it: when the + * file has a header row, `matchTransactionHeaders` reads its labels and the + * labels win. Every lexical hint is a preference, never a constraint: a label + * naming a column the data contradicts (a "Date" column nothing parses as a + * date, a "Solde" column that is the only amount candidate left) is dropped and + * the shape heuristic decides, exactly as it did before #327. + */ +export function detectImportFormat(rawContent: string): AutoDetectOutcome { + const failed: AutoDetectOutcome = { status: "failed" }; const content = preprocessQuotedCSV(rawContent); const lines = content.split(/\r?\n/).filter((l) => l.trim()); - if (lines.length < 2) return null; + if (lines.length < 2) return failed; // Step 1: Detect delimiter const delimiter = detectDelimiter(lines.slice(0, 10)); - if (!delimiter) return null; + if (!delimiter) return failed; const parsed = Papa.parse(content, { delimiter, skipEmptyLines: true }); const data = parsed.data as string[][]; - if (data.length < 2) return null; + if (data.length < 2) return failed; // Step 1b: Detect preamble lines to skip // Find the expected column count (most frequent count > 1) @@ -92,20 +148,24 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null { } const effectiveData = data.slice(skipLines); - if (effectiveData.length < 2) return null; + if (effectiveData.length < 2) return failed; // Step 2: Detect header const hasHeader = detectHeader(effectiveData[0]); + // Step 2b: Read the header labels. A headerless file yields no map at all, + // which is the explicit fall-back: every step below then runs on shape only. + const lexical = hasHeader ? matchTransactionHeaders(effectiveData[0]) : null; + const dataStartIdx = hasHeader ? 1 : 0; const sampleRows = effectiveData.slice(dataStartIdx, dataStartIdx + 20); - if (sampleRows.length === 0) return null; + if (sampleRows.length === 0) return failed; const colCount = Math.max(...effectiveData.slice(0, 10).map((r) => r.length)); // Step 3: Detect date column + format - const dateResult = detectDateColumn(sampleRows, colCount); - if (!dateResult) return null; + const dateResult = detectDateColumn(sampleRows, colCount, lexical?.date); + if (!dateResult) return failed; // Step 3b: Find ALL date-like columns (to exclude from amount candidates) const dateLikeCols = new Set(); @@ -129,23 +189,51 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null { // Step 4: Detect numeric columns const numericCols = detectNumericColumns(sampleRows, colCount); - // Step 5: Detect balance columns and exclude them + date-like columns + // Step 5: Detect balance columns and exclude them + date-like columns. + // A column LABELLED "Solde"/"Balance" is excluded too — the arithmetic test + // needs three consecutive rows and a matching amount column to fire, so a + // short file or a statement whose balance does not reconcile keeps its + // running balance in the running for the amount column. The exclusion is + // dropped if it would leave nothing to map (see `narrowCandidates`). const balanceCols = detectBalanceColumns(sampleRows, numericCols); - const amountCandidates = numericCols.filter( + const shapeCandidates = numericCols.filter( (c) => !balanceCols.has(c) && !dateLikeCols.has(c) ); + const amountCandidates = narrowCandidates( + shapeCandidates, + (c) => c !== lexical?.balance + ); // Step 6: Detect description column const descriptionCol = detectDescriptionColumn( sampleRows, colCount, dateResult.column, - new Set([...numericCols, ...dateLikeCols]) + new Set([...numericCols, ...dateLikeCols]), + lexical?.description ); // Step 7: Determine amount mode - const amountResult = detectAmountMode(sampleRows, amountCandidates); - if (!amountResult) return null; + const amountResult = detectAmountMode(sampleRows, amountCandidates, lexical); + if (!amountResult) return failed; + + // Step 7b: Refuse the third amount format — unsigned magnitudes plus a + // neighbouring D/C column. Nothing here reads that column, so the file would + // be configured as `positive_expense` and every credit imported as an + // expense. Detected, refused, and left to a future `absolute_indicator` mode. + if (amountResult.mode === "single") { + const indicatorCol = findDirectionIndicatorColumn( + sampleRows, + amountResult.amountCol, + colCount + ); + if (indicatorCol !== null) { + return { + status: "rejected", + reason: "import.errors.absoluteIndicatorFormat", + }; + } + } const mapping: ColumnMapping = { date: dateResult.column, @@ -163,16 +251,35 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null { } return { - delimiter, - hasHeader, - skipLines, - dateFormat: dateResult.format, - columnMapping: mapping, - amountMode: amountResult.mode, - signConvention, + status: "ok", + config: { + delimiter, + hasHeader, + skipLines, + dateFormat: dateResult.format, + columnMapping: mapping, + amountMode: amountResult.mode, + signConvention, + }, }; } +/** + * Apply a lexical restriction to a candidate list, unless it would empty it. + * + * The lexical layer must never be able to turn "detected, possibly wrong" into + * "detected nothing": a file whose only amount column happens to be labelled + * `Solde` is still importable, and a user can fix a mapping far more easily + * than a refusal to configure. + */ +function narrowCandidates( + candidates: number[], + keep: (col: number) => boolean +): number[] { + const narrowed = candidates.filter(keep); + return narrowed.length > 0 ? narrowed : candidates; +} + function detectDelimiter(lines: string[]): string | null { let bestDelimiter: string | null = null; let bestScore = 0; @@ -211,6 +318,24 @@ function detectDelimiter(lines: string[]): string | null { return bestDelimiter; } +/** + * Decide whether the first row is a header. + * + * The shape test — no parseable date, no parseable number — is kept and comes + * first. It cannot classify a header whose cells carry bare numbers (a column + * literally titled `2025`), and treating that row as data costs one lost + * transaction plus a mapping computed from a sample that starts with a label + * row. So a row the shape test rejects ONLY because of a number is put to the + * lexical layer: two named roles in one row is a header. + * + * The date test stays absolute. A row carrying a parseable date is data, no + * matter what its other cells spell — that is the guard that keeps a + * transaction like `DEPOT PAIE EMPLOYEUR` (which contains the credit keyword + * `depot`) from being swallowed as a header. + * + * The holdings flow calls this too. Its headers name no transaction role, so + * the lexical clause never fires there. + */ function detectHeader(firstRow: string[]): boolean { // A header row typically has no parseable dates and no parseable numbers let hasDate = false; @@ -234,33 +359,59 @@ function detectHeader(firstRow: string[]): boolean { } } - return !hasDate && !hasNumber; + if (hasDate) return false; + if (!hasNumber) return true; + + return matchTransactionHeaders(firstRow).roleCount >= MIN_HEADER_ROLE_MATCHES; } +/** + * Pick the date column and its format. + * + * `preferred` is the column the header labels name. It wins only if the data + * agrees — same 0.8 parse rate the shape scan demands — so a file whose "Date + * de production" column holds no date falls back to the best-parsing column + * rather than mapping a label nothing supports. Its real job is arbitrating + * between several equally parseable date columns (`Date de transaction` vs + * `Date comptable`), which the rate alone cannot do. + */ function detectDateColumn( rows: string[][], - colCount: number + colCount: number, + preferred?: number | null ): { column: number; format: string } | null { + const rateOf = (col: number, fmt: string): number => { + let success = 0; + let total = 0; + for (const row of rows) { + const cell = row[col]?.trim(); + if (!cell) continue; + total++; + if (parseDate(cell, fmt)) success++; + } + return total === 0 ? 0 : success / total; + }; + + if (preferred !== undefined && preferred !== null && preferred < colCount) { + let bestFormat = ""; + let bestRate = 0; + for (const fmt of DATE_FORMATS) { + const rate = rateOf(preferred, fmt); + if (rate > bestRate) { + bestRate = rate; + bestFormat = fmt; + } + } + if (bestRate >= 0.8) return { column: preferred, format: bestFormat }; + } + let bestCol = -1; let bestFormat = ""; let bestRate = 0; for (let col = 0; col < colCount; col++) { for (const fmt of DATE_FORMATS) { - let success = 0; - let total = 0; - - for (const row of rows) { - const cell = row[col]?.trim(); - if (!cell) continue; - total++; - if (parseDate(cell, fmt)) { - success++; - } - } - - if (total === 0) continue; - const rate = success / total; + const rate = rateOf(col, fmt); if (rate > bestRate) { bestRate = rate; bestCol = col; @@ -401,12 +552,32 @@ function detectBalanceColumns( return balanceCols; } +/** + * Pick the description column: the one the header labels name, else the + * longest-on-average text column. + * + * The label is only honoured for a column the shape test would have been + * allowed to pick at all — not the date column, not a numeric one. A file + * labelling its amount column `Détail du montant` therefore cannot end up with + * its amounts in the description. + */ function detectDescriptionColumn( rows: string[][], colCount: number, dateCol: number, - numericCols: Set + numericCols: Set, + preferred?: number | null ): number { + if ( + preferred !== undefined && + preferred !== null && + preferred < colCount && + preferred !== dateCol && + !numericCols.has(preferred) + ) { + return preferred; + } + let bestCol = 0; let bestAvgLen = 0; @@ -449,7 +620,8 @@ type AmountModeResult = SingleAmountResult | DebitCreditResult; function detectAmountMode( rows: string[][], - amountCandidates: number[] + amountCandidates: number[], + lexical: LexicalHeaderMap | null ): AmountModeResult | null { if (amountCandidates.length === 0) return null; @@ -464,16 +636,113 @@ function detectAmountMode( const colB = amountCandidates[b]; if (isSparseComplementary(rows, colA, colB)) { - return { mode: "debit_credit", debitCol: colA, creditCol: colB }; + return orderDebitCredit(colA, colB, lexical); } } } - // No complementary pair found — pick best single amount column - const bestCol = pickBestAmountColumn(rows, amountCandidates); + // No complementary pair found — the labelled amount column if there is one, + // else the best single amount column by shape. + const preferred = lexical?.amount; + const bestCol = + preferred !== undefined && + preferred !== null && + amountCandidates.includes(preferred) + ? preferred + : pickBestAmountColumn(rows, amountCandidates); return detectSingleAmount(rows, bestCol); } +/** + * Assign the two halves of a complementary pair to debit and credit. + * + * The loop above enumerates `a < b`, so before #327 the LEFTMOST column was the + * debit, always: a file laid out `Date;Description;Crédit;Débit` was mapped + * backwards and every sign of the import came out inverted — silently, since + * the total is merely negated and no aggregate check notices. + * + * The labels decide when they name either half; position remains the fall-back + * for a headerless file or labels the dictionary does not know. Knowing ONE of + * the two is enough: the other column is the other role. + */ +function orderDebitCredit( + colA: number, + colB: number, + lexical: LexicalHeaderMap | null +): DebitCreditResult { + const debitFirst = { + mode: "debit_credit", + debitCol: colA, + creditCol: colB, + } as const; + const creditFirst = { + mode: "debit_credit", + debitCol: colB, + creditCol: colA, + } as const; + + if (lexical?.debit === colA || lexical?.credit === colB) return debitFirst; + if (lexical?.debit === colB || lexical?.credit === colA) return creditFirst; + return debitFirst; +} + +/** + * Find a direction-indicator column next to an all-positive amount column — + * the third amount format, refused rather than imported. + * + * Three conditions, each one narrowing a way to raise a false alarm: + * - the amount column carries no negative value, since a file that signs its + * amounts needs no indicator and reads correctly today; + * - the candidate is IMMEDIATELY next to it, as every export of this shape + * writes the flag beside the magnitude. A file-wide scan would refuse + * perfectly importable files over an unrelated one-letter flag column, and + * a refusal the user cannot work around is worse than a mapping they can; + * - it holds at least two DISTINCT tokens of the D/C alphabet. A column stuck + * on a single value carries no direction, and a statement of pure expenses + * imports correctly as `positive_expense`. + */ +function findDirectionIndicatorColumn( + rows: string[][], + amountCol: number, + colCount: number +): number | null { + let seen = 0; + for (const row of rows) { + const cell = row[amountCol]?.trim(); + if (!cell) continue; + const val = parseFrenchAmount(cell); + if (isNaN(val)) continue; + seen++; + if (val < 0) return null; + } + if (seen === 0) return null; + + for (const col of [amountCol - 1, amountCol + 1]) { + if (col < 0 || col >= colCount) continue; + + const distinct = new Set(); + let nonEmpty = 0; + let inAlphabet = 0; + + for (const row of rows) { + const cell = row[col]?.trim(); + if (!cell) continue; + nonEmpty++; + const token = cell.toLowerCase(); + if (DIRECTION_INDICATOR_TOKENS.has(token)) { + inAlphabet++; + distinct.add(token); + } + } + + if (nonEmpty > 0 && inAlphabet === nonEmpty && distinct.size >= 2) { + return col; + } + } + + return null; +} + /** Pick the best amount column: prefer columns with decimal values (cents). */ function pickBestAmountColumn(rows: string[][], candidates: number[]): number { let bestCol = candidates[0]; @@ -627,40 +896,12 @@ const BOOKCOST_HEADER_KEYWORDS = [ "cout", ]; // Value/market-value columns must never be auto-picked as price or book_cost. +// NOTE `montant` is an EXCLUSION token here and the PRIMARY amount keyword of +// the transaction dictionary (`headerDictionary.ts`) \u2014 which is why the two +// tables are separate modules. `normalizeHeaderCell` and `matchHeaderColumn` +// are the generic part, shared from there and unchanged. const VALUE_HEADER_KEYWORDS = ["value", "valeur", "montant", "marchande"]; -/** Accent-strip + lowercase + keep alphanumerics only (for header matching). */ -function normalizeHeaderCell(s: string): string { - return (s ?? "") - .normalize("NFD") - .replace(/[\u0300-\u036f]/g, "") - .toLowerCase() - .replace(/[^a-z0-9]/g, ""); -} - -/** - * Find the first unused column whose normalized header contains one of the - * keywords (keywords tried in priority order). `exclude` skips columns whose - * header contains any excluded token (e.g. a "value" column for price). - */ -function matchHeaderColumn( - normalizedHeaders: string[], - keywords: string[], - used: Set, - exclude: string[] = [] -): number | null { - for (const kw of keywords) { - for (let i = 0; i < normalizedHeaders.length; i++) { - if (used.has(i)) continue; - const h = normalizedHeaders[i]; - if (!h) continue; - if (exclude.some((ex) => h.includes(ex))) continue; - if (h.includes(kw)) return i; - } - } - return null; -} - /** Pick the shortest-average-length non-numeric, unused text column (symbols * are short tokens; a name/description column is longer). */ function pickSymbolColumn( diff --git a/src/utils/headerDictionary.test.ts b/src/utils/headerDictionary.test.ts new file mode 100644 index 0000000..5de34b1 --- /dev/null +++ b/src/utils/headerDictionary.test.ts @@ -0,0 +1,160 @@ +// headerDictionary — the lexical layer of transaction-CSV detection (#327). +// +// `csvAutoDetect.test.ts` covers what the dictionary DOES to a detected +// configuration; this file covers the dictionary itself: every keyword it +// claims to know, the exclusions that keep a label from claiming the wrong +// column, and the mute row that hands control back to the shape heuristics. + +import { describe, it, expect } from "vitest"; +import { + matchHeaderColumn, + matchTransactionHeaders, + normalizeHeaderCell, + MIN_HEADER_ROLE_MATCHES, +} from "./headerDictionary"; + +describe("normalizeHeaderCell (#245, shared by #327)", () => { + it("strips accents, case and punctuation", () => { + expect(normalizeHeaderCell("Débit")).toBe("debit"); + expect(normalizeHeaderCell("Libellé")).toBe("libelle"); + expect(normalizeHeaderCell("Date de l'opération")).toBe("datedeloperation"); + expect(normalizeHeaderCell("Montant ($)")).toBe("montant"); + }); + + it("keeps digits and tolerates an empty or missing cell", () => { + expect(normalizeHeaderCell("Solde 2024")).toBe("solde2024"); + expect(normalizeHeaderCell("")).toBe(""); + expect(normalizeHeaderCell(undefined as unknown as string)).toBe(""); + }); +}); + +describe("matchHeaderColumn (#245, shared by #327)", () => { + const headers = ["date", "libelle", "montant", "solde"]; + + it("returns the first unused column containing a keyword", () => { + expect(matchHeaderColumn(headers, ["montant"], new Set())).toBe(2); + }); + + it("honours keyword priority order over column order", () => { + expect(matchHeaderColumn(headers, ["solde", "date"], new Set())).toBe(3); + }); + + it("skips used and excluded columns, and returns null on no match", () => { + expect(matchHeaderColumn(headers, ["date"], new Set([0]))).toBeNull(); + expect(matchHeaderColumn(headers, ["mont"], new Set(), ["solde"])).toBe(2); + expect(matchHeaderColumn(headers, ["quantite"], new Set())).toBeNull(); + }); +}); + +describe("matchTransactionHeaders — the FR/EN dictionary (#327)", () => { + it("reads a French signed-amount header", () => { + const m = matchTransactionHeaders(["Date", "Description", "Montant", "Solde"]); + expect(m).toEqual({ + date: 0, + description: 1, + amount: 2, + debit: null, + credit: null, + balance: 3, + roleCount: 4, + }); + }); + + it("reads an English signed-amount header", () => { + const m = matchTransactionHeaders(["Date", "Description", "Amount", "Balance"]); + expect(m.amount).toBe(2); + expect(m.balance).toBe(3); + }); + + it.each([ + ["Débit", "debit"], + ["Retrait", "debit"], + ["Déboursé", "debit"], + ["Withdrawal", "debit"], + ["Crédit", "credit"], + ["Dépôt", "credit"], + ["Encaissement", "credit"], + ["Deposit", "credit"], + ] as const)("knows %s as the %s column", (label, role) => { + const m = matchTransactionHeaders(["Date", "Description", label]); + expect(m[role]).toBe(2); + }); + + it.each([ + ["Libellé", 1], + ["Détail", 1], + ["Transaction", 1], + ["Description", 1], + ] as const)("knows %s as the description column", (label, col) => { + expect(matchTransactionHeaders(["Date", label, "Montant"]).description).toBe( + col + ); + }); + + it("resolves a reversed pair by label, not by position", () => { + const m = matchTransactionHeaders(["Date", "Description", "Credit", "Debit"]); + expect(m.debit).toBe(3); + expect(m.credit).toBe(2); + }); + + it("claims a column once, so debit/credit beat the amount keyword", () => { + // Both columns contain `montant`. Resolving debit and credit first is what + // stops `Montant débit` from being read as THE amount column. + const m = matchTransactionHeaders([ + "Date", + "Libelle", + "Montant debit", + "Montant credit", + ]); + expect(m.debit).toBe(2); + expect(m.credit).toBe(3); + expect(m.amount).toBeNull(); + }); + + it("never reads a Solde column as the amount", () => { + const m = matchTransactionHeaders(["Date", "Libelle", "Solde du compte"]); + expect(m.balance).toBe(2); + expect(m.amount).toBeNull(); + }); + + it("never reads a date column as the description", () => { + // `Date de transaction` contains the description keyword `transaction`. + const m = matchTransactionHeaders([ + "Date de transaction", + "Date comptable", + "Montant", + ]); + expect(m.date).toBe(0); + expect(m.description).toBeNull(); + }); + + it("returns a map of nulls for labels it does not know", () => { + const m = matchTransactionHeaders(["Col A", "Col B", "Col C"]); + expect(m).toEqual({ + date: null, + description: null, + amount: null, + debit: null, + credit: null, + balance: null, + roleCount: 0, + }); + }); + + it("scores a data row below the header threshold", () => { + // `DEPOT PAIE EMPLOYEUR` contains the credit keyword `depot` — one role, + // which is exactly why one is not enough to call a row a header. + const m = matchTransactionHeaders([ + "05/01/2025", + "DEPOT PAIE EMPLOYEUR", + "-84,32", + ]); + expect(m.credit).toBe(1); + expect(m.roleCount).toBeLessThan(MIN_HEADER_ROLE_MATCHES); + }); + + it("tolerates empty cells and a ragged row", () => { + expect(matchTransactionHeaders([]).roleCount).toBe(0); + expect(matchTransactionHeaders(["", "Date", ""]).date).toBe(1); + }); +}); diff --git a/src/utils/headerDictionary.ts b/src/utils/headerDictionary.ts new file mode 100644 index 0000000..c66a5c4 --- /dev/null +++ b/src/utils/headerDictionary.ts @@ -0,0 +1,176 @@ +/** + * Header dictionary for the TRANSACTION import flow (#327). + * + * Detection used to reason on the SHAPE of the data alone — which column parses + * as a date, which one carries the most characters, which pair is sparse and + * complementary. Shape cannot tell a debit column from a credit column, so a + * file laid out `Date;Description;Crédit;Débit` was mapped backwards and every + * sign of the import was inverted. This module is the lexical layer that reads + * the header labels; `csvAutoDetect.ts` puts it in FRONT of the shape + * heuristics and falls back to them whenever the labels are mute (no header + * row, unknown labels). + * + * WHY ITS OWN MODULE. The holdings flow (#245) already matches header labels, + * with its own keyword tables living in `csvAutoDetect.ts`. Merging the two + * tables is not possible: `montant` is an EXCLUSION token there + * (`VALUE_HEADER_KEYWORDS` — a market-value column must never be read as a unit + * price) and it is the PRIMARY amount keyword here. Two flows, two tables. Only + * the two matching helpers are generic, so they live here and the holdings flow + * imports them, unchanged — one direction of dependency, no cycle. + */ + +// Keyword tables, matched as SUBSTRINGS against accent-stripped, +// alphanumeric-only header cells, so `Date de l'opération` matches `date` and +// `Débit ($)` matches `debit`. Bilingual FR + EN, order inside each list is +// priority order. Kept deliberately tight: a keyword that fires on the wrong +// column is worse than a keyword that never fires, because the shape heuristic +// behind it is the behaviour the corpus already froze. +export const DATE_HEADER_KEYWORDS = ["date"]; +export const DESCRIPTION_HEADER_KEYWORDS = [ + "description", + "libelle", + "detail", + "transaction", +]; +export const AMOUNT_HEADER_KEYWORDS = ["montant", "amount"]; +export const DEBIT_HEADER_KEYWORDS = [ + "debit", + "retrait", + "debourse", + "withdrawal", +]; +export const CREDIT_HEADER_KEYWORDS = [ + "credit", + "depot", + "encaissement", + "deposit", +]; +export const BALANCE_HEADER_KEYWORDS = ["solde", "balance"]; + +/** + * A running-balance column is never an amount column, and `Date de + * transaction` is never the description column. Both are exclusion lists rather + * than ordering tricks, because substring matching has no notion of "the best + * match" — the first column containing the keyword wins. + */ +const AMOUNT_EXCLUDE = BALANCE_HEADER_KEYWORDS; +const DESCRIPTION_EXCLUDE = DATE_HEADER_KEYWORDS; + +/** Accent-strip + lowercase + keep alphanumerics only (for header matching). */ +export function normalizeHeaderCell(s: string): string { + return (s ?? "") + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]/g, ""); +} + +/** + * Find the first unused column whose normalized header contains one of the + * keywords (keywords tried in priority order). `exclude` skips columns whose + * header contains any excluded token (e.g. a "value" column for price). + */ +export function matchHeaderColumn( + normalizedHeaders: string[], + keywords: string[], + used: Set, + exclude: string[] = [] +): number | null { + for (const kw of keywords) { + for (let i = 0; i < normalizedHeaders.length; i++) { + if (used.has(i)) continue; + const h = normalizedHeaders[i]; + if (!h) continue; + if (exclude.some((ex) => h.includes(ex))) continue; + if (h.includes(kw)) return i; + } + } + return null; +} + +/** What the labels of one header row say, role by role. */ +export interface LexicalHeaderMap { + date: number | null; + description: number | null; + amount: number | null; + debit: number | null; + credit: number | null; + balance: number | null; + /** Number of roles the row actually named — 0 when the labels are mute. */ + roleCount: number; +} + +/** + * Read one header row through the dictionary. + * + * Roles are resolved in the order balance -> date -> debit -> credit -> amount + * -> description, each claiming its column so a later role cannot steal it. + * That order is what makes `Montant débit` / `Montant crédit` resolve as a + * debit/credit pair instead of the first one being read as THE amount column. + * + * Every field is `null` when no column names that role: a mute row yields a map + * of nulls and `roleCount: 0`, which is the caller's signal to fall back to the + * shape heuristics. + */ +export function matchTransactionHeaders( + headerRow: readonly string[] +): LexicalHeaderMap { + const normalized = headerRow.map(normalizeHeaderCell); + const used = new Set(); + + const claim = (col: number | null): number | null => { + if (col !== null) used.add(col); + return col; + }; + + const balance = claim( + matchHeaderColumn(normalized, BALANCE_HEADER_KEYWORDS, used) + ); + const date = claim(matchHeaderColumn(normalized, DATE_HEADER_KEYWORDS, used)); + const debit = claim( + matchHeaderColumn(normalized, DEBIT_HEADER_KEYWORDS, used, AMOUNT_EXCLUDE) + ); + const credit = claim( + matchHeaderColumn(normalized, CREDIT_HEADER_KEYWORDS, used, AMOUNT_EXCLUDE) + ); + const amount = claim( + matchHeaderColumn(normalized, AMOUNT_HEADER_KEYWORDS, used, AMOUNT_EXCLUDE) + ); + const description = claim( + matchHeaderColumn( + normalized, + DESCRIPTION_HEADER_KEYWORDS, + used, + DESCRIPTION_EXCLUDE + ) + ); + + const roleCount = [date, description, amount, debit, credit, balance].filter( + (c) => c !== null + ).length; + + return { date, description, amount, debit, credit, balance, roleCount }; +} + +/** + * How many named roles a row must carry before the lexical layer is allowed to + * call it a header row on its own. + * + * Two, not one: a description cell like `DEPOT PAIE EMPLOYEUR` contains + * `depot`, so a single match proves nothing. Two distinct roles in one row is a + * shape a data row does not produce, and the caller keeps the date test in + * front of this check anyway. + */ +export const MIN_HEADER_ROLE_MATCHES = 2; + +/** + * Tokens of a direction-indicator column: the third amount format, where the + * amount column holds unsigned magnitudes and a neighbouring column says which + * way each row goes. `D`/`C` in French exports, `DB`/`CR` in some English ones. + * + * The format is refused rather than imported (spec decision) — see + * `csvAutoDetect.ts`. Reading it correctly is a separate feature; reading it + * wrong imports every debit as income. + */ +export const DEBIT_INDICATOR_TOKENS = ["d", "db"]; +export const CREDIT_INDICATOR_TOKENS = ["c", "cr"];