feat(import): detect transaction columns by header label #337
7 changed files with 961 additions and 127 deletions
|
|
@ -41,7 +41,7 @@ import {
|
||||||
} from "../services/importConfigTemplateService";
|
} from "../services/importConfigTemplateService";
|
||||||
import {
|
import {
|
||||||
preprocessQuotedCSV,
|
preprocessQuotedCSV,
|
||||||
autoDetectConfig as runAutoDetect,
|
detectImportFormat as runAutoDetect,
|
||||||
} from "../utils/csvAutoDetect";
|
} from "../utils/csvAutoDetect";
|
||||||
import {
|
import {
|
||||||
detectAmountSeparators,
|
detectAmountSeparators,
|
||||||
|
|
@ -868,9 +868,10 @@ export function useImportWizard() {
|
||||||
encoding: state.sourceConfig.encoding,
|
encoding: state.sourceConfig.encoding,
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = runAutoDetect(content);
|
const outcome = runAutoDetect(content);
|
||||||
|
|
||||||
if (result) {
|
if (outcome.status === "ok") {
|
||||||
|
const result = outcome.config;
|
||||||
const newConfig = {
|
const newConfig = {
|
||||||
...state.sourceConfig,
|
...state.sourceConfig,
|
||||||
delimiter: result.delimiter,
|
delimiter: result.delimiter,
|
||||||
|
|
@ -893,9 +894,16 @@ export function useImportWizard() {
|
||||||
newConfig.hasHeader
|
newConfig.hasHeader
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
|
// A refused format states WHY (#327): "I cannot read this file" and
|
||||||
|
// "this file is a shape this version would import backwards" are not
|
||||||
|
// the same news, and only the second one is the app's own limitation.
|
||||||
|
// `SET_ERROR` clears the loading flag on its own.
|
||||||
dispatch({
|
dispatch({
|
||||||
type: "SET_ERROR",
|
type: "SET_ERROR",
|
||||||
payload: "Auto-detection failed. Please configure manually.",
|
payload:
|
||||||
|
outcome.status === "rejected"
|
||||||
|
? outcome.reason
|
||||||
|
: "import.errors.autoDetectFailed",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|
|
||||||
|
|
@ -189,7 +189,9 @@
|
||||||
"errors": {
|
"errors": {
|
||||||
"unsupportedAmountMode": "The amount mode saved for this source is not recognized. Reconfigure the source before importing.",
|
"unsupportedAmountMode": "The amount mode saved for this source is not recognized. Reconfigure the source before importing.",
|
||||||
"unsupportedSignConvention": "The sign convention saved for this source is not recognized. Reconfigure the source before importing.",
|
"unsupportedSignConvention": "The sign convention saved for this source is not recognized. Reconfigure the source before importing.",
|
||||||
"invalidColumnMapping": "The column mapping saved for this source cannot be read. Reconfigure the source before importing."
|
"invalidColumnMapping": "The column mapping saved for this source cannot be read. Reconfigure the source before importing.",
|
||||||
|
"absoluteIndicatorFormat": "This file uses positive amounts with a separate column giving the direction (D for debit, C for credit). That format is not supported yet: importing it now would record every deposit as an expense. Export the statement with signed amounts, or with separate debit and credit columns.",
|
||||||
|
"autoDetectFailed": "Auto-detection could not read this file. Configure the format manually."
|
||||||
},
|
},
|
||||||
"rowErrors": {
|
"rowErrors": {
|
||||||
"invalidDate": "Unreadable date",
|
"invalidDate": "Unreadable date",
|
||||||
|
|
|
||||||
|
|
@ -189,7 +189,9 @@
|
||||||
"errors": {
|
"errors": {
|
||||||
"unsupportedAmountMode": "Le mode de montant enregistré pour cette source n'est pas reconnu. Reconfigurez la source avant d'importer.",
|
"unsupportedAmountMode": "Le mode de montant enregistré pour cette source n'est pas reconnu. Reconfigurez la source avant d'importer.",
|
||||||
"unsupportedSignConvention": "La convention de signe enregistrée pour cette source n'est pas reconnue. Reconfigurez la source avant d'importer.",
|
"unsupportedSignConvention": "La convention de signe enregistrée pour cette source n'est pas reconnue. Reconfigurez la source avant d'importer.",
|
||||||
"invalidColumnMapping": "Le mapping de colonnes enregistré pour cette source est illisible. Reconfigurez la source avant d'importer."
|
"invalidColumnMapping": "Le mapping de colonnes enregistré pour cette source est illisible. Reconfigurez la source avant d'importer.",
|
||||||
|
"absoluteIndicatorFormat": "Ce fichier utilise des montants positifs accompagnés d'une colonne indiquant le sens (D pour débit, C pour crédit). Ce format n'est pas encore pris en charge : l'importer maintenant enregistrerait chaque dépôt comme une dépense. Exportez le relevé avec des montants signés ou avec deux colonnes débit et crédit.",
|
||||||
|
"autoDetectFailed": "La détection automatique n'a pas pu lire ce fichier. Configurez le format manuellement."
|
||||||
},
|
},
|
||||||
"rowErrors": {
|
"rowErrors": {
|
||||||
"invalidDate": "Date illisible",
|
"invalidDate": "Date illisible",
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ import {
|
||||||
autoDetectHoldingColumns,
|
autoDetectHoldingColumns,
|
||||||
analyzeHoldingsCsv,
|
analyzeHoldingsCsv,
|
||||||
autoDetectConfig,
|
autoDetectConfig,
|
||||||
|
detectImportFormat,
|
||||||
preprocessQuotedCSV,
|
preprocessQuotedCSV,
|
||||||
} from "./csvAutoDetect";
|
} from "./csvAutoDetect";
|
||||||
import type { DecimalSeparator } from "./amountParser";
|
import type { DecimalSeparator } from "./amountParser";
|
||||||
|
|
@ -27,6 +28,8 @@ import {
|
||||||
readCsvFixture,
|
readCsvFixture,
|
||||||
type CsvFixtureName,
|
type CsvFixtureName,
|
||||||
} from "../__fixtures__/csv";
|
} from "../__fixtures__/csv";
|
||||||
|
import fr from "../i18n/locales/fr.json";
|
||||||
|
import en from "../i18n/locales/en.json";
|
||||||
|
|
||||||
describe("autoDetectHoldingColumns (#245)", () => {
|
describe("autoDetectHoldingColumns (#245)", () => {
|
||||||
it("detects symbol/quantity/price/book_cost from EN headers", () => {
|
it("detects symbol/quantity/price/book_cost from EN headers", () => {
|
||||||
|
|
@ -220,12 +223,23 @@ describe("corpus integrity (#326)", () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("detects a configuration for every fixture (none returns null)", () => {
|
it("detects a configuration for every fixture but the refused one", () => {
|
||||||
|
// `absolute-indicator` left this list in #327: it is the one shape
|
||||||
|
// detection now REFUSES instead of configuring, and it says so — the case
|
||||||
|
// below asserts the reason rather than a bare null.
|
||||||
for (const name of CSV_FIXTURE_NAMES) {
|
for (const name of CSV_FIXTURE_NAMES) {
|
||||||
|
if (name === "absolute-indicator") continue;
|
||||||
expect(autoDetectConfig(readCsvFixture(name)), name).not.toBeNull();
|
expect(autoDetectConfig(readCsvFixture(name)), name).not.toBeNull();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("never fails silently: every fixture is a config or a stated reason", () => {
|
||||||
|
for (const name of CSV_FIXTURE_NAMES) {
|
||||||
|
const outcome = detectImportFormat(readCsvFixture(name));
|
||||||
|
expect(outcome.status, name).not.toBe("failed");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("runs the production rule, with no mirror left to drift (#325)", () => {
|
it("runs the production rule, with no mirror left to drift (#325)", () => {
|
||||||
// The guard that used to live here pinned five expressions of
|
// The guard that used to live here pinned five expressions of
|
||||||
// `parseFilesInternal` so the hand-written mirror could not drift. Both are
|
// `parseFilesInternal` so the hand-written mirror could not drift. Both are
|
||||||
|
|
@ -301,36 +315,42 @@ describe("autoDetectConfig — debit/credit pair (#326)", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("autoDetectConfig — KNOWN DEFECT: debit/credit order guessed by position (#326, fixed by #328)", () => {
|
describe("autoDetectConfig — debit/credit order read from the labels (#326, fixed by #327)", () => {
|
||||||
// `detectAmountMode` assigns the LEFTMOST column of a complementary pair to
|
// FIXED. `detectAmountMode` used to assign the LEFTMOST column of a
|
||||||
// `debitAmount` and the rightmost to `creditAmount` (csvAutoDetect.ts:461-470).
|
// complementary pair to `debitAmount` and the rightmost to `creditAmount`
|
||||||
// It never looks at the header labels. A file laid out `Credit;Debit` is
|
// (the loop enumerates `a < b`), without ever looking at the header labels.
|
||||||
// therefore mapped backwards, and every sign in the import is inverted.
|
// A file laid out `Credit;Debit` was mapped backwards and every sign of the
|
||||||
|
// import came out inverted.
|
||||||
//
|
//
|
||||||
// #328 resolves the order from the header dictionary instead of the position.
|
// The pair is still found by shape — sparse and complementary — but which
|
||||||
|
// half is the debit now comes from the dictionary (`orderDebitCredit`).
|
||||||
|
|
||||||
it("maps Credit to debitAmount and Debit to creditAmount", () => {
|
it("maps Debit to debitAmount and Credit to creditAmount", () => {
|
||||||
const cfg = autoDetectConfig(readCsvFixture("debit-credit-reversed"))!;
|
const cfg = autoDetectConfig(readCsvFixture("debit-credit-reversed"))!;
|
||||||
// Column 2 is labelled "Credit" and column 3 "Debit" in the fixture.
|
// Column 2 is labelled "Credit" and column 3 "Debit" in the fixture.
|
||||||
expect(cfg.columnMapping.debitAmount).toBe(2); // DEFECT — should be 3
|
expect(cfg.columnMapping.debitAmount).toBe(3);
|
||||||
expect(cfg.columnMapping.creditAmount).toBe(3); // DEFECT — should be 2
|
expect(cfg.columnMapping.creditAmount).toBe(2);
|
||||||
expect(cfg.amountMode).toBe("debit_credit");
|
expect(cfg.amountMode).toBe("debit_credit");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("inverts every sign end to end — groceries become income", () => {
|
it("parses end to end to the reference amounts", () => {
|
||||||
expect(amountsOf("debit-credit-reversed")).toEqual(
|
expect(amountsOf("debit-credit-reversed")).toEqual(REFERENCE_AMOUNTS);
|
||||||
// DEFECT — should be REFERENCE_AMOUNTS
|
|
||||||
[84.32, -1250, 142.18, 56.75, -300, 6.95]
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("still nets to the same total, which is why the bug hides", () => {
|
it("reads the reversed file identically to its Debit-first twin", () => {
|
||||||
// The signs are wrong one by one, but the sum is merely negated — no
|
// The same six transactions, columns swapped. Nothing but the labels tells
|
||||||
// aggregate check catches it. Only a per-row review does.
|
// them apart, which is the whole point of the lexical layer.
|
||||||
|
expect(amountsOf("debit-credit-reversed")).toEqual(amountsOf("debit-credit"));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("nets to the reference total, no longer to its negation", () => {
|
||||||
|
// While the bug was live the signs were wrong one by one but the sum was
|
||||||
|
// merely negated, so no aggregate check could catch it. That is why the
|
||||||
|
// per-row equality above is the assertion that matters.
|
||||||
const sum = (xs: (number | string)[]) =>
|
const sum = (xs: (number | string)[]) =>
|
||||||
xs.reduce<number>((a, b) => a + (typeof b === "number" ? b : 0), 0);
|
xs.reduce<number>((a, b) => a + (typeof b === "number" ? b : 0), 0);
|
||||||
expect(sum(amountsOf("debit-credit-reversed"))).toBeCloseTo(
|
expect(sum(amountsOf("debit-credit-reversed"))).toBeCloseTo(
|
||||||
-sum(REFERENCE_AMOUNTS),
|
sum(REFERENCE_AMOUNTS),
|
||||||
6
|
6
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
@ -419,11 +439,12 @@ describe("autoDetectConfig — header cell starting with digits (#326, fixed by
|
||||||
// normalizes to "2025Montant"; `parseFloat` returned the 2025 prefix,
|
// normalizes to "2025Montant"; `parseFloat` returned the 2025 prefix,
|
||||||
// `hasNumber` flipped true, and the header row was taken for data.
|
// `hasNumber` flipped true, and the header row was taken for data.
|
||||||
//
|
//
|
||||||
// Link 1's note: "#328 adds a lexical signal to `detectHeader`, and #325
|
// Link 1's note: "#327 adds a lexical signal to `detectHeader`, and #325
|
||||||
// anchors the parser so a numeric prefix no longer reads as a number. Either
|
// anchors the parser so a numeric prefix no longer reads as a number. Either
|
||||||
// fix closes this." #325 landed first — the anchored parser returns NaN for
|
// fix closes this." #325 landed first — the anchored parser returns NaN for
|
||||||
// that cell, so `detectHeader` sees a row with no number and no date.
|
// that cell, so `detectHeader` sees a row with no number and no date.
|
||||||
// #328 keeps its lexical signal for header cells that carry a BARE number.
|
// #327 keeps its lexical signal for header cells that carry a BARE number,
|
||||||
|
// which the anchored parser reads as a number by design (see below).
|
||||||
|
|
||||||
it("recognises the header row", () => {
|
it("recognises the header row", () => {
|
||||||
const cfg = autoDetectConfig(readCsvFixture("header-numeric-label"))!;
|
const cfg = autoDetectConfig(readCsvFixture("header-numeric-label"))!;
|
||||||
|
|
@ -530,41 +551,265 @@ describe("preprocessQuotedCSV + autoDetectConfig — whole-line-quoted file (#32
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("autoDetectConfig — KNOWN DEFECT: absolute amount + D/C indicator (#326, fixed by #328)", () => {
|
describe("autoDetectConfig — absolute amount + D/C indicator, refused (#326, fixed by #327)", () => {
|
||||||
// The third amount mode: one column of unsigned magnitudes plus a
|
// FIXED, by refusing rather than by reading. The third amount mode is one
|
||||||
// single-letter D/C column carrying the direction. Nothing reads that
|
// column of unsigned magnitudes plus a single-letter D/C column carrying the
|
||||||
// column. Detection sees only positive numbers, picks `positive_expense`,
|
// direction. Nothing reads that column: detection saw only positive numbers,
|
||||||
// and every credit is imported as an expense.
|
// picked `positive_expense`, and every credit was imported as an expense —
|
||||||
|
// the file was indistinguishable from `all-positive`, byte for byte of
|
||||||
|
// config.
|
||||||
//
|
//
|
||||||
// #328 must DETECT this shape and REFUSE the file with a dedicated message,
|
// The spec decision is explicit — such a file is refused, not silently
|
||||||
// rather than import it backwards. The spec decision is explicit: refused,
|
// mis-imported — and `amount_mode` carries no CHECK constraint, so reading
|
||||||
// not silently mis-imported.
|
// this shape properly can ship later without a migration.
|
||||||
|
|
||||||
it("ignores the indicator column entirely", () => {
|
it("refuses the file with a dedicated reason", () => {
|
||||||
const cfg = autoDetectConfig(readCsvFixture("absolute-indicator"))!;
|
const outcome = detectImportFormat(readCsvFixture("absolute-indicator"));
|
||||||
|
expect(outcome.status).toBe("rejected");
|
||||||
|
expect(outcome).toEqual({
|
||||||
|
status: "rejected",
|
||||||
|
reason: "import.errors.absoluteIndicatorFormat",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("produces no configuration, so no row can be imported backwards", () => {
|
||||||
|
expect(autoDetectConfig(readCsvFixture("absolute-indicator"))).toBeNull();
|
||||||
|
// Rows 2 and 5 are flagged "C" in the fixture; they used to import as
|
||||||
|
// -1250 and -300.
|
||||||
|
expect(() => amountsOf("absolute-indicator")).toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is no longer indistinguishable from the all-positive fixture", () => {
|
||||||
|
// Same amount column, same absence of negatives — the indicator column is
|
||||||
|
// the only discriminator, and it is now read.
|
||||||
|
const without = autoDetectConfig(readCsvFixture("all-positive"))!;
|
||||||
|
expect(without.signConvention).toBe("positive_expense");
|
||||||
|
expect(autoDetectConfig(readCsvFixture("absolute-indicator"))).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses on the values alone, with no header row to help", () => {
|
||||||
|
const headerless = readCsvFixture("absolute-indicator")
|
||||||
|
.split("\n")
|
||||||
|
.slice(1)
|
||||||
|
.join("\n");
|
||||||
|
expect(detectImportFormat(headerless).status).toBe("rejected");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("carries a translated message in both languages", () => {
|
||||||
|
// The reason is an i18n key, resolved by the error banner
|
||||||
|
// (`ImportPage.tsx`) like every other key the wizard reports.
|
||||||
|
expect(fr.import.errors.absoluteIndicatorFormat.length).toBeGreaterThan(0);
|
||||||
|
expect(en.import.errors.absoluteIndicatorFormat.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is the reason the wizard reports, in place of an English literal", () => {
|
||||||
|
// The hook lives in a `useCallback` and the repository has no jsdom, so
|
||||||
|
// the wiring is asserted on the source — same technique as the guard above.
|
||||||
|
const SRC = readFileSync(
|
||||||
|
resolve(import.meta.dirname, "..", "hooks", "useImportWizard.ts"),
|
||||||
|
"utf-8"
|
||||||
|
);
|
||||||
|
expect(SRC).toContain("detectImportFormat as runAutoDetect");
|
||||||
|
expect(SRC).toContain("outcome.reason");
|
||||||
|
// The banner used to render this literal, in English, whatever the
|
||||||
|
// interface language.
|
||||||
|
expect(SRC).not.toContain("Auto-detection failed. Please configure");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// =============================================================================
|
||||||
|
// Issue #327 — the lexical layer, beyond what the corpus covers
|
||||||
|
// =============================================================================
|
||||||
|
//
|
||||||
|
// The corpus fixtures are frozen files with FR labels. These cases exercise the
|
||||||
|
// clauses they cannot: EN labels, a header row the shape test alone cannot
|
||||||
|
// classify, and the fall-back to shape when the labels say nothing.
|
||||||
|
|
||||||
|
/** Build a semicolon CSV from a header line and the standard six rows. */
|
||||||
|
function withHeader(header: string, rows: string[]): string {
|
||||||
|
return [header, ...rows].join("\n") + "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
const SIGNED_ROWS = [
|
||||||
|
"05/01/2025;EPICERIE METRO SAINTE-FOY;-84,32",
|
||||||
|
"15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00",
|
||||||
|
"18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,18",
|
||||||
|
"22/01/2025;RESTAURANT LE BISTRO;-56,75",
|
||||||
|
"27/01/2025;VIREMENT RECU;300,00",
|
||||||
|
"31/01/2025;FRAIS MENSUELS;-6,95",
|
||||||
|
];
|
||||||
|
|
||||||
|
const PAIR_ROWS = [
|
||||||
|
"05/01/2025;EPICERIE METRO SAINTE-FOY;;84,32",
|
||||||
|
"15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;",
|
||||||
|
"18/01/2025;HYDRO QUEBEC PREAUTORISE;;142,18",
|
||||||
|
"22/01/2025;RESTAURANT LE BISTRO;;56,75",
|
||||||
|
"27/01/2025;VIREMENT RECU;300,00;",
|
||||||
|
"31/01/2025;FRAIS MENSUELS;;6,95",
|
||||||
|
];
|
||||||
|
|
||||||
|
describe("detectHeader — lexical signal on a bare-number header cell (#327)", () => {
|
||||||
|
// The residual case link 1 named. `2025 Montant` is closed by #325's anchored
|
||||||
|
// parser, but a column titled `2025` outright IS a number by every rule the
|
||||||
|
// parser has, and the shape test reads `hasNumber` as "this row is data".
|
||||||
|
const csv = withHeader("Date;Description;2025", SIGNED_ROWS);
|
||||||
|
|
||||||
|
it("recognises the header row from its labels", () => {
|
||||||
|
const cfg = autoDetectConfig(csv)!;
|
||||||
|
expect(cfg.hasHeader).toBe(true);
|
||||||
|
expect(cfg.columnMapping).toEqual({ date: 0, description: 1, amount: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the label row out of the data", () => {
|
||||||
|
const cfg = autoDetectConfig(csv)!;
|
||||||
|
const rows = Papa.parse(csv, {
|
||||||
|
delimiter: cfg.delimiter,
|
||||||
|
skipEmptyLines: true,
|
||||||
|
}).data as string[][];
|
||||||
|
const parsed = rows
|
||||||
|
.slice(cfg.hasHeader ? 1 : 0)
|
||||||
|
.map((raw) => mapCorpusRow(raw, cfg));
|
||||||
|
expect(parsed).toHaveLength(6);
|
||||||
|
expect(parsed.every((r) => !("error" in r))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("needs two named roles, not one", () => {
|
||||||
|
// A single keyword hit proves nothing: a description cell like `DEPOT PAIE
|
||||||
|
// EMPLOYEUR` contains `depot`. One label plus one number is left to shape.
|
||||||
|
const oneRole = withHeader("Compte;Description;2025", SIGNED_ROWS);
|
||||||
|
expect(autoDetectConfig(oneRole)!.hasHeader).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves a row carrying a real date to the shape test", () => {
|
||||||
|
// The date test stays absolute — a data row is data however its other
|
||||||
|
// cells read, which is what stops a transaction from being eaten as a
|
||||||
|
// header.
|
||||||
|
expect(autoDetectConfig(readCsvFixture("no-header"))!.hasHeader).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("autoDetectConfig — English labels (#327)", () => {
|
||||||
|
it("orders a Withdrawal/Deposit pair from the labels", () => {
|
||||||
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date;Description;Deposit;Withdrawal", PAIR_ROWS)
|
||||||
|
)!;
|
||||||
|
expect(cfg.amountMode).toBe("debit_credit");
|
||||||
|
expect(cfg.columnMapping.debitAmount).toBe(3);
|
||||||
|
expect(cfg.columnMapping.creditAmount).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads an Amount/Balance file without mapping the balance", () => {
|
||||||
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date;Description;Amount;Balance", [
|
||||||
|
"05/01/2025;EPICERIE METRO SAINTE-FOY;-84,32;1415,68",
|
||||||
|
"15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;2665,68",
|
||||||
|
"18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,18;2523,50",
|
||||||
|
])
|
||||||
|
)!;
|
||||||
expect(cfg.amountMode).toBe("single");
|
expect(cfg.amountMode).toBe("single");
|
||||||
expect(cfg.columnMapping.amount).toBe(2);
|
expect(cfg.columnMapping.amount).toBe(2);
|
||||||
// Column 3 ("Sens", values D/C) appears nowhere in the mapping.
|
|
||||||
expect(Object.values(cfg.columnMapping)).not.toContain(3);
|
|
||||||
expect(cfg.signConvention).toBe("positive_expense");
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("imports both credit rows as expenses", () => {
|
it("excludes a balance column the arithmetic cannot recognise", () => {
|
||||||
const amounts = amountsOf("absolute-indicator");
|
// The running-balance test needs the column to reconcile row to row. These
|
||||||
// Rows 2 and 5 are flagged "C" in the fixture.
|
// rows do not — a mid-period export whose opening balance is missing — and
|
||||||
expect(amounts[1]).toBe(-1250); // DEFECT — should be +1250
|
// the balance is the only column carrying cents, which is what
|
||||||
expect(amounts[4]).toBe(-300); // DEFECT — should be +300
|
// `pickBestAmountColumn` goes by.
|
||||||
expect(amounts.every((a) => typeof a === "number" && a < 0)).toBe(true);
|
const rows = [
|
||||||
|
"05/01/2025;EPICERIE METRO SAINTE-FOY;-84,00;1415,68",
|
||||||
|
"15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;9002,11",
|
||||||
|
"18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,00;4477,03",
|
||||||
|
];
|
||||||
|
|
||||||
|
// Labelled with a word the dictionary does not know, shape wins and maps
|
||||||
|
// the balance as the amount.
|
||||||
|
expect(
|
||||||
|
autoDetectConfig(withHeader("Date;Description;Sortie;Cumul", rows))!
|
||||||
|
.columnMapping.amount
|
||||||
|
).toBe(3);
|
||||||
|
|
||||||
|
// Labelled `Solde`, it is excluded and the real amount column is mapped.
|
||||||
|
expect(
|
||||||
|
autoDetectConfig(withHeader("Date;Description;Sortie;Solde", rows))!
|
||||||
|
.columnMapping.amount
|
||||||
|
).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("autoDetectConfig — one labelled half is enough (#327)", () => {
|
||||||
|
it("infers the credit column from the labelled debit", () => {
|
||||||
|
// `Sorties` is not in the dictionary; `Débit` is. Knowing one half names
|
||||||
|
// the other, because the pair has exactly two members.
|
||||||
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date;Description;Sorties;Debit", PAIR_ROWS)
|
||||||
|
)!;
|
||||||
|
expect(cfg.columnMapping.debitAmount).toBe(3);
|
||||||
|
expect(cfg.columnMapping.creditAmount).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("autoDetectConfig — falling back to shape when labels are mute (#327)", () => {
|
||||||
|
it("keeps the positional order when neither half is labelled", () => {
|
||||||
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date;Description;Sorties;Entrees", PAIR_ROWS)
|
||||||
|
)!;
|
||||||
|
// Unknown labels: the leftmost column of the pair stays the debit, exactly
|
||||||
|
// as before #327. The file reads backwards — and that is the honest
|
||||||
|
// outcome, since nothing in it says otherwise.
|
||||||
|
expect(cfg.columnMapping.debitAmount).toBe(2);
|
||||||
|
expect(cfg.columnMapping.creditAmount).toBe(3);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("is currently indistinguishable from the all-positive fixture", () => {
|
it("keeps mapping a headerless file positionally", () => {
|
||||||
// Same detected config, same wrong outcome — which is exactly why #328
|
expect(autoDetectConfig(readCsvFixture("no-header"))!.hasHeader).toBe(false);
|
||||||
// needs the indicator column as its discriminator.
|
expect(amountsOf("no-header")).toEqual(REFERENCE_AMOUNTS);
|
||||||
const withIndicator = autoDetectConfig(readCsvFixture("absolute-indicator"))!;
|
});
|
||||||
const without = autoDetectConfig(readCsvFixture("all-positive"))!;
|
|
||||||
expect(withIndicator.amountMode).toBe(without.amountMode);
|
it("ignores a label the data contradicts", () => {
|
||||||
expect(withIndicator.signConvention).toBe(without.signConvention);
|
// The column labelled `Date` holds no date; the one that does is mapped.
|
||||||
expect(withIndicator.columnMapping).toEqual(without.columnMapping);
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date de production;Description;Date;Montant", [
|
||||||
|
"N/A;EPICERIE METRO SAINTE-FOY;05/01/2025;-84,32",
|
||||||
|
"N/A;DEPOT PAIE EMPLOYEUR;15/01/2025;1250,00",
|
||||||
|
"N/A;HYDRO QUEBEC PREAUTORISE;18/01/2025;-142,18",
|
||||||
|
])
|
||||||
|
)!;
|
||||||
|
expect(cfg.columnMapping.date).toBe(2);
|
||||||
|
expect(cfg.columnMapping.amount).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps a Solde column rather than nothing when it is all there is", () => {
|
||||||
|
// The lexical exclusion of a balance column must never empty the candidate
|
||||||
|
// list: a mapping the user can fix beats a file that cannot be configured.
|
||||||
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date;Description;Solde", SIGNED_ROWS)
|
||||||
|
)!;
|
||||||
|
expect(cfg.amountMode).toBe("single");
|
||||||
|
expect(cfg.columnMapping.amount).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("autoDetectConfig — description column read from the labels (#327)", () => {
|
||||||
|
it("prefers the labelled column over the longest one", () => {
|
||||||
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date;Libelle;Note;Montant", [
|
||||||
|
"05/01/2025;EPICERIE METRO;une note nettement plus longue que le libelle;-84,32",
|
||||||
|
"15/01/2025;DEPOT PAIE;une autre note nettement plus longue encore ici;1250,00",
|
||||||
|
"18/01/2025;HYDRO QUEBEC;troisieme note nettement plus longue que tout;-142,18",
|
||||||
|
])
|
||||||
|
)!;
|
||||||
|
expect(cfg.columnMapping.description).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to the longest column when no label names one", () => {
|
||||||
|
const cfg = autoDetectConfig(
|
||||||
|
withHeader("Date;Ref;Note;Montant", [
|
||||||
|
"05/01/2025;A1;une note nettement plus longue que la reference;-84,32",
|
||||||
|
"15/01/2025;B2;une autre note nettement plus longue encore ici;1250,00",
|
||||||
|
"18/01/2025;C3;troisieme note nettement plus longue que tout;-142,18",
|
||||||
|
])
|
||||||
|
)!;
|
||||||
|
expect(cfg.columnMapping.description).toBe(2);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,15 @@
|
||||||
import Papa from "papaparse";
|
import Papa from "papaparse";
|
||||||
import { parseDate } from "./dateParser";
|
import { parseDate } from "./dateParser";
|
||||||
import { parseFrenchAmount } from "./amountParser";
|
import { parseFrenchAmount } from "./amountParser";
|
||||||
|
import {
|
||||||
|
CREDIT_INDICATOR_TOKENS,
|
||||||
|
DEBIT_INDICATOR_TOKENS,
|
||||||
|
matchHeaderColumn,
|
||||||
|
matchTransactionHeaders,
|
||||||
|
MIN_HEADER_ROLE_MATCHES,
|
||||||
|
normalizeHeaderCell,
|
||||||
|
type LexicalHeaderMap,
|
||||||
|
} from "./headerDictionary";
|
||||||
import type { ColumnMapping, AmountMode, SignConvention } from "../shared/types";
|
import type { ColumnMapping, AmountMode, SignConvention } from "../shared/types";
|
||||||
|
|
||||||
export interface AutoDetectResult {
|
export interface AutoDetectResult {
|
||||||
|
|
@ -13,6 +22,26 @@ export interface AutoDetectResult {
|
||||||
signConvention: SignConvention;
|
signConvention: SignConvention;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* i18n key of a file detection recognised but REFUSES to configure. Refusing is
|
||||||
|
* a feature: the alternative for this shape is a config that imports every
|
||||||
|
* debit as income (see `findDirectionIndicatorColumn`).
|
||||||
|
*/
|
||||||
|
export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The full-fidelity outcome of detection.
|
||||||
|
*
|
||||||
|
* `autoDetectConfig` keeps its `AutoDetectResult | null` shape for the callers
|
||||||
|
* that only need the configuration, but null cannot say WHY: "I could not read
|
||||||
|
* this file" and "I read this file and it is a format this version imports
|
||||||
|
* backwards" call for different messages. `detectImportFormat` separates them.
|
||||||
|
*/
|
||||||
|
export type AutoDetectOutcome =
|
||||||
|
| { status: "ok"; config: AutoDetectResult }
|
||||||
|
| { status: "rejected"; reason: AutoDetectRejectionKey }
|
||||||
|
| { status: "failed" };
|
||||||
|
|
||||||
const DATE_FORMATS = [
|
const DATE_FORMATS = [
|
||||||
"DD/MM/YYYY",
|
"DD/MM/YYYY",
|
||||||
"MM/DD/YYYY",
|
"MM/DD/YYYY",
|
||||||
|
|
@ -25,6 +54,12 @@ const DATE_FORMATS = [
|
||||||
|
|
||||||
const DELIMITERS = [",", ";", "\t"];
|
const DELIMITERS = [",", ";", "\t"];
|
||||||
|
|
||||||
|
/** Every token a direction-indicator column may hold, both directions. */
|
||||||
|
const DIRECTION_INDICATOR_TOKENS = new Set([
|
||||||
|
...DEBIT_INDICATOR_TOKENS,
|
||||||
|
...CREDIT_INDICATOR_TOKENS,
|
||||||
|
]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Detect and unwrap Desjardins-style CSVs where each entire line is
|
* Detect and unwrap Desjardins-style CSVs where each entire line is
|
||||||
* wrapped in quotes with "" escaping inside.
|
* wrapped in quotes with "" escaping inside.
|
||||||
|
|
@ -52,20 +87,41 @@ export function preprocessQuotedCSV(content: string): string {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Analyze raw CSV content and return a suggested configuration,
|
* Analyze raw CSV content and return a suggested configuration,
|
||||||
* or null if detection fails.
|
* or null if detection fails OR the file is a refused format.
|
||||||
|
*
|
||||||
|
* Callers that must tell those two apart — the wizard, which owes the user a
|
||||||
|
* message — use `detectImportFormat` instead.
|
||||||
*/
|
*/
|
||||||
export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
|
export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
|
||||||
|
const outcome = detectImportFormat(rawContent);
|
||||||
|
return outcome.status === "ok" ? outcome.config : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyze raw CSV content and return a suggested configuration, the reason the
|
||||||
|
* file is refused, or a plain failure.
|
||||||
|
*
|
||||||
|
* The pipeline is unchanged in its bones — delimiter, preamble, header, date,
|
||||||
|
* numeric columns, amount mode — with a LEXICAL layer in front of it: when the
|
||||||
|
* file has a header row, `matchTransactionHeaders` reads its labels and the
|
||||||
|
* labels win. Every lexical hint is a preference, never a constraint: a label
|
||||||
|
* naming a column the data contradicts (a "Date" column nothing parses as a
|
||||||
|
* date, a "Solde" column that is the only amount candidate left) is dropped and
|
||||||
|
* the shape heuristic decides, exactly as it did before #327.
|
||||||
|
*/
|
||||||
|
export function detectImportFormat(rawContent: string): AutoDetectOutcome {
|
||||||
|
const failed: AutoDetectOutcome = { status: "failed" };
|
||||||
const content = preprocessQuotedCSV(rawContent);
|
const content = preprocessQuotedCSV(rawContent);
|
||||||
const lines = content.split(/\r?\n/).filter((l) => l.trim());
|
const lines = content.split(/\r?\n/).filter((l) => l.trim());
|
||||||
if (lines.length < 2) return null;
|
if (lines.length < 2) return failed;
|
||||||
|
|
||||||
// Step 1: Detect delimiter
|
// Step 1: Detect delimiter
|
||||||
const delimiter = detectDelimiter(lines.slice(0, 10));
|
const delimiter = detectDelimiter(lines.slice(0, 10));
|
||||||
if (!delimiter) return null;
|
if (!delimiter) return failed;
|
||||||
|
|
||||||
const parsed = Papa.parse(content, { delimiter, skipEmptyLines: true });
|
const parsed = Papa.parse(content, { delimiter, skipEmptyLines: true });
|
||||||
const data = parsed.data as string[][];
|
const data = parsed.data as string[][];
|
||||||
if (data.length < 2) return null;
|
if (data.length < 2) return failed;
|
||||||
|
|
||||||
// Step 1b: Detect preamble lines to skip
|
// Step 1b: Detect preamble lines to skip
|
||||||
// Find the expected column count (most frequent count > 1)
|
// Find the expected column count (most frequent count > 1)
|
||||||
|
|
@ -92,20 +148,24 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
|
||||||
}
|
}
|
||||||
|
|
||||||
const effectiveData = data.slice(skipLines);
|
const effectiveData = data.slice(skipLines);
|
||||||
if (effectiveData.length < 2) return null;
|
if (effectiveData.length < 2) return failed;
|
||||||
|
|
||||||
// Step 2: Detect header
|
// Step 2: Detect header
|
||||||
const hasHeader = detectHeader(effectiveData[0]);
|
const hasHeader = detectHeader(effectiveData[0]);
|
||||||
|
|
||||||
|
// Step 2b: Read the header labels. A headerless file yields no map at all,
|
||||||
|
// which is the explicit fall-back: every step below then runs on shape only.
|
||||||
|
const lexical = hasHeader ? matchTransactionHeaders(effectiveData[0]) : null;
|
||||||
|
|
||||||
const dataStartIdx = hasHeader ? 1 : 0;
|
const dataStartIdx = hasHeader ? 1 : 0;
|
||||||
const sampleRows = effectiveData.slice(dataStartIdx, dataStartIdx + 20);
|
const sampleRows = effectiveData.slice(dataStartIdx, dataStartIdx + 20);
|
||||||
if (sampleRows.length === 0) return null;
|
if (sampleRows.length === 0) return failed;
|
||||||
|
|
||||||
const colCount = Math.max(...effectiveData.slice(0, 10).map((r) => r.length));
|
const colCount = Math.max(...effectiveData.slice(0, 10).map((r) => r.length));
|
||||||
|
|
||||||
// Step 3: Detect date column + format
|
// Step 3: Detect date column + format
|
||||||
const dateResult = detectDateColumn(sampleRows, colCount);
|
const dateResult = detectDateColumn(sampleRows, colCount, lexical?.date);
|
||||||
if (!dateResult) return null;
|
if (!dateResult) return failed;
|
||||||
|
|
||||||
// Step 3b: Find ALL date-like columns (to exclude from amount candidates)
|
// Step 3b: Find ALL date-like columns (to exclude from amount candidates)
|
||||||
const dateLikeCols = new Set<number>();
|
const dateLikeCols = new Set<number>();
|
||||||
|
|
@ -129,23 +189,51 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
|
||||||
// Step 4: Detect numeric columns
|
// Step 4: Detect numeric columns
|
||||||
const numericCols = detectNumericColumns(sampleRows, colCount);
|
const numericCols = detectNumericColumns(sampleRows, colCount);
|
||||||
|
|
||||||
// Step 5: Detect balance columns and exclude them + date-like columns
|
// Step 5: Detect balance columns and exclude them + date-like columns.
|
||||||
|
// A column LABELLED "Solde"/"Balance" is excluded too — the arithmetic test
|
||||||
|
// needs three consecutive rows and a matching amount column to fire, so a
|
||||||
|
// short file or a statement whose balance does not reconcile keeps its
|
||||||
|
// running balance in the running for the amount column. The exclusion is
|
||||||
|
// dropped if it would leave nothing to map (see `narrowCandidates`).
|
||||||
const balanceCols = detectBalanceColumns(sampleRows, numericCols);
|
const balanceCols = detectBalanceColumns(sampleRows, numericCols);
|
||||||
const amountCandidates = numericCols.filter(
|
const shapeCandidates = numericCols.filter(
|
||||||
(c) => !balanceCols.has(c) && !dateLikeCols.has(c)
|
(c) => !balanceCols.has(c) && !dateLikeCols.has(c)
|
||||||
);
|
);
|
||||||
|
const amountCandidates = narrowCandidates(
|
||||||
|
shapeCandidates,
|
||||||
|
(c) => c !== lexical?.balance
|
||||||
|
);
|
||||||
|
|
||||||
// Step 6: Detect description column
|
// Step 6: Detect description column
|
||||||
const descriptionCol = detectDescriptionColumn(
|
const descriptionCol = detectDescriptionColumn(
|
||||||
sampleRows,
|
sampleRows,
|
||||||
colCount,
|
colCount,
|
||||||
dateResult.column,
|
dateResult.column,
|
||||||
new Set([...numericCols, ...dateLikeCols])
|
new Set([...numericCols, ...dateLikeCols]),
|
||||||
|
lexical?.description
|
||||||
);
|
);
|
||||||
|
|
||||||
// Step 7: Determine amount mode
|
// Step 7: Determine amount mode
|
||||||
const amountResult = detectAmountMode(sampleRows, amountCandidates);
|
const amountResult = detectAmountMode(sampleRows, amountCandidates, lexical);
|
||||||
if (!amountResult) return null;
|
if (!amountResult) return failed;
|
||||||
|
|
||||||
|
// Step 7b: Refuse the third amount format — unsigned magnitudes plus a
|
||||||
|
// neighbouring D/C column. Nothing here reads that column, so the file would
|
||||||
|
// be configured as `positive_expense` and every credit imported as an
|
||||||
|
// expense. Detected, refused, and left to a future `absolute_indicator` mode.
|
||||||
|
if (amountResult.mode === "single") {
|
||||||
|
const indicatorCol = findDirectionIndicatorColumn(
|
||||||
|
sampleRows,
|
||||||
|
amountResult.amountCol,
|
||||||
|
colCount
|
||||||
|
);
|
||||||
|
if (indicatorCol !== null) {
|
||||||
|
return {
|
||||||
|
status: "rejected",
|
||||||
|
reason: "import.errors.absoluteIndicatorFormat",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const mapping: ColumnMapping = {
|
const mapping: ColumnMapping = {
|
||||||
date: dateResult.column,
|
date: dateResult.column,
|
||||||
|
|
@ -163,6 +251,8 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
status: "ok",
|
||||||
|
config: {
|
||||||
delimiter,
|
delimiter,
|
||||||
hasHeader,
|
hasHeader,
|
||||||
skipLines,
|
skipLines,
|
||||||
|
|
@ -170,9 +260,26 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
|
||||||
columnMapping: mapping,
|
columnMapping: mapping,
|
||||||
amountMode: amountResult.mode,
|
amountMode: amountResult.mode,
|
||||||
signConvention,
|
signConvention,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply a lexical restriction to a candidate list, unless it would empty it.
|
||||||
|
*
|
||||||
|
* The lexical layer must never be able to turn "detected, possibly wrong" into
|
||||||
|
* "detected nothing": a file whose only amount column happens to be labelled
|
||||||
|
* `Solde` is still importable, and a user can fix a mapping far more easily
|
||||||
|
* than a refusal to configure.
|
||||||
|
*/
|
||||||
|
function narrowCandidates(
|
||||||
|
candidates: number[],
|
||||||
|
keep: (col: number) => boolean
|
||||||
|
): number[] {
|
||||||
|
const narrowed = candidates.filter(keep);
|
||||||
|
return narrowed.length > 0 ? narrowed : candidates;
|
||||||
|
}
|
||||||
|
|
||||||
function detectDelimiter(lines: string[]): string | null {
|
function detectDelimiter(lines: string[]): string | null {
|
||||||
let bestDelimiter: string | null = null;
|
let bestDelimiter: string | null = null;
|
||||||
let bestScore = 0;
|
let bestScore = 0;
|
||||||
|
|
@ -211,6 +318,24 @@ function detectDelimiter(lines: string[]): string | null {
|
||||||
return bestDelimiter;
|
return bestDelimiter;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decide whether the first row is a header.
|
||||||
|
*
|
||||||
|
* The shape test — no parseable date, no parseable number — is kept and comes
|
||||||
|
* first. It cannot classify a header whose cells carry bare numbers (a column
|
||||||
|
* literally titled `2025`), and treating that row as data costs one lost
|
||||||
|
* transaction plus a mapping computed from a sample that starts with a label
|
||||||
|
* row. So a row the shape test rejects ONLY because of a number is put to the
|
||||||
|
* lexical layer: two named roles in one row is a header.
|
||||||
|
*
|
||||||
|
* The date test stays absolute. A row carrying a parseable date is data, no
|
||||||
|
* matter what its other cells spell — that is the guard that keeps a
|
||||||
|
* transaction like `DEPOT PAIE EMPLOYEUR` (which contains the credit keyword
|
||||||
|
* `depot`) from being swallowed as a header.
|
||||||
|
*
|
||||||
|
* The holdings flow calls this too. Its headers name no transaction role, so
|
||||||
|
* the lexical clause never fires there.
|
||||||
|
*/
|
||||||
function detectHeader(firstRow: string[]): boolean {
|
function detectHeader(firstRow: string[]): boolean {
|
||||||
// A header row typically has no parseable dates and no parseable numbers
|
// A header row typically has no parseable dates and no parseable numbers
|
||||||
let hasDate = false;
|
let hasDate = false;
|
||||||
|
|
@ -234,33 +359,59 @@ function detectHeader(firstRow: string[]): boolean {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return !hasDate && !hasNumber;
|
if (hasDate) return false;
|
||||||
|
if (!hasNumber) return true;
|
||||||
|
|
||||||
|
return matchTransactionHeaders(firstRow).roleCount >= MIN_HEADER_ROLE_MATCHES;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pick the date column and its format.
|
||||||
|
*
|
||||||
|
* `preferred` is the column the header labels name. It wins only if the data
|
||||||
|
* agrees — same 0.8 parse rate the shape scan demands — so a file whose "Date
|
||||||
|
* de production" column holds no date falls back to the best-parsing column
|
||||||
|
* rather than mapping a label nothing supports. Its real job is arbitrating
|
||||||
|
* between several equally parseable date columns (`Date de transaction` vs
|
||||||
|
* `Date comptable`), which the rate alone cannot do.
|
||||||
|
*/
|
||||||
function detectDateColumn(
|
function detectDateColumn(
|
||||||
rows: string[][],
|
rows: string[][],
|
||||||
colCount: number
|
colCount: number,
|
||||||
|
preferred?: number | null
|
||||||
): { column: number; format: string } | null {
|
): { column: number; format: string } | null {
|
||||||
|
const rateOf = (col: number, fmt: string): number => {
|
||||||
|
let success = 0;
|
||||||
|
let total = 0;
|
||||||
|
for (const row of rows) {
|
||||||
|
const cell = row[col]?.trim();
|
||||||
|
if (!cell) continue;
|
||||||
|
total++;
|
||||||
|
if (parseDate(cell, fmt)) success++;
|
||||||
|
}
|
||||||
|
return total === 0 ? 0 : success / total;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (preferred !== undefined && preferred !== null && preferred < colCount) {
|
||||||
|
let bestFormat = "";
|
||||||
|
let bestRate = 0;
|
||||||
|
for (const fmt of DATE_FORMATS) {
|
||||||
|
const rate = rateOf(preferred, fmt);
|
||||||
|
if (rate > bestRate) {
|
||||||
|
bestRate = rate;
|
||||||
|
bestFormat = fmt;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (bestRate >= 0.8) return { column: preferred, format: bestFormat };
|
||||||
|
}
|
||||||
|
|
||||||
let bestCol = -1;
|
let bestCol = -1;
|
||||||
let bestFormat = "";
|
let bestFormat = "";
|
||||||
let bestRate = 0;
|
let bestRate = 0;
|
||||||
|
|
||||||
for (let col = 0; col < colCount; col++) {
|
for (let col = 0; col < colCount; col++) {
|
||||||
for (const fmt of DATE_FORMATS) {
|
for (const fmt of DATE_FORMATS) {
|
||||||
let success = 0;
|
const rate = rateOf(col, fmt);
|
||||||
let total = 0;
|
|
||||||
|
|
||||||
for (const row of rows) {
|
|
||||||
const cell = row[col]?.trim();
|
|
||||||
if (!cell) continue;
|
|
||||||
total++;
|
|
||||||
if (parseDate(cell, fmt)) {
|
|
||||||
success++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (total === 0) continue;
|
|
||||||
const rate = success / total;
|
|
||||||
if (rate > bestRate) {
|
if (rate > bestRate) {
|
||||||
bestRate = rate;
|
bestRate = rate;
|
||||||
bestCol = col;
|
bestCol = col;
|
||||||
|
|
@ -401,12 +552,32 @@ function detectBalanceColumns(
|
||||||
return balanceCols;
|
return balanceCols;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pick the description column: the one the header labels name, else the
|
||||||
|
* longest-on-average text column.
|
||||||
|
*
|
||||||
|
* The label is only honoured for a column the shape test would have been
|
||||||
|
* allowed to pick at all — not the date column, not a numeric one. A file
|
||||||
|
* labelling its amount column `Détail du montant` therefore cannot end up with
|
||||||
|
* its amounts in the description.
|
||||||
|
*/
|
||||||
function detectDescriptionColumn(
|
function detectDescriptionColumn(
|
||||||
rows: string[][],
|
rows: string[][],
|
||||||
colCount: number,
|
colCount: number,
|
||||||
dateCol: number,
|
dateCol: number,
|
||||||
numericCols: Set<number>
|
numericCols: Set<number>,
|
||||||
|
preferred?: number | null
|
||||||
): number {
|
): number {
|
||||||
|
if (
|
||||||
|
preferred !== undefined &&
|
||||||
|
preferred !== null &&
|
||||||
|
preferred < colCount &&
|
||||||
|
preferred !== dateCol &&
|
||||||
|
!numericCols.has(preferred)
|
||||||
|
) {
|
||||||
|
return preferred;
|
||||||
|
}
|
||||||
|
|
||||||
let bestCol = 0;
|
let bestCol = 0;
|
||||||
let bestAvgLen = 0;
|
let bestAvgLen = 0;
|
||||||
|
|
||||||
|
|
@ -449,7 +620,8 @@ type AmountModeResult = SingleAmountResult | DebitCreditResult;
|
||||||
|
|
||||||
function detectAmountMode(
|
function detectAmountMode(
|
||||||
rows: string[][],
|
rows: string[][],
|
||||||
amountCandidates: number[]
|
amountCandidates: number[],
|
||||||
|
lexical: LexicalHeaderMap | null
|
||||||
): AmountModeResult | null {
|
): AmountModeResult | null {
|
||||||
if (amountCandidates.length === 0) return null;
|
if (amountCandidates.length === 0) return null;
|
||||||
|
|
||||||
|
|
@ -464,16 +636,113 @@ function detectAmountMode(
|
||||||
const colB = amountCandidates[b];
|
const colB = amountCandidates[b];
|
||||||
|
|
||||||
if (isSparseComplementary(rows, colA, colB)) {
|
if (isSparseComplementary(rows, colA, colB)) {
|
||||||
return { mode: "debit_credit", debitCol: colA, creditCol: colB };
|
return orderDebitCredit(colA, colB, lexical);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// No complementary pair found — pick best single amount column
|
// No complementary pair found — the labelled amount column if there is one,
|
||||||
const bestCol = pickBestAmountColumn(rows, amountCandidates);
|
// else the best single amount column by shape.
|
||||||
|
const preferred = lexical?.amount;
|
||||||
|
const bestCol =
|
||||||
|
preferred !== undefined &&
|
||||||
|
preferred !== null &&
|
||||||
|
amountCandidates.includes(preferred)
|
||||||
|
? preferred
|
||||||
|
: pickBestAmountColumn(rows, amountCandidates);
|
||||||
return detectSingleAmount(rows, bestCol);
|
return detectSingleAmount(rows, bestCol);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Assign the two halves of a complementary pair to debit and credit.
|
||||||
|
*
|
||||||
|
* The loop above enumerates `a < b`, so before #327 the LEFTMOST column was the
|
||||||
|
* debit, always: a file laid out `Date;Description;Crédit;Débit` was mapped
|
||||||
|
* backwards and every sign of the import came out inverted — silently, since
|
||||||
|
* the total is merely negated and no aggregate check notices.
|
||||||
|
*
|
||||||
|
* The labels decide when they name either half; position remains the fall-back
|
||||||
|
* for a headerless file or labels the dictionary does not know. Knowing ONE of
|
||||||
|
* the two is enough: the other column is the other role.
|
||||||
|
*/
|
||||||
|
function orderDebitCredit(
|
||||||
|
colA: number,
|
||||||
|
colB: number,
|
||||||
|
lexical: LexicalHeaderMap | null
|
||||||
|
): DebitCreditResult {
|
||||||
|
const debitFirst = {
|
||||||
|
mode: "debit_credit",
|
||||||
|
debitCol: colA,
|
||||||
|
creditCol: colB,
|
||||||
|
} as const;
|
||||||
|
const creditFirst = {
|
||||||
|
mode: "debit_credit",
|
||||||
|
debitCol: colB,
|
||||||
|
creditCol: colA,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
if (lexical?.debit === colA || lexical?.credit === colB) return debitFirst;
|
||||||
|
if (lexical?.debit === colB || lexical?.credit === colA) return creditFirst;
|
||||||
|
return debitFirst;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find a direction-indicator column next to an all-positive amount column —
|
||||||
|
* the third amount format, refused rather than imported.
|
||||||
|
*
|
||||||
|
* Three conditions, each one narrowing a way to raise a false alarm:
|
||||||
|
* - the amount column carries no negative value, since a file that signs its
|
||||||
|
* amounts needs no indicator and reads correctly today;
|
||||||
|
* - the candidate is IMMEDIATELY next to it, as every export of this shape
|
||||||
|
* writes the flag beside the magnitude. A file-wide scan would refuse
|
||||||
|
* perfectly importable files over an unrelated one-letter flag column, and
|
||||||
|
* a refusal the user cannot work around is worse than a mapping they can;
|
||||||
|
* - it holds at least two DISTINCT tokens of the D/C alphabet. A column stuck
|
||||||
|
* on a single value carries no direction, and a statement of pure expenses
|
||||||
|
* imports correctly as `positive_expense`.
|
||||||
|
*/
|
||||||
|
function findDirectionIndicatorColumn(
|
||||||
|
rows: string[][],
|
||||||
|
amountCol: number,
|
||||||
|
colCount: number
|
||||||
|
): number | null {
|
||||||
|
let seen = 0;
|
||||||
|
for (const row of rows) {
|
||||||
|
const cell = row[amountCol]?.trim();
|
||||||
|
if (!cell) continue;
|
||||||
|
const val = parseFrenchAmount(cell);
|
||||||
|
if (isNaN(val)) continue;
|
||||||
|
seen++;
|
||||||
|
if (val < 0) return null;
|
||||||
|
}
|
||||||
|
if (seen === 0) return null;
|
||||||
|
|
||||||
|
for (const col of [amountCol - 1, amountCol + 1]) {
|
||||||
|
if (col < 0 || col >= colCount) continue;
|
||||||
|
|
||||||
|
const distinct = new Set<string>();
|
||||||
|
let nonEmpty = 0;
|
||||||
|
let inAlphabet = 0;
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const cell = row[col]?.trim();
|
||||||
|
if (!cell) continue;
|
||||||
|
nonEmpty++;
|
||||||
|
const token = cell.toLowerCase();
|
||||||
|
if (DIRECTION_INDICATOR_TOKENS.has(token)) {
|
||||||
|
inAlphabet++;
|
||||||
|
distinct.add(token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nonEmpty > 0 && inAlphabet === nonEmpty && distinct.size >= 2) {
|
||||||
|
return col;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/** Pick the best amount column: prefer columns with decimal values (cents). */
|
/** Pick the best amount column: prefer columns with decimal values (cents). */
|
||||||
function pickBestAmountColumn(rows: string[][], candidates: number[]): number {
|
function pickBestAmountColumn(rows: string[][], candidates: number[]): number {
|
||||||
let bestCol = candidates[0];
|
let bestCol = candidates[0];
|
||||||
|
|
@ -627,40 +896,12 @@ const BOOKCOST_HEADER_KEYWORDS = [
|
||||||
"cout",
|
"cout",
|
||||||
];
|
];
|
||||||
// Value/market-value columns must never be auto-picked as price or book_cost.
|
// Value/market-value columns must never be auto-picked as price or book_cost.
|
||||||
|
// NOTE `montant` is an EXCLUSION token here and the PRIMARY amount keyword of
|
||||||
|
// the transaction dictionary (`headerDictionary.ts`) \u2014 which is why the two
|
||||||
|
// tables are separate modules. `normalizeHeaderCell` and `matchHeaderColumn`
|
||||||
|
// are the generic part, shared from there and unchanged.
|
||||||
const VALUE_HEADER_KEYWORDS = ["value", "valeur", "montant", "marchande"];
|
const VALUE_HEADER_KEYWORDS = ["value", "valeur", "montant", "marchande"];
|
||||||
|
|
||||||
/** Accent-strip + lowercase + keep alphanumerics only (for header matching). */
|
|
||||||
function normalizeHeaderCell(s: string): string {
|
|
||||||
return (s ?? "")
|
|
||||||
.normalize("NFD")
|
|
||||||
.replace(/[\u0300-\u036f]/g, "")
|
|
||||||
.toLowerCase()
|
|
||||||
.replace(/[^a-z0-9]/g, "");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find the first unused column whose normalized header contains one of the
|
|
||||||
* keywords (keywords tried in priority order). `exclude` skips columns whose
|
|
||||||
* header contains any excluded token (e.g. a "value" column for price).
|
|
||||||
*/
|
|
||||||
function matchHeaderColumn(
|
|
||||||
normalizedHeaders: string[],
|
|
||||||
keywords: string[],
|
|
||||||
used: Set<number>,
|
|
||||||
exclude: string[] = []
|
|
||||||
): number | null {
|
|
||||||
for (const kw of keywords) {
|
|
||||||
for (let i = 0; i < normalizedHeaders.length; i++) {
|
|
||||||
if (used.has(i)) continue;
|
|
||||||
const h = normalizedHeaders[i];
|
|
||||||
if (!h) continue;
|
|
||||||
if (exclude.some((ex) => h.includes(ex))) continue;
|
|
||||||
if (h.includes(kw)) return i;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Pick the shortest-average-length non-numeric, unused text column (symbols
|
/** Pick the shortest-average-length non-numeric, unused text column (symbols
|
||||||
* are short tokens; a name/description column is longer). */
|
* are short tokens; a name/description column is longer). */
|
||||||
function pickSymbolColumn(
|
function pickSymbolColumn(
|
||||||
|
|
|
||||||
160
src/utils/headerDictionary.test.ts
Normal file
160
src/utils/headerDictionary.test.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
||||||
|
// headerDictionary — the lexical layer of transaction-CSV detection (#327).
|
||||||
|
//
|
||||||
|
// `csvAutoDetect.test.ts` covers what the dictionary DOES to a detected
|
||||||
|
// configuration; this file covers the dictionary itself: every keyword it
|
||||||
|
// claims to know, the exclusions that keep a label from claiming the wrong
|
||||||
|
// column, and the mute row that hands control back to the shape heuristics.
|
||||||
|
|
||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
import {
|
||||||
|
matchHeaderColumn,
|
||||||
|
matchTransactionHeaders,
|
||||||
|
normalizeHeaderCell,
|
||||||
|
MIN_HEADER_ROLE_MATCHES,
|
||||||
|
} from "./headerDictionary";
|
||||||
|
|
||||||
|
describe("normalizeHeaderCell (#245, shared by #327)", () => {
|
||||||
|
it("strips accents, case and punctuation", () => {
|
||||||
|
expect(normalizeHeaderCell("Débit")).toBe("debit");
|
||||||
|
expect(normalizeHeaderCell("Libellé")).toBe("libelle");
|
||||||
|
expect(normalizeHeaderCell("Date de l'opération")).toBe("datedeloperation");
|
||||||
|
expect(normalizeHeaderCell("Montant ($)")).toBe("montant");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps digits and tolerates an empty or missing cell", () => {
|
||||||
|
expect(normalizeHeaderCell("Solde 2024")).toBe("solde2024");
|
||||||
|
expect(normalizeHeaderCell("")).toBe("");
|
||||||
|
expect(normalizeHeaderCell(undefined as unknown as string)).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("matchHeaderColumn (#245, shared by #327)", () => {
|
||||||
|
const headers = ["date", "libelle", "montant", "solde"];
|
||||||
|
|
||||||
|
it("returns the first unused column containing a keyword", () => {
|
||||||
|
expect(matchHeaderColumn(headers, ["montant"], new Set())).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honours keyword priority order over column order", () => {
|
||||||
|
expect(matchHeaderColumn(headers, ["solde", "date"], new Set())).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips used and excluded columns, and returns null on no match", () => {
|
||||||
|
expect(matchHeaderColumn(headers, ["date"], new Set([0]))).toBeNull();
|
||||||
|
expect(matchHeaderColumn(headers, ["mont"], new Set(), ["solde"])).toBe(2);
|
||||||
|
expect(matchHeaderColumn(headers, ["quantite"], new Set())).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("matchTransactionHeaders — the FR/EN dictionary (#327)", () => {
|
||||||
|
it("reads a French signed-amount header", () => {
|
||||||
|
const m = matchTransactionHeaders(["Date", "Description", "Montant", "Solde"]);
|
||||||
|
expect(m).toEqual({
|
||||||
|
date: 0,
|
||||||
|
description: 1,
|
||||||
|
amount: 2,
|
||||||
|
debit: null,
|
||||||
|
credit: null,
|
||||||
|
balance: 3,
|
||||||
|
roleCount: 4,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads an English signed-amount header", () => {
|
||||||
|
const m = matchTransactionHeaders(["Date", "Description", "Amount", "Balance"]);
|
||||||
|
expect(m.amount).toBe(2);
|
||||||
|
expect(m.balance).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["Débit", "debit"],
|
||||||
|
["Retrait", "debit"],
|
||||||
|
["Déboursé", "debit"],
|
||||||
|
["Withdrawal", "debit"],
|
||||||
|
["Crédit", "credit"],
|
||||||
|
["Dépôt", "credit"],
|
||||||
|
["Encaissement", "credit"],
|
||||||
|
["Deposit", "credit"],
|
||||||
|
] as const)("knows %s as the %s column", (label, role) => {
|
||||||
|
const m = matchTransactionHeaders(["Date", "Description", label]);
|
||||||
|
expect(m[role]).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["Libellé", 1],
|
||||||
|
["Détail", 1],
|
||||||
|
["Transaction", 1],
|
||||||
|
["Description", 1],
|
||||||
|
] as const)("knows %s as the description column", (label, col) => {
|
||||||
|
expect(matchTransactionHeaders(["Date", label, "Montant"]).description).toBe(
|
||||||
|
col
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves a reversed pair by label, not by position", () => {
|
||||||
|
const m = matchTransactionHeaders(["Date", "Description", "Credit", "Debit"]);
|
||||||
|
expect(m.debit).toBe(3);
|
||||||
|
expect(m.credit).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("claims a column once, so debit/credit beat the amount keyword", () => {
|
||||||
|
// Both columns contain `montant`. Resolving debit and credit first is what
|
||||||
|
// stops `Montant débit` from being read as THE amount column.
|
||||||
|
const m = matchTransactionHeaders([
|
||||||
|
"Date",
|
||||||
|
"Libelle",
|
||||||
|
"Montant debit",
|
||||||
|
"Montant credit",
|
||||||
|
]);
|
||||||
|
expect(m.debit).toBe(2);
|
||||||
|
expect(m.credit).toBe(3);
|
||||||
|
expect(m.amount).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never reads a Solde column as the amount", () => {
|
||||||
|
const m = matchTransactionHeaders(["Date", "Libelle", "Solde du compte"]);
|
||||||
|
expect(m.balance).toBe(2);
|
||||||
|
expect(m.amount).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never reads a date column as the description", () => {
|
||||||
|
// `Date de transaction` contains the description keyword `transaction`.
|
||||||
|
const m = matchTransactionHeaders([
|
||||||
|
"Date de transaction",
|
||||||
|
"Date comptable",
|
||||||
|
"Montant",
|
||||||
|
]);
|
||||||
|
expect(m.date).toBe(0);
|
||||||
|
expect(m.description).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns a map of nulls for labels it does not know", () => {
|
||||||
|
const m = matchTransactionHeaders(["Col A", "Col B", "Col C"]);
|
||||||
|
expect(m).toEqual({
|
||||||
|
date: null,
|
||||||
|
description: null,
|
||||||
|
amount: null,
|
||||||
|
debit: null,
|
||||||
|
credit: null,
|
||||||
|
balance: null,
|
||||||
|
roleCount: 0,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("scores a data row below the header threshold", () => {
|
||||||
|
// `DEPOT PAIE EMPLOYEUR` contains the credit keyword `depot` — one role,
|
||||||
|
// which is exactly why one is not enough to call a row a header.
|
||||||
|
const m = matchTransactionHeaders([
|
||||||
|
"05/01/2025",
|
||||||
|
"DEPOT PAIE EMPLOYEUR",
|
||||||
|
"-84,32",
|
||||||
|
]);
|
||||||
|
expect(m.credit).toBe(1);
|
||||||
|
expect(m.roleCount).toBeLessThan(MIN_HEADER_ROLE_MATCHES);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("tolerates empty cells and a ragged row", () => {
|
||||||
|
expect(matchTransactionHeaders([]).roleCount).toBe(0);
|
||||||
|
expect(matchTransactionHeaders(["", "Date", ""]).date).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
176
src/utils/headerDictionary.ts
Normal file
176
src/utils/headerDictionary.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
||||||
|
/**
|
||||||
|
* Header dictionary for the TRANSACTION import flow (#327).
|
||||||
|
*
|
||||||
|
* Detection used to reason on the SHAPE of the data alone — which column parses
|
||||||
|
* as a date, which one carries the most characters, which pair is sparse and
|
||||||
|
* complementary. Shape cannot tell a debit column from a credit column, so a
|
||||||
|
* file laid out `Date;Description;Crédit;Débit` was mapped backwards and every
|
||||||
|
* sign of the import was inverted. This module is the lexical layer that reads
|
||||||
|
* the header labels; `csvAutoDetect.ts` puts it in FRONT of the shape
|
||||||
|
* heuristics and falls back to them whenever the labels are mute (no header
|
||||||
|
* row, unknown labels).
|
||||||
|
*
|
||||||
|
* WHY ITS OWN MODULE. The holdings flow (#245) already matches header labels,
|
||||||
|
* with its own keyword tables living in `csvAutoDetect.ts`. Merging the two
|
||||||
|
* tables is not possible: `montant` is an EXCLUSION token there
|
||||||
|
* (`VALUE_HEADER_KEYWORDS` — a market-value column must never be read as a unit
|
||||||
|
* price) and it is the PRIMARY amount keyword here. Two flows, two tables. Only
|
||||||
|
* the two matching helpers are generic, so they live here and the holdings flow
|
||||||
|
* imports them, unchanged — one direction of dependency, no cycle.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Keyword tables, matched as SUBSTRINGS against accent-stripped,
|
||||||
|
// alphanumeric-only header cells, so `Date de l'opération` matches `date` and
|
||||||
|
// `Débit ($)` matches `debit`. Bilingual FR + EN, order inside each list is
|
||||||
|
// priority order. Kept deliberately tight: a keyword that fires on the wrong
|
||||||
|
// column is worse than a keyword that never fires, because the shape heuristic
|
||||||
|
// behind it is the behaviour the corpus already froze.
|
||||||
|
export const DATE_HEADER_KEYWORDS = ["date"];
|
||||||
|
export const DESCRIPTION_HEADER_KEYWORDS = [
|
||||||
|
"description",
|
||||||
|
"libelle",
|
||||||
|
"detail",
|
||||||
|
"transaction",
|
||||||
|
];
|
||||||
|
export const AMOUNT_HEADER_KEYWORDS = ["montant", "amount"];
|
||||||
|
export const DEBIT_HEADER_KEYWORDS = [
|
||||||
|
"debit",
|
||||||
|
"retrait",
|
||||||
|
"debourse",
|
||||||
|
"withdrawal",
|
||||||
|
];
|
||||||
|
export const CREDIT_HEADER_KEYWORDS = [
|
||||||
|
"credit",
|
||||||
|
"depot",
|
||||||
|
"encaissement",
|
||||||
|
"deposit",
|
||||||
|
];
|
||||||
|
export const BALANCE_HEADER_KEYWORDS = ["solde", "balance"];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A running-balance column is never an amount column, and `Date de
|
||||||
|
* transaction` is never the description column. Both are exclusion lists rather
|
||||||
|
* than ordering tricks, because substring matching has no notion of "the best
|
||||||
|
* match" — the first column containing the keyword wins.
|
||||||
|
*/
|
||||||
|
const AMOUNT_EXCLUDE = BALANCE_HEADER_KEYWORDS;
|
||||||
|
const DESCRIPTION_EXCLUDE = DATE_HEADER_KEYWORDS;
|
||||||
|
|
||||||
|
/** Accent-strip + lowercase + keep alphanumerics only (for header matching). */
|
||||||
|
export function normalizeHeaderCell(s: string): string {
|
||||||
|
return (s ?? "")
|
||||||
|
.normalize("NFD")
|
||||||
|
.replace(/[\u0300-\u036f]/g, "")
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9]/g, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the first unused column whose normalized header contains one of the
|
||||||
|
* keywords (keywords tried in priority order). `exclude` skips columns whose
|
||||||
|
* header contains any excluded token (e.g. a "value" column for price).
|
||||||
|
*/
|
||||||
|
export function matchHeaderColumn(
|
||||||
|
normalizedHeaders: string[],
|
||||||
|
keywords: string[],
|
||||||
|
used: Set<number>,
|
||||||
|
exclude: string[] = []
|
||||||
|
): number | null {
|
||||||
|
for (const kw of keywords) {
|
||||||
|
for (let i = 0; i < normalizedHeaders.length; i++) {
|
||||||
|
if (used.has(i)) continue;
|
||||||
|
const h = normalizedHeaders[i];
|
||||||
|
if (!h) continue;
|
||||||
|
if (exclude.some((ex) => h.includes(ex))) continue;
|
||||||
|
if (h.includes(kw)) return i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What the labels of one header row say, role by role. */
|
||||||
|
export interface LexicalHeaderMap {
|
||||||
|
date: number | null;
|
||||||
|
description: number | null;
|
||||||
|
amount: number | null;
|
||||||
|
debit: number | null;
|
||||||
|
credit: number | null;
|
||||||
|
balance: number | null;
|
||||||
|
/** Number of roles the row actually named — 0 when the labels are mute. */
|
||||||
|
roleCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read one header row through the dictionary.
|
||||||
|
*
|
||||||
|
* Roles are resolved in the order balance -> date -> debit -> credit -> amount
|
||||||
|
* -> description, each claiming its column so a later role cannot steal it.
|
||||||
|
* That order is what makes `Montant débit` / `Montant crédit` resolve as a
|
||||||
|
* debit/credit pair instead of the first one being read as THE amount column.
|
||||||
|
*
|
||||||
|
* Every field is `null` when no column names that role: a mute row yields a map
|
||||||
|
* of nulls and `roleCount: 0`, which is the caller's signal to fall back to the
|
||||||
|
* shape heuristics.
|
||||||
|
*/
|
||||||
|
export function matchTransactionHeaders(
|
||||||
|
headerRow: readonly string[]
|
||||||
|
): LexicalHeaderMap {
|
||||||
|
const normalized = headerRow.map(normalizeHeaderCell);
|
||||||
|
const used = new Set<number>();
|
||||||
|
|
||||||
|
const claim = (col: number | null): number | null => {
|
||||||
|
if (col !== null) used.add(col);
|
||||||
|
return col;
|
||||||
|
};
|
||||||
|
|
||||||
|
const balance = claim(
|
||||||
|
matchHeaderColumn(normalized, BALANCE_HEADER_KEYWORDS, used)
|
||||||
|
);
|
||||||
|
const date = claim(matchHeaderColumn(normalized, DATE_HEADER_KEYWORDS, used));
|
||||||
|
const debit = claim(
|
||||||
|
matchHeaderColumn(normalized, DEBIT_HEADER_KEYWORDS, used, AMOUNT_EXCLUDE)
|
||||||
|
);
|
||||||
|
const credit = claim(
|
||||||
|
matchHeaderColumn(normalized, CREDIT_HEADER_KEYWORDS, used, AMOUNT_EXCLUDE)
|
||||||
|
);
|
||||||
|
const amount = claim(
|
||||||
|
matchHeaderColumn(normalized, AMOUNT_HEADER_KEYWORDS, used, AMOUNT_EXCLUDE)
|
||||||
|
);
|
||||||
|
const description = claim(
|
||||||
|
matchHeaderColumn(
|
||||||
|
normalized,
|
||||||
|
DESCRIPTION_HEADER_KEYWORDS,
|
||||||
|
used,
|
||||||
|
DESCRIPTION_EXCLUDE
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const roleCount = [date, description, amount, debit, credit, balance].filter(
|
||||||
|
(c) => c !== null
|
||||||
|
).length;
|
||||||
|
|
||||||
|
return { date, description, amount, debit, credit, balance, roleCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How many named roles a row must carry before the lexical layer is allowed to
|
||||||
|
* call it a header row on its own.
|
||||||
|
*
|
||||||
|
* Two, not one: a description cell like `DEPOT PAIE EMPLOYEUR` contains
|
||||||
|
* `depot`, so a single match proves nothing. Two distinct roles in one row is a
|
||||||
|
* shape a data row does not produce, and the caller keeps the date test in
|
||||||
|
* front of this check anyway.
|
||||||
|
*/
|
||||||
|
export const MIN_HEADER_ROLE_MATCHES = 2;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tokens of a direction-indicator column: the third amount format, where the
|
||||||
|
* amount column holds unsigned magnitudes and a neighbouring column says which
|
||||||
|
* way each row goes. `D`/`C` in French exports, `DB`/`CR` in some English ones.
|
||||||
|
*
|
||||||
|
* The format is refused rather than imported (spec decision) — see
|
||||||
|
* `csvAutoDetect.ts`. Reading it correctly is a separate feature; reading it
|
||||||
|
* wrong imports every debit as income.
|
||||||
|
*/
|
||||||
|
export const DEBIT_INDICATOR_TOKENS = ["d", "db"];
|
||||||
|
export const CREDIT_INDICATOR_TOKENS = ["c", "cr"];
|
||||||
Loading…
Reference in a new issue