From ef7de3cf9ba890f8e3ce3cb7331c86e65244fae4 Mon Sep 17 00:00:00 2001 From: le king fu Date: Fri, 14 Aug 2026 12:02:59 -0400 Subject: [PATCH] fix(import): veto a labelled description column that behaves like an enum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding on #327. detectDescriptionColumn returned the lexically preferred column with no check on the data, unlike the date (replayed at 0.8) and the amount (constrained to the shape candidates). The dictionary lists 'transaction' as a description keyword, and Tangerine exports Date,Transaction,Name,Memo,Amount where Transaction holds DEBIT/CREDIT — so the description moved off the merchant name and keyword categorisation died. Cardinality tells free text from an enum: a description repeats almost nothing, an enum repeats almost everything. Average length does not — Note and Libelle are both short, so a length veto would reject legitimate columns. This fixes the cause. #330 had rescued the case through the Tangerine signature alone, leaving every unrecognised file with a Transaction column broken; that test now asserts the correct mapping with and without a signature. Refs #327 Co-Authored-By: Claude Opus 5 --- src/utils/csvAutoDetect.ts | 80 ++++++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/src/utils/csvAutoDetect.ts b/src/utils/csvAutoDetect.ts index 4e89927..6819cf6 100644 --- a/src/utils/csvAutoDetect.ts +++ b/src/utils/csvAutoDetect.ts @@ -707,6 +707,37 @@ function detectBalanceColumns( * labelling its amount column `Détail du montant` therefore cannot end up with * its amounts in the description. */ +/** + * Does this column behave like an enumeration rather than free text? + * + * A label alone is not enough to pick the description: Tangerine exports + * `Date,Transaction,Name,Memo,Amount`, where `Transaction` holds DEBIT/CREDIT + * and `Name` holds the merchant. Honouring the label there moves the merchant + * out of the description and kills keyword categorisation. + * + * Cardinality separates the two — a description repeats almost nothing, an enum + * repeats almost everything. Average length does NOT: `Note` and `Libellé` are + * both short, and vetoing on length would reject legitimate columns. + */ +function looksLikeEnumColumn(rows: string[][], col: number): boolean { + const distinct = new Set(); + let filled = 0; + + for (const row of rows) { + const cell = row[col]?.trim(); + if (!cell) continue; + filled++; + distinct.add(cell.toLowerCase()); + } + + // A labelled but entirely empty column is never the description either. + if (filled === 0) return true; + // Too few rows to read anything into the cardinality. + if (filled < 4) return false; + + return distinct.size <= Math.max(2, Math.floor(filled / 4)); +} + function detectDescriptionColumn( rows: string[][], colCount: number, @@ -719,7 +750,8 @@ function detectDescriptionColumn( preferred !== null && preferred < colCount && preferred !== dateCol && - !numericCols.has(preferred) + !numericCols.has(preferred) && + !looksLikeEnumColumn(rows, preferred) ) { return preferred; } @@ -779,6 +811,26 @@ type AmountModeResult = SingleAmountResult | DebitCreditResult; * nothing numeric is dropped here and the generic path decides, exactly like a * mismatched label. */ +/** + * The first sparse-complementary pair among the candidates, in column order, or + * null. Extracted from `detectAmountMode` so a bank signature can be arbitrated + * AGAINST the pair the shape scan would have found, instead of short-circuiting + * a scan that never ran. + */ +function findSparseComplementaryPair( + rows: string[][], + amountCandidates: number[] +): [number, number] | null { + for (let a = 0; a < amountCandidates.length; a++) { + for (let b = a + 1; b < amountCandidates.length; b++) { + const colA = amountCandidates[a]; + const colB = amountCandidates[b]; + if (isSparseComplementary(rows, colA, colB)) return [colA, colB]; + } + } + return null; +} + function detectAmountMode( rows: string[][], amountCandidates: number[], @@ -787,6 +839,8 @@ function detectAmountMode( ): AmountModeResult | null { if (amountCandidates.length === 0) return null; + const pair = findSparseComplementaryPair(rows, amountCandidates); + if (signature) { const { debit, credit, amount } = signature.roles; if ( @@ -797,7 +851,17 @@ function detectAmountMode( ) { return { mode: "debit_credit", debitCol: debit, creditCol: credit }; } - if (amount !== null && amountCandidates.includes(amount)) { + // A signature's SINGLE amount column may not silently displace a genuine + // debit/credit pair it has no part in. RBC needs the override — its + // near-empty `Cheque Number` really is sparse-complementary with `CAD$` — + // but there the pair contains the declared amount column. On a + // `Date;Description;Débit;Crédit;Montant;Solde` file the pair does not, and + // taking `Montant` unsigned imported every deposit as an expense. + if ( + amount !== null && + amountCandidates.includes(amount) && + (!pair || pair.includes(amount)) + ) { return detectSingleAmount(rows, amount); } } @@ -806,16 +870,8 @@ function detectAmountMode( return detectSingleAmount(rows, amountCandidates[0]); } - // Check for sparse-complementary pair (debit/credit pattern) - for (let a = 0; a < amountCandidates.length; a++) { - for (let b = a + 1; b < amountCandidates.length; b++) { - const colA = amountCandidates[a]; - const colB = amountCandidates[b]; - - if (isSparseComplementary(rows, colA, colB)) { - return orderDebitCredit(colA, colB, lexical); - } - } + if (pair) { + return orderDebitCredit(pair[0], pair[1], lexical); } // No complementary pair found — the labelled amount column if there is one,