maximus/Simpl-Resultat
1
0
Fork 0
Code Issues 5 Pull requests Projects Releases 29 Packages Wiki Activity Actions

feat(import): recognise known bank layouts and report format drift #340

Closed
maximus wants to merge 1 commit from issue-330-bank-signatures-drift into issue-329-mandatory-preview
pull from: issue-330-bank-signatures-drift
merge into: maximus:issue-329-mandatory-preview
Conversation 3 Commits 1 Files changed 18 +1732 -29
18 changed files with 1732 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit c9872fc36b - Show all commits

7
src/__fixtures__/csv/bank-bnc.csv Normal file
View file

@ -0,0 +1,7 @@
Date;Description;Catégorie;Débit;Crédit;Solde
05/01/2025;EPICERIE METRO SAINTE-FOY;Alimentation;84,32;;915,68
15/01/2025;DEPOT PAIE EMPLOYEUR;Revenu;;1250,00;2165,68
18/01/2025;HYDRO QUEBEC PREAUTORISE;Services publics;142,18;;2023,50
22/01/2025;RESTAURANT LE BISTRO;Restaurants;56,75;;1966,75
27/01/2025;VIREMENT RECU;Transferts;;300,00;2266,75
30/01/2025;FRAIS MENSUELS;Frais bancaires;6,95;;2259,80
1 Date Description Catégorie Débit Crédit Solde
2 05/01/2025 EPICERIE METRO SAINTE-FOY Alimentation 84,32 915,68
3 15/01/2025 DEPOT PAIE EMPLOYEUR Revenu 1250,00 2165,68
4 18/01/2025 HYDRO QUEBEC PREAUTORISE Services publics 142,18 2023,50
5 22/01/2025 RESTAURANT LE BISTRO Restaurants 56,75 1966,75
6 27/01/2025 VIREMENT RECU Transferts 300,00 2266,75
7 30/01/2025 FRAIS MENSUELS Frais bancaires 6,95 2259,80

7
src/__fixtures__/csv/bank-desjardins.csv Normal file
View file

@ -0,0 +1,7 @@
Date;Description;Montant;Solde
05/01/2025;EPICERIE METRO SAINTE-FOY;-84,32;915,68
15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;2165,68
18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,18;2023,50
22/01/2025;RESTAURANT LE BISTRO;-56,75;1966,75
27/01/2025;VIREMENT RECU;300,00;2266,75
30/01/2025;FRAIS MENSUELS;-6,95;2259,80
1 Date Description Montant Solde
2 05/01/2025 EPICERIE METRO SAINTE-FOY -84,32 915,68
3 15/01/2025 DEPOT PAIE EMPLOYEUR 1250,00 2165,68
4 18/01/2025 HYDRO QUEBEC PREAUTORISE -142,18 2023,50
5 22/01/2025 RESTAURANT LE BISTRO -56,75 1966,75
6 27/01/2025 VIREMENT RECU 300,00 2266,75
7 30/01/2025 FRAIS MENSUELS -6,95 2259,80

7
src/__fixtures__/csv/bank-rbc.csv Normal file
View file

@ -0,0 +1,7 @@
Account Type,Account Number,Transaction Date,Cheque Number,Description 1,Description 2,CAD$,USD$
Chequing,1234567,05/01/2025,,EPICERIE METRO SAINTE-FOY,,-84.32,
Chequing,1234567,15/01/2025,,DEPOT PAIE EMPLOYEUR,PAIE,1250.00,
Chequing,1234567,18/01/2025,,HYDRO QUEBEC PREAUTORISE,,-142.18,
Chequing,1234567,22/01/2025,,RESTAURANT LE BISTRO,,-56.75,
Chequing,1234567,27/01/2025,,VIREMENT RECU,,300.00,
Chequing,1234567,30/01/2025,241,FRAIS MENSUELS,,-6.95,
1 Account Type Account Number Transaction Date Cheque Number Description 1 Description 2 CAD$ USD$
2 Chequing 1234567 05/01/2025 EPICERIE METRO SAINTE-FOY -84.32
3 Chequing 1234567 15/01/2025 DEPOT PAIE EMPLOYEUR PAIE 1250.00
4 Chequing 1234567 18/01/2025 HYDRO QUEBEC PREAUTORISE -142.18
5 Chequing 1234567 22/01/2025 RESTAURANT LE BISTRO -56.75
6 Chequing 1234567 27/01/2025 VIREMENT RECU 300.00
7 Chequing 1234567 30/01/2025 241 FRAIS MENSUELS -6.95

7
src/__fixtures__/csv/bank-tangerine.csv Normal file
View file

@ -0,0 +1,7 @@
Date,Transaction,Name,Memo,Amount
05/01/2025,DEBIT,EPICERIE METRO SAINTE-FOY,,-84.32
15/01/2025,CREDIT,DEPOT PAIE EMPLOYEUR,Paie bimensuelle,1250.00
18/01/2025,DEBIT,HYDRO QUEBEC PREAUTORISE,,-142.18
22/01/2025,DEBIT,RESTAURANT LE BISTRO,,-56.75
27/01/2025,CREDIT,VIREMENT RECU,,300.00
30/01/2025,DEBIT,FRAIS MENSUELS,,-6.95
1 Date Transaction Name Memo Amount
2 05/01/2025 DEBIT EPICERIE METRO SAINTE-FOY -84.32
3 15/01/2025 CREDIT DEPOT PAIE EMPLOYEUR Paie bimensuelle 1250.00
4 18/01/2025 DEBIT HYDRO QUEBEC PREAUTORISE -142.18
5 22/01/2025 DEBIT RESTAURANT LE BISTRO -56.75
6 27/01/2025 CREDIT VIREMENT RECU 300.00
7 30/01/2025 DEBIT FRAIS MENSUELS -6.95

16
src/__fixtures__/csv/index.ts
View file

@ -28,7 +28,17 @@ export type CsvFixtureName =
| "no-header"
| "all-positive"
| "desjardins-quoted"
| "absolute-indicator";
| "absolute-indicator"
// Bank-signature cases (#330). Each one is the DOCUMENTED header layout of a
// bank, invented row by row like the rest of the corpus. Three of them are
// shapes the generic dictionary reads WRONG — Tangerine's `Transaction` type
// column taken for the description, RBC's `CAD$` paired with the near-empty
// `Cheque Number` as a debit/credit pair — which is what the signatures exist
// for.
| "bank-desjardins"
| "bank-rbc"
| "bank-bnc"
| "bank-tangerine";
/** Every case in the corpus, in the order the issue lists them. */
export const CSV_FIXTURE_NAMES: readonly CsvFixtureName[] = [
@ -43,6 +53,10 @@ export const CSV_FIXTURE_NAMES: readonly CsvFixtureName[] = [
"all-positive",
"desjardins-quoted",
"absolute-indicator",
"bank-desjardins",
"bank-rbc",
"bank-bnc",
"bank-tangerine",
];
/**

11
src/components/import/FilePreviewTable.tsx
View file

@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next";
import { AlertCircle, ArrowDownLeft, ArrowUpRight, RefreshCw } from "lucide-react";
import type { ParsedRow } from "../../shared/types";
import { isRowErrorKey, summarizeParsedRows } from "../../utils/importFormat";
import RepairPathNotice from "./RepairPathNotice";
/** Rows rendered in the table. The recap above it always covers the whole file. */
const DISPLAYED_ROWS = 20;
@ -119,6 +120,16 @@ export default function FilePreviewTable({
</div>
)}
</div>
{/*
The repair path (#330), stated where the correction is offered. A user
who flips the signs here has just learned that earlier imports of the
same source read backwards; re-importing those files is the natural
next move and it double-books every row instead of fixing them.
*/}
<div className="mt-4">
<RepairPathNotice />
</div>
</div>
<div className="overflow-x-auto rounded-xl border border-[var(--border)]">

133
src/components/import/FormatDriftPanel.tsx Normal file
View file

@ -0,0 +1,133 @@
import { useTranslation } from "react-i18next";
import { AlertTriangle, ArrowRight, Minus, Plus } from "lucide-react";
import type { HeaderDriftEntry } from "../../utils/bankSignatures";
import RepairPathNotice from "./RepairPathNotice";
/**
* The header row of this source changed since its last successful import
* (#330).
*
* This is the failure mode the whole chantier is most afraid of, because it is
* the one nobody sees: a bank moves a column, the stored mapping keeps reading
* position 3, and the import succeeds — with the balance in the amount column,
* for as long as it takes someone to notice. There is no signal in the totals,
* no error, no red row. The only signal is the header itself, which is why it
* is recorded and compared.
*
* The panel does not decide. It shows what moved, column by column, and offers
* the two answers that exist: read the file the way it looks now, or keep
* reading it the way it used to. Nothing here writes to the database.
*/
interface FormatDriftPanelProps {
entries: HeaderDriftEntry[];
/** False when detection could not read the new shape — nothing to adopt. */
canAdopt: boolean;
onAdopt: () => void;
onKeep: () => void;
isBusy?: boolean;
}
export default function FormatDriftPanel({
entries,
canAdopt,
onAdopt,
onKeep,
isBusy = false,
}: FormatDriftPanelProps) {
const { t } = useTranslation();
const buttonClass =
"px-3 py-2 text-sm rounded-lg transition-colors disabled:opacity-50 disabled:cursor-not-allowed";
return (
<div className="p-4 rounded-xl bg-[var(--card)] border-2 border-[var(--accent)] space-y-4">
<div className="flex items-start gap-2">
<AlertTriangle
size={18}
className="text-[var(--accent)] shrink-0 mt-0.5"
/>
<div>
<h3 className="text-sm font-semibold text-[var(--foreground)]">
{t("import.drift.title")}
</h3>
<p className="text-sm text-[var(--muted-foreground)] mt-0.5">
{t("import.drift.intro")}
</p>
</div>
</div>
{/*
Column by column, naming the columns. This is what the stored signature
being a label list rather than a hash buys: a hash can say "something
changed" and nothing more.
*/}
<ul className="space-y-1 text-sm font-mono">
{entries.map((entry) => (
<li
key={`${entry.kind}-${entry.normalizedLabel}`}
className="flex items-center gap-2 text-[var(--foreground)]"
>
{entry.kind === "moved" && (
<ArrowRight size={14} className="text-[var(--accent)] shrink-0" />
)}
{entry.kind === "added" && (
<Plus size={14} className="text-[var(--positive)] shrink-0" />
)}
{entry.kind === "removed" && (
<Minus size={14} className="text-[var(--negative)] shrink-0" />
)}
<span>
{entry.kind === "moved" &&
t("import.drift.columnMoved", {
label: entry.label,
from: entry.previousIndex,
to: entry.currentIndex,
})}
{entry.kind === "added" &&
t("import.drift.columnAdded", {
label: entry.label,
to: entry.currentIndex,
})}
{entry.kind === "removed" &&
t("import.drift.columnRemoved", {
label: entry.label,
from: entry.previousIndex,
})}
</span>
</li>
))}
</ul>
<RepairPathNotice />
<div className="flex flex-wrap items-start gap-4">
<div>
<button
onClick={onAdopt}
disabled={!canAdopt || isBusy}
className={`${buttonClass} bg-[var(--primary)] text-white hover:opacity-90`}
>
{t("import.drift.adopt")}
</button>
<p className="mt-1 text-xs text-[var(--muted-foreground)] max-w-xs">
{canAdopt
? t("import.drift.adoptHint")
: t("import.drift.adoptUnavailable")}
</p>
</div>
<div>
<button
onClick={onKeep}
disabled={isBusy}
className={`${buttonClass} border border-[var(--border)] text-[var(--foreground)] hover:bg-[var(--muted)]`}
>
{t("import.drift.keep")}
</button>
<p className="mt-1 text-xs text-[var(--muted-foreground)] max-w-xs">
{t("import.drift.keepHint")}
</p>
</div>
</div>
</div>
);
}

44
src/components/import/RepairPathNotice.tsx Normal file
View file

@ -0,0 +1,44 @@
import { useTranslation } from "react-i18next";
import { LifeBuoy } from "lucide-react";
/**
* The only safe way to repair an import that was already written wrong (#330).
*
* WHY THIS IS IN THE INTERFACE AND NOT ONLY IN A RISK TABLE. Every correction
* this chantier delivers — the restored format (#324), the anchored amount
* parser (#325), the sign flip and the drift panel — changes the amounts a
* source PRODUCES. The natural next move is to re-import the file "now that it
* reads right", and that move silently doubles the ledger:
* `findDuplicates` (`transactionService.ts:164`) matches on
* `date AND description AND amount`, so a corrected row does not match the
* wrong row already stored. It is filed as new.
*
* A flipped sign is the worst shape of it: the wrong row and the corrected row
* are mirror images, they net to about zero in every report, and nothing on any
* screen looks off. The only path that leaves a correct ledger is deleting the
* faulty import from the history first — `deleteImportWithTransactions` takes
* its transactions with it — and replaying the file afterwards.
*
* Shown in both places a user can be about to do exactly that: the preview,
* next to the sign flip, and the format-drift panel.
*/
export default function RepairPathNotice() {
const { t } = useTranslation();
return (
<div className="flex items-start gap-2 p-3 rounded-lg bg-[var(--muted)] border border-[var(--border)]">
<LifeBuoy
size={14}
className="text-[var(--muted-foreground)] shrink-0 mt-0.5"
/>
<div>
<p className="text-xs font-semibold text-[var(--foreground)]">
{t("import.repairPath.title")}
</p>
<p className="text-xs text-[var(--muted-foreground)] mt-0.5">
{t("import.repairPath.body")}
</p>
</div>
</div>
);
}

43
src/components/import/SourceConfigPanel.tsx
View file

@ -10,6 +10,10 @@ import type {
ImportConfigTemplate,
} from "../../shared/types";
import type { DetectionScore } from "../../utils/csvAutoDetect";
import {
bankSignatureById,
type BankSignatureId,
} from "../../utils/bankSignatures";
import ColumnMappingEditor from "./ColumnMappingEditor";
interface SourceConfigPanelProps {
@ -30,6 +34,8 @@ interface SourceConfigPanelProps {
selectedTemplateId: number | null;
/** Result of the last detection run, or null when none ran on this source. */
detectionScore?: DetectionScore | null;
/** Bank recognised by signature during that same run, or null (#330). */
detectedBank?: BankSignatureId | null;
isLoading?: boolean;
}
@ -50,12 +56,23 @@ export default function SourceConfigPanel({
onDeleteTemplate,
selectedTemplateId,
detectionScore,
detectedBank,
isLoading,
}: SourceConfigPanelProps) {
const { t } = useTranslation();
const [showSaveTemplate, setShowSaveTemplate] = useState(false);
const [templateName, setTemplateName] = useState("");
/*
A recognised bank replaces the generic wording, but ONLY on the confident
arm (#330). "Format Desjardins reconnu" over a file two thirds of whose
rows would not read is a claim the app cannot back: the label matched, the
data did not follow, and the uncertain wording is the one that helps.
*/
const bank = detectionScore?.confident
? bankSignatureById(detectedBank)
: null;
const selectClass =
"w-full px-3 py-2 text-sm rounded-lg border border-[var(--border)] bg-[var(--card)] text-[var(--foreground)] focus:outline-none focus:ring-2 focus:ring-[var(--primary)]";
const inputClass = selectClass;
@ -107,15 +124,23 @@ export default function SourceConfigPanel({
)}
<div>
<p className="text-sm text-[var(--foreground)]">
{t(
detectionScore.confident
? "import.config.detectionRecognized"
: "import.config.detectionUncertain",
{
read: detectionScore.readRows,
total: detectionScore.totalRows,
}
)}
{bank
? t("import.config.detectionBank", {
// A bank name is a proper noun: it is interpolated, never
// translated.
bank: bank.label,
read: detectionScore.readRows,
total: detectionScore.totalRows,
})
: t(
detectionScore.confident
? "import.config.detectionRecognized"
: "import.config.detectionUncertain",
{
read: detectionScore.readRows,
total: detectionScore.totalRows,
}
)}
</p>
{!detectionScore.confident && (
<p className="text-xs text-[var(--muted-foreground)] mt-0.5">

176
src/hooks/useImportWizard.ts
View file

@ -44,6 +44,12 @@ import {
detectImportFormat as runAutoDetect,
type DetectionScore,
} from "../utils/csvAutoDetect";
import {
buildHeaderSignature,
detectHeaderDrift,
type BankSignatureId,
type HeaderDriftEntry,
} from "../utils/bankSignatures";
import {
detectAmountSeparators,
flipSignFormat,
@ -85,6 +91,24 @@ interface WizardState {
* ran — a source opened on its stored format, which is never re-detected.
*/
detectionScore: DetectionScore | null;
/**
* Bank whose documented layout the last detection recognised (#330). Tied to
* `detectionScore`: it is set with it and cleared with it, so a badge naming
* a bank can never outlive the detection that named it.
*/
detectedBank: BankSignatureId | null;
/**
* How the header row of the files being imported differs from the one the
* source recorded at its last successful import — null when there is nothing
* to report, which includes every source that has no stored signature.
*/
formatDrift: HeaderDriftEntry[] | null;
/**
* The configuration re-detected on the drifted file, offered by the panel as
* "adopt". Null when detection could not read the new shape either: the drift
* is still worth reporting, there is just nothing to adopt.
*/
driftConfig: SourceConfig | null;
}
type WizardAction =
@ -107,6 +131,11 @@ type WizardAction =
| { type: "SET_CONFIG_TEMPLATES"; payload: ImportConfigTemplate[] }
| { type: "SET_SELECTED_TEMPLATE_ID"; payload: number | null }
| { type: "SET_DETECTION_SCORE"; payload: DetectionScore | null }
| { type: "SET_DETECTED_BANK"; payload: BankSignatureId | null }
| {
type: "SET_FORMAT_DRIFT";
payload: { entries: HeaderDriftEntry[]; config: SourceConfig | null } | null;
}
| { type: "RESET" };
const defaultConfig: SourceConfig = {
@ -142,6 +171,9 @@ const initialState: WizardState = {
configTemplates: [],
selectedTemplateId: null,
detectionScore: null,
detectedBank: null,
formatDrift: null,
driftConfig: null,
};
function reducer(state: WizardState, action: WizardAction): WizardState {
@ -159,8 +191,17 @@ function reducer(state: WizardState, action: WizardAction): WizardState {
case "SET_SELECTED_SOURCE":
// The score belongs to the source it was measured on. Carrying it over
// would let a source opened on its stored format inherit the confidence
// of the previous one, which is worse than showing nothing.
return { ...state, selectedSource: action.payload, detectionScore: null };
// of the previous one, which is worse than showing nothing. The bank
// badge and the drift report are scoped the same way, for the same
// reason — both describe one source's files.
return {
...state,
selectedSource: action.payload,
detectionScore: null,
detectedBank: null,
formatDrift: null,
driftConfig: null,
};
case "SET_SELECTED_FILES":
return { ...state, selectedFiles: action.payload };
case "SET_SOURCE_CONFIG":
@ -212,7 +253,24 @@ function reducer(state: WizardState, action: WizardAction): WizardState {
case "SET_SELECTED_TEMPLATE_ID":
return { ...state, selectedTemplateId: action.payload };
case "SET_DETECTION_SCORE":
return { ...state, detectionScore: action.payload };
// Clearing the score clears the bank with it, in ONE place: the three
// sites that drop the score (a hand edit, a template, a sign flip) then
// cannot leave "Format Desjardins reconnu" standing over a format
// detection never produced. A bank is only ever set by dispatching
// `SET_DETECTED_BANK` right after a non-null score.
return {
...state,
detectionScore: action.payload,
detectedBank: action.payload === null ? null : state.detectedBank,
};
case "SET_DETECTED_BANK":
return { ...state, detectedBank: action.payload };
case "SET_FORMAT_DRIFT":
return {
...state,
formatDrift: action.payload?.entries ?? null,
driftConfig: action.payload?.config ?? null,
};
case "RESET":
return {
...initialState,
@ -346,6 +404,9 @@ export function useImportWizard() {
}
dispatch({ type: "SET_DETECTION_SCORE", payload: outcome.score });
// Always dispatched, including the `null` of an unknown file: a bank left
// over from the previously detected file would name the wrong bank.
dispatch({ type: "SET_DETECTED_BANK", payload: outcome.bank });
return { ...base, ...outcome.config };
},
[]
@ -672,6 +733,48 @@ export function useImportWizard() {
type: "SET_PARSED_PREVIEW",
payload: result,
});
// Format drift (#330). Compared HERE and not at the configuration step
// because this is where the header of the files actually being imported
// is known — `previewHeaders` at the configuration step is read from the
// first file alone, under a delimiter the user may still be editing.
//
// `hasHeader` gates the whole thing: a headerless file has `Col 0`,
// `Col 1` … for headers, which is not a signature and must never be
// compared to one.
const drift = state.sourceConfig.hasHeader
? detectHeaderDrift(
state.existingSource?.header_signature,
result.headers
)
: null;
if (drift) {
// Re-detect on the drifted file so the panel has something to offer.
// This is the THIRD caller of the single detection path, and it goes
// through it rather than around it on purpose: a second detector is the
// divergence class this whole chantier removes. A file the re-detection
// cannot read leaves `null` — the drift is still reported, with nothing
// to adopt.
const reDetected = await detectFormatForFile(
state.selectedFiles[0].file_path,
state.sourceConfig
);
// That run measured the RE-DETECTED format, which is not the one the
// wizard is using — it is only what the panel offers. Leaving its score
// and its bank badge behind would put "Format reconnu — 6 of 6 rows
// read" next to the stored mapping the moment the user steps back to
// the configuration. Same rule as the sign flip: a badge describes the
// format in use or it describes nothing.
dispatch({ type: "SET_DETECTION_SCORE", payload: null });
dispatch({
type: "SET_FORMAT_DRIFT",
payload: { entries: drift, config: reDetected },
});
} else {
dispatch({ type: "SET_FORMAT_DRIFT", payload: null });
}
dispatch({ type: "SET_STEP", payload: "file-preview" });
} catch (e) {
dispatch({
@ -679,7 +782,13 @@ export function useImportWizard() {
payload: e instanceof Error ? e.message : String(e),
});
}
}, [state.selectedFiles, parseFilesInternal]);
}, [
state.selectedFiles,
state.sourceConfig,
state.existingSource,
parseFilesInternal,
detectFormatForFile,
]);
// Internal helper: runs duplicate checking against parsed rows.
//
@ -836,18 +945,31 @@ export function useImportWizard() {
// behind, and it goes through `formatToRow` so no field can be dropped.
// It has to precede the file records, which carry a `source_id` FK.
const formatRow = formatToRow(config);
// The header row this import actually read, recorded so the NEXT one can
// be compared against it (#330). It is not a format field and does not go
// through the codec: it says what the file looked like, not how to read
// it. A headerless file writes null — `previewHeaders` holds `Col 0`,
// `Col 1` … there, and storing that would make every later import look
// like drift.
const headerSignature = buildHeaderSignature(
config.hasHeader ? state.previewHeaders : null
);
let sourceId: number;
if (state.existingSource) {
sourceId = state.existingSource.id;
await updateSource(sourceId, {
name: config.name,
...formatRow,
header_signature: headerSignature,
template_id: state.selectedTemplateId,
});
} else {
sourceId = await createSource({
name: config.name,
...formatRow,
header_signature: headerSignature,
template_id: state.selectedTemplateId,
});
}
@ -993,6 +1115,50 @@ export function useImportWizard() {
dispatch({ type: "RESET" });
}, []);
/**
* Take the format re-detected on the drifted file, and re-read the preview
* under it (#330).
*
* Same shape as the sign flip, and for the same reason: the adopted format is
* PASSED to the parse instead of being read back from `state`, which has not
* re-rendered yet. Reading the stale one would redisplay the table the user
* just chose to replace.
*
* Nothing is written here. The adopted format reaches `import_sources` at
* `executeImport` like every other configuration — an import abandoned on
* this screen leaves the stored format exactly as it was.
*/
const adoptDriftFormat = useCallback(async () => {
const adopted = state.driftConfig;
if (!adopted) return;
dispatch({ type: "SET_SOURCE_CONFIG", payload: adopted });
dispatch({ type: "SET_FORMAT_DRIFT", payload: null });
dispatch({ type: "SET_LOADING", payload: true });
dispatch({ type: "SET_ERROR", payload: null });
try {
const result = await parseFilesInternal(adopted);
dispatch({ type: "SET_PARSED_PREVIEW", payload: result });
} catch (e) {
dispatch({
type: "SET_ERROR",
payload: e instanceof Error ? e.message : String(e),
});
}
}, [state.driftConfig, parseFilesInternal]);
/**
* Keep the stored configuration and dismiss the panel.
*
* The rows on screen were already parsed under that configuration, so there
* is nothing to re-read. The stored signature is still refreshed at import
* time: the new header IS what this import read, whichever format read it.
*/
const keepCurrentFormat = useCallback(() => {
dispatch({ type: "SET_FORMAT_DRIFT", payload: null });
}, []);
const autoDetectConfig = useCallback(async () => {
if (state.selectedFiles.length === 0) return;
@ -1108,6 +1274,8 @@ export function useImportWizard() {
executeImport,
goToStep,
reset,
adoptDriftFormat,
keepCurrentFormat,
autoDetectConfig,
saveConfigAsTemplate,
applyConfigTemplate,

19
src/i18n/locales/en.json
View file

@ -116,7 +116,24 @@
"updateTemplate": "Update template",
"detectionRecognized": "Format recognized — {{read}} of {{total}} rows read",
"detectionUncertain": "Uncertain format — only {{read}} of {{total}} rows read",
"detectionUncertainHint": "Check the column mapping and the date format, then look at the preview before importing."
"detectionUncertainHint": "Check the column mapping and the date format, then look at the preview before importing.",
"detectionBank": "{{bank}} format recognized — {{read}} of {{total}} rows read"
},
"drift": {
"title": "This source's format has changed",
"intro": "This file's header row no longer matches the one recorded at the last successful import. The stored columns may no longer point at the right data.",
"columnMoved": "{{label}}: column {{from}} → {{to}}",
"columnAdded": "{{label}}: new column, at position {{to}}",
"columnRemoved": "{{label}}: column gone, it was at position {{from}}",
"adopt": "Adopt the re-detected format",
"adoptHint": "Reads the file the way it looks today. The new configuration is saved at import time.",
"adoptUnavailable": "Detection could not read this new shape: fix the mapping at the previous step.",
"keep": "Keep the current configuration",
"keepHint": "Keeps the stored mapping. Use this when the header changed but the columns did not move."
},
"repairPath": {
"title": "Repairing an import that was already written wrong",
"body": "Do not re-import a file you have already imported in order to correct it: duplicates are matched on the date, the description AND the amount, so a row with a corrected amount does not match the faulty row — it is added next to it. A flipped sign also produces two mirror rows that cancel each other out in every report. Delete the faulty import from the import history first, then replay the file."
},
"preview": {
"title": "Data Preview",

19
src/i18n/locales/fr.json
View file

@ -116,7 +116,24 @@
"updateTemplate": "Mettre à jour le modèle",
"detectionRecognized": "Format reconnu — {{read}} des {{total}} lignes lues",
"detectionUncertain": "Format incertain — {{read}} des {{total}} lignes lues seulement",
"detectionUncertainHint": "Vérifiez le mapping des colonnes et le format de date, puis regardez l'aperçu avant d'importer."
"detectionUncertainHint": "Vérifiez le mapping des colonnes et le format de date, puis regardez l'aperçu avant d'importer.",
"detectionBank": "Format {{bank}} reconnu — {{read}} des {{total}} lignes lues"
},
"drift": {
"title": "Le format de cette source a changé",
"intro": "L'en-tête de ce fichier ne correspond plus à celui du dernier import réussi. Les colonnes mémorisées ne pointent peut-être plus sur les bonnes données.",
"columnMoved": "{{label}} : colonne {{from}} → {{to}}",
"columnAdded": "{{label}} : nouvelle colonne, en position {{to}}",
"columnRemoved": "{{label}} : colonne disparue, elle était en position {{from}}",
"adopt": "Adopter le format re-détecté",
"adoptHint": "Relit le fichier tel qu'il se présente aujourd'hui. La nouvelle configuration est mémorisée à l'import.",
"adoptUnavailable": "La détection n'a pas su lire cette nouvelle forme : corrigez le mapping à l'étape précédente.",
"keep": "Conserver la configuration actuelle",
"keepHint": "Garde le mapping mémorisé. À utiliser si l'en-tête a changé sans que les colonnes bougent."
},
"repairPath": {
"title": "Réparer un import déjà écrit de travers",
"body": "Ne réimportez pas un fichier déjà importé pour le corriger : les doublons sont repérés sur la date, la description ET le montant, donc une ligne au montant corrigé ne s'apparie pas à la ligne fautive — elle s'ajoute. Une inversion de signe crée en prime deux lignes miroir qui s'annulent dans les rapports. Supprimez d'abord l'import fautif dans l'historique des imports, puis rejouez le fichier."
},
"preview": {
"title": "Aperçu des données",

18
src/pages/ImportPage.tsx
View file

@ -4,6 +4,7 @@ import ImportFolderConfig from "../components/import/ImportFolderConfig";
import SourceList from "../components/import/SourceList";
import SourceConfigPanel from "../components/import/SourceConfigPanel";
import FilePreviewTable from "../components/import/FilePreviewTable";
import FormatDriftPanel from "../components/import/FormatDriftPanel";
import DuplicateCheckPanel from "../components/import/DuplicateCheckPanel";
import ImportConfirmation from "../components/import/ImportConfirmation";
import ImportProgress from "../components/import/ImportProgress";
@ -29,6 +30,8 @@ export default function ImportPage() {
executeImport,
goToStep,
reset,
adoptDriftFormat,
keepCurrentFormat,
autoDetectConfig,
saveConfigAsTemplate,
applyConfigTemplate,
@ -102,6 +105,7 @@ export default function ImportPage() {
onDeleteTemplate={deleteConfigTemplate}
selectedTemplateId={state.selectedTemplateId}
detectionScore={state.detectionScore}
detectedBank={state.detectedBank}
isLoading={state.isLoading}
/>
{/*
@ -122,6 +126,20 @@ export default function ImportPage() {
{state.step === "file-preview" && (
<div className="space-y-6">
{/*
Format drift (#330) — ABOVE the table, because it tells the user
whether the table below is worth reading. Present only when the
header row moved since the last successful import of this source.
*/}
{state.formatDrift && (
<FormatDriftPanel
entries={state.formatDrift}
canAdopt={state.driftConfig !== null}
onAdopt={adoptDriftFormat}
onKeep={keepCurrentFormat}
isBusy={state.isLoading}
/>
)}
<FilePreviewTable
rows={state.parsedPreview}
onFlipSigns={flipSignConvention}

9
src/services/importSourceService.ts
View file

@ -52,8 +52,8 @@ export async function createSource(
): Promise<number> {
const db = await getDb();
const result = await db.execute(
`INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention, template_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
`INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention, header_signature, template_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT(name) DO UPDATE SET
description = excluded.description,
date_format = excluded.date_format,
@ -64,6 +64,7 @@ export async function createSource(
has_header = excluded.has_header,
amount_mode = excluded.amount_mode,
sign_convention = excluded.sign_convention,
header_signature = excluded.header_signature,
template_id = excluded.template_id,
updated_at = CURRENT_TIMESTAMP`,
[
@ -77,6 +78,10 @@ export async function createSource(
source.has_header,
source.amount_mode,
source.sign_convention,
// Explicitly null for a headerless file: drift detection is inoperative
// there and a stale signature from an earlier shape would be worse than
// none (#330).
source.header_signature ?? null,
source.template_id ?? null,
]
);

695
src/utils/bankSignatures.test.ts Normal file
View file

@ -0,0 +1,695 @@
// Bank signatures and format drift (#330).
//
// Two contracts are frozen here.
//
// THE SIGNATURES. Each of the four banks has a fixture carrying its documented
// header layout. The test that matters is not "the signature matches" — it is
// the pair: the same file WITH its bank header and WITHOUT it. Three of the
// four layouts are read wrong by the generic dictionary, and the counterfactual
// is what proves the signature earns its place instead of merely agreeing with
// the heuristic. The other half of that pair is the fall-back: every fixture of
// the #326 corpus must still be detected with no bank at all.
//
// THE DRIFT. `header_signature` records the labels of the last successful
// import; a later file whose header normalizes differently is drift. The
// interesting cases are the ones that must NOT fire: no stored signature, a
// headerless file, a stored value that will not parse, and a cosmetic rename
// (`Montant` -> `MONTANT ($)`) that normalizes to the same label.
import { describe, it, expect } from "vitest";
import { readFileSync } from "fs";
import { resolve } from "path";
import {
BANK_SIGNATURES,
MIN_SIGNATURE_LABELS,
bankSignatureById,
buildHeaderSignature,
detectHeaderDrift,
matchBankSignature,
parseHeaderSignature,
type BankSignatureId,
} from "./bankSignatures";
import { normalizeHeaderCell } from "./headerDictionary";
import { detectImportFormat } from "./csvAutoDetect";
import { CSV_FIXTURE_NAMES, readCsvFixture } from "../__fixtures__/csv";
import fr from "../i18n/locales/fr.json";
import en from "../i18n/locales/en.json";
/** The bank a file is recognised as, or null. Throws on a file detection refuses. */
function bankOf(content: string): BankSignatureId | null {
const outcome = detectImportFormat(content);
if (outcome.status !== "ok") {
throw new Error(`detection returned ${outcome.status}`);
}
return outcome.bank;
}
/** The configuration detection settles on. Throws on a file it refuses. */
function configOf(content: string) {
const outcome = detectImportFormat(content);
if (outcome.status !== "ok") {
throw new Error(`detection returned ${outcome.status}`);
}
return outcome.config;
}
/** The same file with its header row replaced — every data row untouched. */
function withHeader(raw: string, header: string): string {
const lines = raw.split("\n");
lines[0] = header;
return lines.join("\n");
}
describe("the signature table is well formed (#330)", () => {
it("gives every bank a distinct id", () => {
const ids = BANK_SIGNATURES.map((s) => s.id);
expect(new Set(ids).size).toBe(ids.length);
});
it("declares every label already normalized", () => {
// A label written `Montant` would never match: the header cell is
// normalized before the comparison and the table's side is not.
for (const signature of BANK_SIGNATURES) {
for (const variant of signature.variants) {
for (const label of variant.labels) {
expect(normalizeHeaderCell(label), `${signature.id}: ${label}`).toBe(
label
);
}
}
}
});
it("keeps every role's label inside the fingerprint it projects", () => {
// A role naming a label absent from `labels` would resolve to a column the
// match never checked for.
for (const signature of BANK_SIGNATURES) {
for (const variant of signature.variants) {
for (const [role, label] of Object.entries(variant.roles)) {
expect(variant.labels, `${signature.id}.${role}`).toContain(label);
}
}
}
});
it("refuses a fingerprint too small to identify anyone", () => {
// `Date;Description;Montant` is the shape of half the corpus and of any
// hand-made export. A three-label variant would put a bank's name over
// files nobody can attribute to it.
for (const signature of BANK_SIGNATURES) {
for (const variant of signature.variants) {
expect(
variant.labels.length,
`${signature.id}: ${variant.labels.join(",")}`
).toBeGreaterThanOrEqual(MIN_SIGNATURE_LABELS);
}
}
});
it("resolves a bank from its id, and nothing from an unknown one", () => {
expect(bankSignatureById("desjardins")?.label).toBe("Desjardins");
expect(bankSignatureById(null)).toBeNull();
expect(bankSignatureById(undefined)).toBeNull();
});
});
describe("each bank is recognised on its own fixture (#330)", () => {
it("reads the Desjardins layout and names it", () => {
expect(bankOf(readCsvFixture("bank-desjardins"))).toBe("desjardins");
expect(configOf(readCsvFixture("bank-desjardins"))).toEqual({
delimiter: ";",
hasHeader: true,
skipLines: 0,
dateFormat: "DD/MM/YYYY",
// Solde (column 3) is the balance the signature names, and it stays out
// of the amount candidates.
columnMapping: { date: 0, description: 1, amount: 2 },
amountMode: "single",
signConvention: "negative_expense",
});
});
it("reads the English Desjardins header too", () => {
const content =
"Date;Description;Amount;Balance\n" +
"05/01/2025;EPICERIE METRO;-84,32;915,68\n" +
"15/01/2025;DEPOT PAIE;1250,00;2165,68\n" +
"18/01/2025;HYDRO QUEBEC;-142,18;2023,50\n" +
"22/01/2025;RESTAURANT;-56,75;1966,75\n";
expect(bankOf(content)).toBe("desjardins");
});
it("reads a whole-line-quoted Desjardins export", () => {
// `preprocessQuotedCSV` unwraps it into a comma-separated file; the
// signature declares the quirk and the comma, so the match survives it.
const content =
'"Date,""Description"",Montant,Solde"\n' +
'"05/01/2025,""EPICERIE METRO"",-84.32,915.68"\n' +
'"15/01/2025,""DEPOT PAIE"",1250.00,2165.68"\n' +
'"18/01/2025,""HYDRO QUEBEC"",-142.18,2023.50"\n' +
'"22/01/2025,""RESTAURANT"",-56.75,1966.75"\n';
expect(bankOf(content)).toBe("desjardins");
});
it("reads the RBC layout and maps CAD$ as the amount", () => {
expect(bankOf(readCsvFixture("bank-rbc"))).toBe("rbc");
expect(configOf(readCsvFixture("bank-rbc"))).toEqual({
delimiter: ",",
hasHeader: true,
skipLines: 0,
dateFormat: "DD/MM/YYYY",
columnMapping: { date: 2, description: 4, amount: 6 },
amountMode: "single",
signConvention: "negative_expense",
});
});
it("reads the Banque Nationale layout as a debit/credit pair", () => {
expect(bankOf(readCsvFixture("bank-bnc"))).toBe("bnc");
expect(configOf(readCsvFixture("bank-bnc"))).toEqual({
delimiter: ";",
hasHeader: true,
skipLines: 0,
dateFormat: "DD/MM/YYYY",
columnMapping: {
date: 0,
description: 1,
debitAmount: 3,
creditAmount: 4,
},
amountMode: "debit_credit",
signConvention: "negative_expense",
});
});
it("reads the Tangerine layout and maps Name as the description", () => {
expect(bankOf(readCsvFixture("bank-tangerine"))).toBe("tangerine");
expect(configOf(readCsvFixture("bank-tangerine"))).toEqual({
delimiter: ",",
hasHeader: true,
skipLines: 0,
dateFormat: "DD/MM/YYYY",
columnMapping: { date: 0, description: 2, amount: 4 },
amountMode: "single",
signConvention: "negative_expense",
});
});
});
describe("what the signatures actually buy (#330)", () => {
it("keeps RBC's cheque number out of the amounts", () => {
// The SAME rows, under a header no signature knows. `CAD$` and
// `Cheque Number` are sparse-complementary — one cheque number in six rows
// — so the shape scan pairs them as debit/credit and the row carrying a
// cheque number imports as -247.95 instead of -6.95. Nothing in the file
// can tell that apart from a genuine debit/credit pair; a documented layout
// can.
const anonymised = withHeader(
readCsvFixture("bank-rbc"),
"Type,Numero,Date,Cheque,Libelle,Note,Montant,Devise"
);
expect(bankOf(anonymised)).toBeNull();
expect(configOf(anonymised).amountMode).toBe("debit_credit");
expect(configOf(anonymised).columnMapping).toEqual({
date: 2,
description: 4,
debitAmount: 3,
creditAmount: 6,
});
// With the header the bank actually prints, the same rows read as one
// signed column.
expect(configOf(readCsvFixture("bank-rbc")).amountMode).toBe("single");
});
it("keeps Tangerine's direction column out of the description", () => {
// `Transaction` is a description keyword of the generic dictionary, and
// Tangerine's `Transaction` column holds DEBIT / CREDIT. Read as the
// description, every transaction of the file is labelled `DEBIT`.
const anonymised = withHeader(
readCsvFixture("bank-tangerine"),
"Date,Transaction,Nom,Note,Montant"
);
expect(bankOf(anonymised)).toBeNull();
expect(configOf(anonymised).columnMapping.description).toBe(1);
expect(configOf(readCsvFixture("bank-tangerine")).columnMapping.description).toBe(
2
);
});
});
describe("an unknown file falls back to the generic dictionary (#330)", () => {
it("names no bank on any of the shape fixtures", () => {
// The corpus frozen by #326 is synthetic and belongs to no bank. A single
// false positive here is a banner claiming a bank the file is not from.
for (const name of CSV_FIXTURE_NAMES) {
if (name.startsWith("bank-")) continue;
if (name === "absolute-indicator") continue; // refused before any bank
expect(bankOf(readCsvFixture(name)), name).toBeNull();
}
});
it("leaves the pre-#330 configurations untouched", () => {
// The signatures must not have moved a single mapping of the corpus.
expect(configOf(readCsvFixture("signed-amount"))).toEqual({
delimiter: ";",
hasHeader: true,
skipLines: 0,
dateFormat: "DD/MM/YYYY",
columnMapping: { date: 0, description: 1, amount: 2 },
amountMode: "single",
signConvention: "negative_expense",
});
});
});
describe("the preconditions of a match (#330)", () => {
const rows =
"05/01/2025;EPICERIE METRO;-84,32;915,68\n" +
"15/01/2025;DEPOT PAIE;1250,00;2165,68\n" +
"18/01/2025;HYDRO QUEBEC;-142,18;2023,50\n" +
"22/01/2025;RESTAURANT;-56,75;1966,75\n";
it("refuses a delimiter the bank does not use", () => {
const tabbed = ("Date;Description;Montant;Solde\n" + rows).replace(
/;/g,
"\t"
);
expect(bankOf(tabbed)).toBeNull();
});
it("refuses an export carrying more preamble than the bank prints", () => {
const content =
"RELEVE DE COMPTE\nCompte 12345\nDate;Description;Montant;Solde\n" + rows;
expect(bankOf(content)).toBeNull();
});
it("refuses a whole-line-quoted file for a bank that does not quote", () => {
// Tangerine's labels, wrapped the way Desjardins wraps its exports. The
// quirk is what says this is not a Tangerine file.
const content =
'"Date,""Transaction"",Name,Memo,Amount"\n' +
'"05/01/2025,""DEBIT"",EPICERIE,,-84.32"\n' +
'"15/01/2025,""CREDIT"",PAIE,,1250.00"\n' +
'"18/01/2025,""DEBIT"",HYDRO,,-142.18"\n' +
'"22/01/2025,""DEBIT"",RESTO,,-56.75"\n';
expect(bankOf(content)).toBeNull();
});
it("names no bank for a headerless file, whatever its shape", () => {
// There is no label to read. This is the same boundary that leaves
// `header_signature` null on those sources.
expect(bankOf(readCsvFixture("no-header"))).toBeNull();
});
it("matches on the whole label, never on a substring", () => {
// `matchHeaderColumn` matches substrings — that is the generic dictionary's
// rule and the reason it mis-reads these layouts. A fingerprint compared
// loosely would inherit the same problem.
expect(
matchBankSignature({
headerRow: ["Date", "Description", "Montant net", "Solde"],
delimiter: ";",
skipLines: 0,
wholeLineQuoted: false,
})
).toBeNull();
});
it("resolves the roles to the columns of THIS file, not to the declared order", () => {
const match = matchBankSignature({
headerRow: ["Solde", "Montant", "Description", "Date"],
delimiter: ";",
skipLines: 0,
wholeLineQuoted: false,
});
expect(match?.signature.id).toBe("desjardins");
expect(match?.roles).toEqual({
date: 3,
description: 2,
amount: 1,
debit: null,
credit: null,
balance: 0,
roleCount: 4,
});
});
});
describe("the stored signature (#330)", () => {
it("stores the normalized labels, in order", () => {
expect(buildHeaderSignature(["Date", "Description", "Montant", "Solde"])).toBe(
'["date","description","montant","solde"]'
);
});
it("stores nothing for a file with no header row", () => {
// The whole point of the boundary: `Col 0`, `Col 1` … is not a signature,
// and inventing one from the data would fire on every change of content.
expect(buildHeaderSignature(null)).toBeNull();
expect(buildHeaderSignature([])).toBeNull();
});
it("reads back what it wrote", () => {
const stored = buildHeaderSignature(["Date", "Montant"]);
expect(parseHeaderSignature(stored)).toEqual(["date", "montant"]);
});
it("returns null rather than throwing on anything it did not write", () => {
// A hash left by an older build, a truncated row, hand-edited SQL: drift
// detection switches off, an import never blows up.
expect(parseHeaderSignature(null)).toBeNull();
expect(parseHeaderSignature("")).toBeNull();
expect(parseHeaderSignature("d41d8cd98f00b204")).toBeNull();
expect(parseHeaderSignature("{}")).toBeNull();
expect(parseHeaderSignature("[1,2,3]")).toBeNull();
});
});
describe("drift detection stays silent when it should (#330)", () => {
const stored = buildHeaderSignature(["Date", "Description", "Montant"]);
it("says nothing about a source that never recorded a signature", () => {
expect(detectHeaderDrift(null, ["Date", "Solde"])).toBeNull();
expect(detectHeaderDrift(undefined, ["Date", "Solde"])).toBeNull();
});
it("says nothing about a headerless file", () => {
expect(detectHeaderDrift(stored, null)).toBeNull();
expect(detectHeaderDrift(stored, [])).toBeNull();
});
it("says nothing about an identical header", () => {
expect(
detectHeaderDrift(stored, ["Date", "Description", "Montant"])
).toBeNull();
});
it("says nothing about a cosmetic rename", () => {
// Accents, case, punctuation and spacing are stripped before the
// comparison: none of them changes which column holds what.
expect(
detectHeaderDrift(stored, ["DATE", "Description ", "MONTANT ($)"])
).toBeNull();
});
it("says nothing when the stored value cannot be read", () => {
expect(detectHeaderDrift("not json", ["Date"])).toBeNull();
});
});
describe("drift detection names the columns that moved (#330)", () => {
it("reports a column that changed position", () => {
const stored = buildHeaderSignature([
"Date",
"Description",
"Montant",
"Solde",
]);
expect(
detectHeaderDrift(stored, ["Date", "Description", "Solde", "Montant"])
).toEqual([
{
kind: "moved",
normalizedLabel: "solde",
label: "Solde",
previousIndex: 3,
currentIndex: 2,
},
{
kind: "moved",
normalizedLabel: "montant",
label: "Montant",
previousIndex: 2,
currentIndex: 3,
},
]);
});
it("reports a new column and the shift it causes", () => {
const stored = buildHeaderSignature(["Date", "Description", "Montant"]);
expect(
detectHeaderDrift(stored, ["Date", "Devise", "Description", "Montant"])
).toEqual([
{
kind: "added",
normalizedLabel: "devise",
label: "Devise",
previousIndex: null,
currentIndex: 1,
},
{
kind: "moved",
normalizedLabel: "description",
label: "Description",
previousIndex: 1,
currentIndex: 2,
},
{
kind: "moved",
normalizedLabel: "montant",
label: "Montant",
previousIndex: 2,
currentIndex: 3,
},
]);
});
it("reports a column that disappeared, under its stored label", () => {
// The raw spelling of a removed column is recorded nowhere — only the
// normalized label survives, and that is what is shown.
const stored = buildHeaderSignature(["Date", "Description", "Solde"]);
expect(detectHeaderDrift(stored, ["Date", "Description"])).toEqual([
{
kind: "removed",
normalizedLabel: "solde",
label: "solde",
previousIndex: 2,
currentIndex: null,
},
]);
});
it("survives a bank swapping its amount column for a debit/credit pair", () => {
const stored = buildHeaderSignature([
"Date",
"Description",
"Montant",
"Solde",
]);
const drift = detectHeaderDrift(stored, [
"Date",
"Description",
"Débit",
"Crédit",
"Solde",
])!;
expect(drift.map((e) => [e.kind, e.normalizedLabel])).toEqual([
["added", "debit"],
["added", "credit"],
["moved", "solde"],
["removed", "montant"],
]);
});
});
// ---------------------------------------------------------------------------
// Static guards. The wiring lives in a React hook and two components, and the
// repository has no jsdom — same technique as the guards of #324 to #329.
// ---------------------------------------------------------------------------
const WIZARD_SRC = readFileSync(
resolve(import.meta.dirname, "..", "hooks", "useImportWizard.ts"),
"utf-8"
);
const PAGE_SRC = readFileSync(
resolve(import.meta.dirname, "..", "pages", "ImportPage.tsx"),
"utf-8"
);
const componentSrc = (name: string) =>
readFileSync(
resolve(import.meta.dirname, "..", "components", "import", name),
"utf-8"
);
describe("the signature is written at every successful import (#330)", () => {
it("builds it from the header row the import actually read", () => {
expect(WIZARD_SRC).toContain("buildHeaderSignature(");
expect(WIZARD_SRC).toContain(
"config.hasHeader ? state.previewHeaders : null"
);
});
it("writes it on both arms of the single write point", () => {
const body = WIZARD_SRC.slice(
WIZARD_SRC.indexOf("const executeImport ="),
WIZARD_SRC.indexOf("const goToStep =")
);
expect(body.match(/header_signature: headerSignature/g)).toHaveLength(2);
});
it("keeps it out of the format codec", () => {
// It is drift metadata, not one of the eight fields that decide how a row
// is read. `formatToRow` naming it would put it back in the format.
const CODEC = readFileSync(
resolve(import.meta.dirname, "importFormat.ts"),
"utf-8"
);
expect(CODEC).not.toContain("header_signature");
expect(CODEC).not.toContain("headerSignature");
});
});
describe("drift is computed on the way to the preview (#330)", () => {
const body = () =>
WIZARD_SRC.slice(
WIZARD_SRC.indexOf("const parseAndPreview ="),
WIZARD_SRC.indexOf("const checkDuplicatesInternal =")
);
it("compares the stored signature to the headers just parsed", () => {
expect(body()).toContain("detectHeaderDrift(");
expect(body()).toContain("state.existingSource?.header_signature");
});
it("does not compare anything on a headerless file", () => {
expect(body()).toContain("state.sourceConfig.hasHeader");
});
it("still reaches the preview step", () => {
expect(body()).toContain(
'dispatch({ type: "SET_STEP", payload: "file-preview" })'
);
});
it("drops the score the re-detection produced, which describes another format", () => {
// The re-detection measures the format the PANEL offers, not the one in
// use. Keeping its score would show "Format reconnu — 6 of 6 rows read"
// beside the stored mapping as soon as the user steps back.
expect(body()).toContain(
'dispatch({ type: "SET_DETECTION_SCORE", payload: null })'
);
});
});
describe("the drift panel offers two outcomes and writes neither (#330)", () => {
const PANEL = componentSrc("FormatDriftPanel.tsx");
it("is rendered on the preview step, above the table", () => {
expect(PAGE_SRC).toContain("{state.formatDrift && (");
expect(PAGE_SRC).toContain("<FormatDriftPanel");
expect(PAGE_SRC).toContain("onAdopt={adoptDriftFormat}");
expect(PAGE_SRC).toContain("onKeep={keepCurrentFormat}");
expect(PAGE_SRC.indexOf("<FormatDriftPanel")).toBeLessThan(
PAGE_SRC.indexOf("<FilePreviewTable")
);
});
it("re-parses under the adopted format instead of re-reading state", () => {
// `state` has not re-rendered when the parse starts, exactly as for the
// sign flip: reading it back would redisplay the table just replaced.
const body = WIZARD_SRC.slice(
WIZARD_SRC.indexOf("const adoptDriftFormat ="),
WIZARD_SRC.indexOf("const keepCurrentFormat =")
);
expect(body).toContain("parseFilesInternal(adopted)");
expect(body).not.toContain("createSource(");
expect(body).not.toContain("updateSource(");
});
it("renders the three kinds of change through i18n", () => {
for (const key of ["columnMoved", "columnAdded", "columnRemoved"]) {
expect(PANEL, key).toContain(`import.drift.${key}`);
}
});
});
describe("the repair path is stated in the interface (#330)", () => {
const NOTICE = componentSrc("RepairPathNotice.tsx");
it("names the history deletion rather than a vague warning", () => {
expect(NOTICE).toContain("import.repairPath.title");
expect(NOTICE).toContain("import.repairPath.body");
for (const locale of [fr, en]) {
expect(locale.import.repairPath.body.length).toBeGreaterThan(0);
}
// The two facts a user needs: duplicates are matched on the amount, and the
// faulty import has to go first.
expect(fr.import.repairPath.body).toContain("historique");
expect(en.import.repairPath.body).toContain("history");
});
it("appears in the preview and in the drift panel, not in one of the two", () => {
expect(componentSrc("FilePreviewTable.tsx")).toContain("<RepairPathNotice />");
expect(componentSrc("FormatDriftPanel.tsx")).toContain("<RepairPathNotice />");
});
});
describe("the recognised-bank banner (#330)", () => {
const PANEL = componentSrc("SourceConfigPanel.tsx");
it("names the bank only when the rows actually read", () => {
// A bank label over a file two thirds of whose rows fail is a claim the app
// cannot back; the uncertain wording is the one that helps there.
expect(PANEL).toContain("detectionScore?.confident");
expect(PANEL).toContain("import.config.detectionBank");
});
it("interpolates the bank name instead of translating it", () => {
// A bank is a proper noun. `Desjardins` is `Desjardins` in both languages.
for (const locale of [fr, en]) {
expect(locale.import.config.detectionBank).toContain("{{bank}}");
expect(locale.import.config.detectionBank).toContain("{{read}}");
expect(locale.import.config.detectionBank).toContain("{{total}}");
}
});
it("clears the bank with the score, in one place", () => {
// Three sites drop the score (a hand edit, a template, a sign flip). The
// reducer clearing the bank alongside it is what keeps a fourth from
// forgetting.
const reducerCase = WIZARD_SRC.slice(
WIZARD_SRC.indexOf('case "SET_DETECTION_SCORE":'),
WIZARD_SRC.indexOf('case "SET_DETECTED_BANK":')
);
expect(reducerCase).toContain("detectedBank: action.payload === null");
});
});
describe("every new string exists in both languages (#330)", () => {
it("carries the drift panel", () => {
for (const key of [
"title",
"intro",
"columnMoved",
"columnAdded",
"columnRemoved",
"adopt",
"adoptHint",
"adoptUnavailable",
"keep",
"keepHint",
] as const) {
expect(fr.import.drift[key].length, `fr.${key}`).toBeGreaterThan(0);
expect(en.import.drift[key].length, `en.${key}`).toBeGreaterThan(0);
}
});
it("interpolates the column positions in both languages", () => {
for (const locale of [fr, en]) {
expect(locale.import.drift.columnMoved).toContain("{{label}}");
expect(locale.import.drift.columnMoved).toContain("{{from}}");
expect(locale.import.drift.columnMoved).toContain("{{to}}");
expect(locale.import.drift.columnAdded).toContain("{{to}}");
expect(locale.import.drift.columnRemoved).toContain("{{from}}");
}
});
it("carries the repair path", () => {
for (const key of ["title", "body"] as const) {
expect(fr.import.repairPath[key].length, `fr.${key}`).toBeGreaterThan(0);
expect(en.import.repairPath[key].length, `en.${key}`).toBeGreaterThan(0);
}
});
});

439
src/utils/bankSignatures.ts Normal file
View file

@ -0,0 +1,439 @@
/**
* Bank signatures and header-signature drift (#330).
*
* TWO THINGS LIVE HERE, and they are the same thing seen twice: a header row
* reduced to its normalized labels.
*
* - A BANK SIGNATURE is that label list written down in advance, per bank, so
* a known export is recognised by name instead of being guessed at.
* - A STORED SIGNATURE is that same label list recorded on the source at the
* last successful import (`import_sources.header_signature`), so the next
* file from the same bank can be compared against it column by column.
*
* WHY SIGNATURES RUN BEFORE THE GENERIC DICTIONARY. `headerDictionary.ts`
* matches keywords as SUBSTRINGS, one role at a time, and has no notion of a
* best match — the first column containing the keyword wins. That is the right
* rule for an unknown file and the wrong one for a known layout:
* - Tangerine writes `Date,Transaction,Name,Memo,Amount`. The dictionary reads
* `Transaction` as the description (it is a description keyword) and maps
* the transaction TYPE column — `DEBIT`/`CREDIT` — as the label of every
* row. The signature names `Name`.
* - RBC writes its amount column `CAD$`, which contains neither `montant` nor
* `amount`, so the dictionary finds no amount column at all and the shape
* heuristic pairs `CAD$` with the (usually empty) `USD$` as a debit/credit
* pair. The signature names `CAD$` as a single signed amount.
*
* WHY A FAILING SIGNATURE CANNOT BREAK ANYTHING. These signatures are written
* from documented export layouts, WITHOUT real statements — the app is
* privacy-first and no real statement lands in this repository. So a signature
* is only ever allowed to do what the generic dictionary already does: hand
* `detectImportFormat` a set of PREFERENCES. Every one of them is dropped the
* moment the data contradicts it (a date column nothing parses as a date, an
* amount column the shape scan never proposed). A file no signature recognises
* falls through to the dictionary, unchanged. Failing degrades; it never breaks.
*
* WHY THE STORED SIGNATURE IS A LABEL LIST AND NOT A HASH. The drift panel has
* to name the columns that moved ("Montant : 3 → 4"), which a hash forbids.
*/
import {
normalizeHeaderCell,
type LexicalHeaderMap,
} from "./headerDictionary";
/** Every bank the table knows, and the i18n-free proper noun it is shown as. */
export type BankSignatureId = "desjardins" | "rbc" | "bnc" | "tangerine";
/** The transaction roles a signature can name — `LexicalHeaderMap` minus its count. */
export type HeaderRole = keyof Omit<LexicalHeaderMap, "roleCount">;
/**
* One documented layout of one bank.
*
* `labels` is the FINGERPRINT: every one of them must appear in the header row,
* compared on the whole normalized cell (not as a substring). `roles` is a
* projection of that fingerprint onto the columns detection cares about —
* labels carrying no role (`Catégorie`, `Memo`, `N° de chèque`) stay in
* `labels` precisely because they are what makes the fingerprint distinctive.
*/
export interface BankHeaderVariant {
readonly labels: readonly string[];
readonly roles: Readonly<Partial<Record<HeaderRole, string>>>;
}
export interface BankSignature {
readonly id: BankSignatureId;
/** Proper noun. Displayed as-is in every language, never translated. */
readonly label: string;
/** Delimiters this bank's exports use. A precondition of the match, never an override. */
readonly delimiters: readonly string[];
/** Preamble lines this export prints before its header row. */
readonly maxPreambleLines: number;
/** The export wraps whole lines in quotes — see `preprocessQuotedCSV`. */
readonly wholeLineQuoted: boolean;
readonly variants: readonly BankHeaderVariant[];
}
/**
* Labels a variant must carry before it is allowed to claim a file.
*
* Three is not enough: `Date;Description;Montant` is the shape of half the
* synthetic corpus and of any hand-made export, so a three-label Desjardins
* variant would put "Format Desjardins reconnu" over files no one can attribute
* to Desjardins. Announcing the wrong bank is worse than announcing none — the
* generic path reads those files correctly already.
*/
export const MIN_SIGNATURE_LABELS = 4;
/**
* The table. Written from the banks' documented export layouts; none of it has
* been verified against a real statement (see the file header). Adding a bank
* is adding an entry — there is no code to write.
*/
export const BANK_SIGNATURES: readonly BankSignature[] = [
{
// AccèsD exports Date / Description / Montant / Solde, semicolon-separated,
// with comma decimals, in French or in English. Some credit-card exports
// wrap every line in quotes (handled upstream by `preprocessQuotedCSV`,
// which leaves a comma-separated file behind) and some carry NO header row
// at all — those cannot be signed, by construction.
id: "desjardins",
label: "Desjardins",
delimiters: [";", ","],
maxPreambleLines: 0,
wholeLineQuoted: true,
variants: [
{
labels: ["date", "description", "montant", "solde"],
roles: {
date: "date",
description: "description",
amount: "montant",
balance: "solde",
},
},
{
labels: ["date", "description", "amount", "balance"],
roles: {
date: "date",
description: "description",
amount: "amount",
balance: "balance",
},
},
],
},
{
// RBC exports one comma-separated file for every account, amounts in a
// currency-named column. `CAD$` normalizes to `cad`, which no generic
// amount keyword matches — this variant is the only thing that maps it.
id: "rbc",
label: "RBC",
delimiters: [","],
maxPreambleLines: 0,
wholeLineQuoted: false,
variants: [
{
labels: [
"accounttype",
"accountnumber",
"transactiondate",
"chequenumber",
"description1",
"cad",
],
roles: {
date: "transactiondate",
description: "description1",
amount: "cad",
},
},
],
},
{
// Banque Nationale exports a debit/credit pair next to a category column,
// semicolon-separated in French.
id: "bnc",
label: "Banque Nationale",
delimiters: [";", ","],
maxPreambleLines: 0,
wholeLineQuoted: false,
variants: [
{
labels: ["date", "description", "categorie", "debit", "credit", "solde"],
roles: {
date: "date",
description: "description",
debit: "debit",
credit: "credit",
balance: "solde",
},
},
{
labels: ["date", "description", "category", "debit", "credit", "balance"],
roles: {
date: "date",
description: "description",
debit: "debit",
credit: "credit",
balance: "balance",
},
},
],
},
{
// Tangerine exports Date,Transaction,Name,Memo,Amount — comma-separated,
// signed amounts. `Transaction` holds the direction word, NOT the label of
// the operation; reading it as the description makes every row read
// `DEBIT` or `CREDIT`.
id: "tangerine",
label: "Tangerine",
delimiters: [","],
maxPreambleLines: 0,
wholeLineQuoted: false,
variants: [
{
labels: ["date", "transaction", "name", "memo", "amount"],
roles: { date: "date", description: "name", amount: "amount" },
},
],
},
];
/** What the caller needs to know before a signature may claim a file. */
export interface BankSignatureInput {
/** The header row as parsed, raw cells. */
readonly headerRow: readonly string[];
/** Delimiter detection settled on. */
readonly delimiter: string;
/** Preamble lines detection had to skip. */
readonly skipLines: number;
/** True when `preprocessQuotedCSV` actually unwrapped the file. */
readonly wholeLineQuoted: boolean;
}
export interface BankSignatureMatch {
readonly signature: BankSignature;
readonly variant: BankHeaderVariant;
/** The variant's roles, resolved to column indices of THIS file. */
readonly roles: LexicalHeaderMap;
}
/**
* First-occurrence index of every normalized label of a header row.
*
* First occurrence, not last: a file repeating a label (`Montant;…;Montant`)
* is degenerate either way, and taking the leftmost keeps the result stable
* between the matcher and the drift diff, which both read this map.
*/
function labelIndex(headerRow: readonly string[]): Map<string, number> {
const index = new Map<string, number>();
headerRow.forEach((cell, i) => {
const label = normalizeHeaderCell(cell);
if (label && !index.has(label)) index.set(label, i);
});
return index;
}
function rolesOf(
variant: BankHeaderVariant,
index: ReadonlyMap<string, number>
): LexicalHeaderMap {
const columnOf = (role: HeaderRole): number | null => {
const label = variant.roles[role];
if (label === undefined) return null;
return index.get(label) ?? null;
};
const date = columnOf("date");
const description = columnOf("description");
const amount = columnOf("amount");
const debit = columnOf("debit");
const credit = columnOf("credit");
const balance = columnOf("balance");
return {
date,
description,
amount,
debit,
credit,
balance,
roleCount: [date, description, amount, debit, credit, balance].filter(
(c) => c !== null
).length,
};
}
/**
* Find the bank whose documented layout this header row is.
*
* Three preconditions guard every match, each one a way the file says it is not
* this export: a delimiter the bank does not use, more preamble than the bank
* prints, or a whole-line-quoted file claiming to be a bank that does not quote.
* They can only ever turn a match into a fall-back to the generic dictionary.
*
* Returns null for an unknown file — which is the common case and not an error.
*/
export function matchBankSignature(
input: BankSignatureInput
): BankSignatureMatch | null {
const index = labelIndex(input.headerRow);
if (index.size === 0) return null;
for (const signature of BANK_SIGNATURES) {
if (!signature.delimiters.includes(input.delimiter)) continue;
if (input.skipLines > signature.maxPreambleLines) continue;
if (input.wholeLineQuoted && !signature.wholeLineQuoted) continue;
for (const variant of signature.variants) {
if (!variant.labels.every((label) => index.has(label))) continue;
return { signature, variant, roles: rolesOf(variant, index) };
}
}
return null;
}
/** The bank behind an id, for the banner. Null for an id the table lost. */
export function bankSignatureById(
id: BankSignatureId | null | undefined
): BankSignature | null {
if (!id) return null;
return BANK_SIGNATURES.find((s) => s.id === id) ?? null;
}
// ---------------------------------------------------------------------------
// The stored signature and the drift it detects
// ---------------------------------------------------------------------------
/**
* The header row of a successful import, as `import_sources.header_signature`
* stores it: a JSON array of normalized labels.
*
* Returns null for a file with NO header row — and that null is written to the
* column as-is. A headerless source has no signature to compare against and
* drift detection is inoperative on it, deliberately: a signature invented from
* the data would fire on every change of content, which is every import.
*/
export function buildHeaderSignature(
headers: readonly string[] | null | undefined
): string | null {
if (!headers || headers.length === 0) return null;
return JSON.stringify(headers.map(normalizeHeaderCell));
}
/**
* Read back a stored signature. Anything that is not an array of strings — a
* hash written by an older build, a truncated row, hand-edited SQL — comes back
* null, which switches drift detection off rather than throwing inside an
* import. Same rule as everywhere else here: degrade, never break.
*/
export function parseHeaderSignature(
stored: string | null | undefined
): string[] | null {
if (!stored) return null;
try {
const parsed: unknown = JSON.parse(stored);
if (!Array.isArray(parsed)) return null;
if (!parsed.every((label) => typeof label === "string")) return null;
return parsed as string[];
} catch {
return null;
}
}
export type HeaderDriftKind = "moved" | "added" | "removed";
export interface HeaderDriftEntry {
readonly kind: HeaderDriftKind;
/** The normalized label — the key the two signatures are compared on. */
readonly normalizedLabel: string;
/**
* What to show. The file's own spelling for a column that still exists
* (`Montant`), the normalized label for one that disappeared — the raw
* spelling of a removed column is not recorded anywhere.
*/
readonly label: string;
/** Column index at the last successful import; null for a new column. */
readonly previousIndex: number | null;
/** Column index in the file being imported; null for a column that vanished. */
readonly currentIndex: number | null;
}
/**
* Compare the header row of the file being imported against the one the last
* successful import recorded.
*
* Returns null when there is nothing to say — no stored signature, a headerless
* file, an unreadable stored value, or a header that normalizes to exactly the
* same labels in the same order. `Montant` becoming `MONTANT ($)` is NOT drift:
* it normalizes identically and changes nothing about how the file is read.
*
* A non-null result is always a non-empty list, so the caller can treat it as
* "show the panel".
*/
export function detectHeaderDrift(
stored: string | null | undefined,
currentHeaders: readonly string[] | null | undefined
): HeaderDriftEntry[] | null {
const previous = parseHeaderSignature(stored);
if (!previous || previous.length === 0) return null;
if (!currentHeaders || currentHeaders.length === 0) return null;
const current = currentHeaders.map(normalizeHeaderCell);
if (
previous.length === current.length &&
previous.every((label, i) => label === current[i])
) {
return null;
}
const previousIndex = new Map<string, number>();
previous.forEach((label, i) => {
if (label && !previousIndex.has(label)) previousIndex.set(label, i);
});
const currentIndex = labelIndex(currentHeaders);
const entries: HeaderDriftEntry[] = [];
// Walk the file being imported first, so the panel reads in its column order.
current.forEach((label, i) => {
if (!label) return;
if (currentIndex.get(label) !== i) return; // duplicate label, already handled
const before = previousIndex.get(label);
if (before === undefined) {
entries.push({
kind: "added",
normalizedLabel: label,
label: (currentHeaders[i] ?? "").trim() || label,
previousIndex: null,
currentIndex: i,
});
} else if (before !== i) {
entries.push({
kind: "moved",
normalizedLabel: label,
label: (currentHeaders[i] ?? "").trim() || label,
previousIndex: before,
currentIndex: i,
});
}
});
// Then the columns that are simply gone, in the order they used to be in.
previous.forEach((label, i) => {
if (!label) return;
if (previousIndex.get(label) !== i) return;
if (currentIndex.has(label)) return;
entries.push({
kind: "removed",
normalizedLabel: label,
label,
previousIndex: i,
currentIndex: null,
});
});
return entries.length > 0 ? entries : null;
}

13
src/utils/csvAutoDetect.test.ts
View file

@ -260,7 +260,8 @@ const REFERENCE_AMOUNTS = [-84.32, 1250, -142.18, -56.75, 300, -6.95];
describe("corpus integrity (#326)", () => {
it("exposes every declared fixture as readable, non-empty content", () => {
expect(CSV_FIXTURE_NAMES).toHaveLength(11);
// 11 shape cases (#326) + the 4 bank layouts (#330).
expect(CSV_FIXTURE_NAMES).toHaveLength(15);
for (const name of CSV_FIXTURE_NAMES) {
expect(readCsvFixture(name).trim().length).toBeGreaterThan(0);
}
@ -1126,10 +1127,14 @@ describe("useImportWizard — detection fires on its own (#328)", () => {
expect(restoreArm).not.toContain("runAutoDetect");
});
it("keeps a single detection path for the button and the automatic run", () => {
// One detector, two callers: the wand button and `selectSource`.
it("keeps a single detection path for every caller", () => {
// ONE detector. The count of callers is deliberate and rises when a
// feature earns it: the wand button and `selectSource` (#328), then the
// format-drift re-detection in `parseAndPreview` (#330), which goes through
// this path rather than calling `runAutoDetect` a second time — a second
// detector is the divergence class this chantier removes.
expect(WIZARD.match(/runAutoDetect\(/g)).toHaveLength(1);
expect(WIZARD.match(/await detectFormatForFile\(/g)).toHaveLength(2);
expect(WIZARD.match(/await detectFormatForFile\(/g)).toHaveLength(3);
});
it("drops the score as soon as the format is edited by hand", () => {

98
src/utils/csvAutoDetect.ts
View file

@ -1,6 +1,11 @@
import Papa from "papaparse";
import { parseDate } from "./dateParser";
import { parseFrenchAmount } from "./amountParser";
import {
matchBankSignature,
type BankSignatureId,
type BankSignatureMatch,
} from "./bankSignatures";
import {
CREDIT_INDICATOR_TOKENS,
DEBIT_INDICATOR_TOKENS,
@ -78,7 +83,19 @@ export const CONFIDENCE_THRESHOLD = 0.9;
* backwards" call for different messages. `detectImportFormat` separates them.
*/
export type AutoDetectOutcome =
| { status: "ok"; config: AutoDetectResult; score: DetectionScore }
| {
status: "ok";
config: AutoDetectResult;
score: DetectionScore;
/**
* The bank whose documented layout this file matched, or null when the
* generic dictionary read the header (#330). Deliberately NOT part of
* `config`: it says where the configuration came from, it is not one of
* the fields that decide how a row is read, and it is never persisted as
* format.
*/
bank: BankSignatureId | null;
}
| { status: "rejected"; reason: AutoDetectRejectionKey }
| { status: "failed" };
@ -148,6 +165,13 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
* naming a column the data contradicts (a "Date" column nothing parses as a
* date, a "Solde" column that is the only amount candidate left) is dropped and
* the shape heuristic decides, exactly as it did before #327.
*
* In FRONT of that lexical layer sits the bank-signature table (#330): a header
* row matching a documented export layout gets its roles from that layout and
* the generic dictionary is not consulted at all. The hints it produces are the
* same kind of preference — the file is reported as recognised, not read
* differently on trust. An unknown header falls straight through to the
* dictionary, which is what every file did before.
*/
export function detectImportFormat(rawContent: string): AutoDetectOutcome {
const failed: AutoDetectOutcome = { status: "failed" };
@ -193,9 +217,28 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
// Step 2: Detect header
const hasHeader = detectHeader(effectiveData[0]);
// Step 2b: Read the header labels. A headerless file yields no map at all,
// which is the explicit fall-back: every step below then runs on shape only.
const lexical = hasHeader ? matchTransactionHeaders(effectiveData[0]) : null;
// Step 2b: Try the known banks first (#330). A signature is claimed on the
// WHOLE normalized label, delimiter and preamble included, so an unknown file
// simply does not match and the generic dictionary reads it as before. A
// headerless file matches nothing by construction — there is no label to read.
const signature = hasHeader
? matchBankSignature({
headerRow: effectiveData[0],
delimiter,
skipLines,
// `preprocessQuotedCSV` returns its input untouched when the file is not
// whole-line-quoted, so this comparison IS the quirk.
wholeLineQuoted: content !== rawContent,
})
: null;
// Step 2c: Read the header labels. A signature that matched has already named
// the roles; otherwise the generic dictionary does. A headerless file yields
// no map at all, which is the explicit fall-back: every step below then runs
// on shape only.
const lexical = hasHeader
? (signature?.roles ?? matchTransactionHeaders(effectiveData[0]))
: null;
const dataStartIdx = hasHeader ? 1 : 0;
const sampleRows = effectiveData.slice(dataStartIdx, dataStartIdx + 20);
@ -254,7 +297,12 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
);
// Step 7: Determine amount mode
const amountResult = detectAmountMode(sampleRows, amountCandidates, lexical);
const amountResult = detectAmountMode(
sampleRows,
amountCandidates,
lexical,
signature
);
if (!amountResult) return failed;
// Step 7b: Refuse the third amount format — unsigned magnitudes plus a
@ -303,7 +351,12 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
// Step 8: Replay what we just decided, over the WHOLE file. The sample above
// is 20 rows because that is enough to decide a shape; the score is what the
// user is told, so it has to describe the actual file.
return { status: "ok", config, score: scoreConfig(config, data) };
return {
status: "ok",
config,
score: scoreConfig(config, data),
bank: signature?.signature.id ?? null,
};
}
/**
@ -711,13 +764,44 @@ interface DebitCreditResult {
type AmountModeResult = SingleAmountResult | DebitCreditResult;
/**
* Decide the amount mode, and which column(s) carry it.
*
* `signature` is the one hint that outranks the sparse-complementary scan, and
* only because that scan cannot be told apart from the truth by shape alone:
* RBC's `CAD$` and `USD$` ARE complementary — the USD column is empty on a
* Canadian account — so a file whose amounts are one signed column reads as a
* debit/credit pair, with every credit imported as an expense. A bank that
* documents its layout settles that; nothing else in the file can.
*
* It stays a preference all the same: the columns it names must be candidates
* the shape scan itself proposed. A signature naming a column that parses as
* nothing numeric is dropped here and the generic path decides, exactly like a
* mismatched label.
*/
function detectAmountMode(
rows: string[][],
amountCandidates: number[],
lexical: LexicalHeaderMap | null
lexical: LexicalHeaderMap | null,
signature: BankSignatureMatch | null
): AmountModeResult | null {
if (amountCandidates.length === 0) return null;
if (signature) {
const { debit, credit, amount } = signature.roles;
if (
debit !== null &&
credit !== null &&
amountCandidates.includes(debit) &&
amountCandidates.includes(credit)
) {
return { mode: "debit_credit", debitCol: debit, creditCol: credit };
}
if (amount !== null && amountCandidates.includes(amount)) {
return detectSingleAmount(rows, amount);
}
}
if (amountCandidates.length === 1) {
return detectSingleAmount(rows, amountCandidates[0]);
}