feat(import): recognise known bank layouts and report format drift #340
18 changed files with 1732 additions and 29 deletions
7
src/__fixtures__/csv/bank-bnc.csv
Normal file
7
src/__fixtures__/csv/bank-bnc.csv
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
Date;Description;Catégorie;Débit;Crédit;Solde
|
||||||
|
05/01/2025;EPICERIE METRO SAINTE-FOY;Alimentation;84,32;;915,68
|
||||||
|
15/01/2025;DEPOT PAIE EMPLOYEUR;Revenu;;1250,00;2165,68
|
||||||
|
18/01/2025;HYDRO QUEBEC PREAUTORISE;Services publics;142,18;;2023,50
|
||||||
|
22/01/2025;RESTAURANT LE BISTRO;Restaurants;56,75;;1966,75
|
||||||
|
27/01/2025;VIREMENT RECU;Transferts;;300,00;2266,75
|
||||||
|
30/01/2025;FRAIS MENSUELS;Frais bancaires;6,95;;2259,80
|
||||||
|
7
src/__fixtures__/csv/bank-desjardins.csv
Normal file
7
src/__fixtures__/csv/bank-desjardins.csv
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
Date;Description;Montant;Solde
|
||||||
|
05/01/2025;EPICERIE METRO SAINTE-FOY;-84,32;915,68
|
||||||
|
15/01/2025;DEPOT PAIE EMPLOYEUR;1250,00;2165,68
|
||||||
|
18/01/2025;HYDRO QUEBEC PREAUTORISE;-142,18;2023,50
|
||||||
|
22/01/2025;RESTAURANT LE BISTRO;-56,75;1966,75
|
||||||
|
27/01/2025;VIREMENT RECU;300,00;2266,75
|
||||||
|
30/01/2025;FRAIS MENSUELS;-6,95;2259,80
|
||||||
|
7
src/__fixtures__/csv/bank-rbc.csv
Normal file
7
src/__fixtures__/csv/bank-rbc.csv
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
Account Type,Account Number,Transaction Date,Cheque Number,Description 1,Description 2,CAD$,USD$
|
||||||
|
Chequing,1234567,05/01/2025,,EPICERIE METRO SAINTE-FOY,,-84.32,
|
||||||
|
Chequing,1234567,15/01/2025,,DEPOT PAIE EMPLOYEUR,PAIE,1250.00,
|
||||||
|
Chequing,1234567,18/01/2025,,HYDRO QUEBEC PREAUTORISE,,-142.18,
|
||||||
|
Chequing,1234567,22/01/2025,,RESTAURANT LE BISTRO,,-56.75,
|
||||||
|
Chequing,1234567,27/01/2025,,VIREMENT RECU,,300.00,
|
||||||
|
Chequing,1234567,30/01/2025,241,FRAIS MENSUELS,,-6.95,
|
||||||
|
7
src/__fixtures__/csv/bank-tangerine.csv
Normal file
7
src/__fixtures__/csv/bank-tangerine.csv
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
Date,Transaction,Name,Memo,Amount
|
||||||
|
05/01/2025,DEBIT,EPICERIE METRO SAINTE-FOY,,-84.32
|
||||||
|
15/01/2025,CREDIT,DEPOT PAIE EMPLOYEUR,Paie bimensuelle,1250.00
|
||||||
|
18/01/2025,DEBIT,HYDRO QUEBEC PREAUTORISE,,-142.18
|
||||||
|
22/01/2025,DEBIT,RESTAURANT LE BISTRO,,-56.75
|
||||||
|
27/01/2025,CREDIT,VIREMENT RECU,,300.00
|
||||||
|
30/01/2025,DEBIT,FRAIS MENSUELS,,-6.95
|
||||||
|
|
|
@ -28,7 +28,17 @@ export type CsvFixtureName =
|
||||||
| "no-header"
|
| "no-header"
|
||||||
| "all-positive"
|
| "all-positive"
|
||||||
| "desjardins-quoted"
|
| "desjardins-quoted"
|
||||||
| "absolute-indicator";
|
| "absolute-indicator"
|
||||||
|
// Bank-signature cases (#330). Each one is the DOCUMENTED header layout of a
|
||||||
|
// bank, invented row by row like the rest of the corpus. Three of them are
|
||||||
|
// shapes the generic dictionary reads WRONG — Tangerine's `Transaction` type
|
||||||
|
// column taken for the description, RBC's `CAD$` paired with the near-empty
|
||||||
|
// `Cheque Number` as a debit/credit pair — which is what the signatures exist
|
||||||
|
// for.
|
||||||
|
| "bank-desjardins"
|
||||||
|
| "bank-rbc"
|
||||||
|
| "bank-bnc"
|
||||||
|
| "bank-tangerine";
|
||||||
|
|
||||||
/** Every case in the corpus, in the order the issue lists them. */
|
/** Every case in the corpus, in the order the issue lists them. */
|
||||||
export const CSV_FIXTURE_NAMES: readonly CsvFixtureName[] = [
|
export const CSV_FIXTURE_NAMES: readonly CsvFixtureName[] = [
|
||||||
|
|
@ -43,6 +53,10 @@ export const CSV_FIXTURE_NAMES: readonly CsvFixtureName[] = [
|
||||||
"all-positive",
|
"all-positive",
|
||||||
"desjardins-quoted",
|
"desjardins-quoted",
|
||||||
"absolute-indicator",
|
"absolute-indicator",
|
||||||
|
"bank-desjardins",
|
||||||
|
"bank-rbc",
|
||||||
|
"bank-bnc",
|
||||||
|
"bank-tangerine",
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next";
|
||||||
import { AlertCircle, ArrowDownLeft, ArrowUpRight, RefreshCw } from "lucide-react";
|
import { AlertCircle, ArrowDownLeft, ArrowUpRight, RefreshCw } from "lucide-react";
|
||||||
import type { ParsedRow } from "../../shared/types";
|
import type { ParsedRow } from "../../shared/types";
|
||||||
import { isRowErrorKey, summarizeParsedRows } from "../../utils/importFormat";
|
import { isRowErrorKey, summarizeParsedRows } from "../../utils/importFormat";
|
||||||
|
import RepairPathNotice from "./RepairPathNotice";
|
||||||
|
|
||||||
/** Rows rendered in the table. The recap above it always covers the whole file. */
|
/** Rows rendered in the table. The recap above it always covers the whole file. */
|
||||||
const DISPLAYED_ROWS = 20;
|
const DISPLAYED_ROWS = 20;
|
||||||
|
|
@ -119,6 +120,16 @@ export default function FilePreviewTable({
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/*
|
||||||
|
The repair path (#330), stated where the correction is offered. A user
|
||||||
|
who flips the signs here has just learned that earlier imports of the
|
||||||
|
same source read backwards; re-importing those files is the natural
|
||||||
|
next move and it double-books every row instead of fixing them.
|
||||||
|
*/}
|
||||||
|
<div className="mt-4">
|
||||||
|
<RepairPathNotice />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="overflow-x-auto rounded-xl border border-[var(--border)]">
|
<div className="overflow-x-auto rounded-xl border border-[var(--border)]">
|
||||||
|
|
|
||||||
133
src/components/import/FormatDriftPanel.tsx
Normal file
133
src/components/import/FormatDriftPanel.tsx
Normal file
|
|
@ -0,0 +1,133 @@
|
||||||
|
import { useTranslation } from "react-i18next";
|
||||||
|
import { AlertTriangle, ArrowRight, Minus, Plus } from "lucide-react";
|
||||||
|
import type { HeaderDriftEntry } from "../../utils/bankSignatures";
|
||||||
|
import RepairPathNotice from "./RepairPathNotice";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The header row of this source changed since its last successful import
|
||||||
|
* (#330).
|
||||||
|
*
|
||||||
|
* This is the failure mode the whole chantier is most afraid of, because it is
|
||||||
|
* the one nobody sees: a bank moves a column, the stored mapping keeps reading
|
||||||
|
* position 3, and the import succeeds — with the balance in the amount column,
|
||||||
|
* for as long as it takes someone to notice. There is no signal in the totals,
|
||||||
|
* no error, no red row. The only signal is the header itself, which is why it
|
||||||
|
* is recorded and compared.
|
||||||
|
*
|
||||||
|
* The panel does not decide. It shows what moved, column by column, and offers
|
||||||
|
* the two answers that exist: read the file the way it looks now, or keep
|
||||||
|
* reading it the way it used to. Nothing here writes to the database.
|
||||||
|
*/
|
||||||
|
interface FormatDriftPanelProps {
|
||||||
|
entries: HeaderDriftEntry[];
|
||||||
|
/** False when detection could not read the new shape — nothing to adopt. */
|
||||||
|
canAdopt: boolean;
|
||||||
|
onAdopt: () => void;
|
||||||
|
onKeep: () => void;
|
||||||
|
isBusy?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function FormatDriftPanel({
|
||||||
|
entries,
|
||||||
|
canAdopt,
|
||||||
|
onAdopt,
|
||||||
|
onKeep,
|
||||||
|
isBusy = false,
|
||||||
|
}: FormatDriftPanelProps) {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
|
||||||
|
const buttonClass =
|
||||||
|
"px-3 py-2 text-sm rounded-lg transition-colors disabled:opacity-50 disabled:cursor-not-allowed";
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="p-4 rounded-xl bg-[var(--card)] border-2 border-[var(--accent)] space-y-4">
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<AlertTriangle
|
||||||
|
size={18}
|
||||||
|
className="text-[var(--accent)] shrink-0 mt-0.5"
|
||||||
|
/>
|
||||||
|
<div>
|
||||||
|
<h3 className="text-sm font-semibold text-[var(--foreground)]">
|
||||||
|
{t("import.drift.title")}
|
||||||
|
</h3>
|
||||||
|
<p className="text-sm text-[var(--muted-foreground)] mt-0.5">
|
||||||
|
{t("import.drift.intro")}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/*
|
||||||
|
Column by column, naming the columns. This is what the stored signature
|
||||||
|
being a label list rather than a hash buys: a hash can say "something
|
||||||
|
changed" and nothing more.
|
||||||
|
*/}
|
||||||
|
<ul className="space-y-1 text-sm font-mono">
|
||||||
|
{entries.map((entry) => (
|
||||||
|
<li
|
||||||
|
key={`${entry.kind}-${entry.normalizedLabel}`}
|
||||||
|
className="flex items-center gap-2 text-[var(--foreground)]"
|
||||||
|
>
|
||||||
|
{entry.kind === "moved" && (
|
||||||
|
<ArrowRight size={14} className="text-[var(--accent)] shrink-0" />
|
||||||
|
)}
|
||||||
|
{entry.kind === "added" && (
|
||||||
|
<Plus size={14} className="text-[var(--positive)] shrink-0" />
|
||||||
|
)}
|
||||||
|
{entry.kind === "removed" && (
|
||||||
|
<Minus size={14} className="text-[var(--negative)] shrink-0" />
|
||||||
|
)}
|
||||||
|
<span>
|
||||||
|
{entry.kind === "moved" &&
|
||||||
|
t("import.drift.columnMoved", {
|
||||||
|
label: entry.label,
|
||||||
|
from: entry.previousIndex,
|
||||||
|
to: entry.currentIndex,
|
||||||
|
})}
|
||||||
|
{entry.kind === "added" &&
|
||||||
|
t("import.drift.columnAdded", {
|
||||||
|
label: entry.label,
|
||||||
|
to: entry.currentIndex,
|
||||||
|
})}
|
||||||
|
{entry.kind === "removed" &&
|
||||||
|
t("import.drift.columnRemoved", {
|
||||||
|
label: entry.label,
|
||||||
|
from: entry.previousIndex,
|
||||||
|
})}
|
||||||
|
</span>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<RepairPathNotice />
|
||||||
|
|
||||||
|
<div className="flex flex-wrap items-start gap-4">
|
||||||
|
<div>
|
||||||
|
<button
|
||||||
|
onClick={onAdopt}
|
||||||
|
disabled={!canAdopt || isBusy}
|
||||||
|
className={`${buttonClass} bg-[var(--primary)] text-white hover:opacity-90`}
|
||||||
|
>
|
||||||
|
{t("import.drift.adopt")}
|
||||||
|
</button>
|
||||||
|
<p className="mt-1 text-xs text-[var(--muted-foreground)] max-w-xs">
|
||||||
|
{canAdopt
|
||||||
|
? t("import.drift.adoptHint")
|
||||||
|
: t("import.drift.adoptUnavailable")}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<button
|
||||||
|
onClick={onKeep}
|
||||||
|
disabled={isBusy}
|
||||||
|
className={`${buttonClass} border border-[var(--border)] text-[var(--foreground)] hover:bg-[var(--muted)]`}
|
||||||
|
>
|
||||||
|
{t("import.drift.keep")}
|
||||||
|
</button>
|
||||||
|
<p className="mt-1 text-xs text-[var(--muted-foreground)] max-w-xs">
|
||||||
|
{t("import.drift.keepHint")}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
44
src/components/import/RepairPathNotice.tsx
Normal file
44
src/components/import/RepairPathNotice.tsx
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
import { useTranslation } from "react-i18next";
|
||||||
|
import { LifeBuoy } from "lucide-react";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The only safe way to repair an import that was already written wrong (#330).
|
||||||
|
*
|
||||||
|
* WHY THIS IS IN THE INTERFACE AND NOT ONLY IN A RISK TABLE. Every correction
|
||||||
|
* this chantier delivers — the restored format (#324), the anchored amount
|
||||||
|
* parser (#325), the sign flip and the drift panel — changes the amounts a
|
||||||
|
* source PRODUCES. The natural next move is to re-import the file "now that it
|
||||||
|
* reads right", and that move silently doubles the ledger:
|
||||||
|
* `findDuplicates` (`transactionService.ts:164`) matches on
|
||||||
|
* `date AND description AND amount`, so a corrected row does not match the
|
||||||
|
* wrong row already stored. It is filed as new.
|
||||||
|
*
|
||||||
|
* A flipped sign is the worst shape of it: the wrong row and the corrected row
|
||||||
|
* are mirror images, they net to about zero in every report, and nothing on any
|
||||||
|
* screen looks off. The only path that leaves a correct ledger is deleting the
|
||||||
|
* faulty import from the history first — `deleteImportWithTransactions` takes
|
||||||
|
* its transactions with it — and replaying the file afterwards.
|
||||||
|
*
|
||||||
|
* Shown in both places a user can be about to do exactly that: the preview,
|
||||||
|
* next to the sign flip, and the format-drift panel.
|
||||||
|
*/
|
||||||
|
export default function RepairPathNotice() {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex items-start gap-2 p-3 rounded-lg bg-[var(--muted)] border border-[var(--border)]">
|
||||||
|
<LifeBuoy
|
||||||
|
size={14}
|
||||||
|
className="text-[var(--muted-foreground)] shrink-0 mt-0.5"
|
||||||
|
/>
|
||||||
|
<div>
|
||||||
|
<p className="text-xs font-semibold text-[var(--foreground)]">
|
||||||
|
{t("import.repairPath.title")}
|
||||||
|
</p>
|
||||||
|
<p className="text-xs text-[var(--muted-foreground)] mt-0.5">
|
||||||
|
{t("import.repairPath.body")}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -10,6 +10,10 @@ import type {
|
||||||
ImportConfigTemplate,
|
ImportConfigTemplate,
|
||||||
} from "../../shared/types";
|
} from "../../shared/types";
|
||||||
import type { DetectionScore } from "../../utils/csvAutoDetect";
|
import type { DetectionScore } from "../../utils/csvAutoDetect";
|
||||||
|
import {
|
||||||
|
bankSignatureById,
|
||||||
|
type BankSignatureId,
|
||||||
|
} from "../../utils/bankSignatures";
|
||||||
import ColumnMappingEditor from "./ColumnMappingEditor";
|
import ColumnMappingEditor from "./ColumnMappingEditor";
|
||||||
|
|
||||||
interface SourceConfigPanelProps {
|
interface SourceConfigPanelProps {
|
||||||
|
|
@ -30,6 +34,8 @@ interface SourceConfigPanelProps {
|
||||||
selectedTemplateId: number | null;
|
selectedTemplateId: number | null;
|
||||||
/** Result of the last detection run, or null when none ran on this source. */
|
/** Result of the last detection run, or null when none ran on this source. */
|
||||||
detectionScore?: DetectionScore | null;
|
detectionScore?: DetectionScore | null;
|
||||||
|
/** Bank recognised by signature during that same run, or null (#330). */
|
||||||
|
detectedBank?: BankSignatureId | null;
|
||||||
isLoading?: boolean;
|
isLoading?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -50,12 +56,23 @@ export default function SourceConfigPanel({
|
||||||
onDeleteTemplate,
|
onDeleteTemplate,
|
||||||
selectedTemplateId,
|
selectedTemplateId,
|
||||||
detectionScore,
|
detectionScore,
|
||||||
|
detectedBank,
|
||||||
isLoading,
|
isLoading,
|
||||||
}: SourceConfigPanelProps) {
|
}: SourceConfigPanelProps) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const [showSaveTemplate, setShowSaveTemplate] = useState(false);
|
const [showSaveTemplate, setShowSaveTemplate] = useState(false);
|
||||||
const [templateName, setTemplateName] = useState("");
|
const [templateName, setTemplateName] = useState("");
|
||||||
|
|
||||||
|
/*
|
||||||
|
A recognised bank replaces the generic wording, but ONLY on the confident
|
||||||
|
arm (#330). "Format Desjardins reconnu" over a file two thirds of whose
|
||||||
|
rows would not read is a claim the app cannot back: the label matched, the
|
||||||
|
data did not follow, and the uncertain wording is the one that helps.
|
||||||
|
*/
|
||||||
|
const bank = detectionScore?.confident
|
||||||
|
? bankSignatureById(detectedBank)
|
||||||
|
: null;
|
||||||
|
|
||||||
const selectClass =
|
const selectClass =
|
||||||
"w-full px-3 py-2 text-sm rounded-lg border border-[var(--border)] bg-[var(--card)] text-[var(--foreground)] focus:outline-none focus:ring-2 focus:ring-[var(--primary)]";
|
"w-full px-3 py-2 text-sm rounded-lg border border-[var(--border)] bg-[var(--card)] text-[var(--foreground)] focus:outline-none focus:ring-2 focus:ring-[var(--primary)]";
|
||||||
const inputClass = selectClass;
|
const inputClass = selectClass;
|
||||||
|
|
@ -107,15 +124,23 @@ export default function SourceConfigPanel({
|
||||||
)}
|
)}
|
||||||
<div>
|
<div>
|
||||||
<p className="text-sm text-[var(--foreground)]">
|
<p className="text-sm text-[var(--foreground)]">
|
||||||
{t(
|
{bank
|
||||||
detectionScore.confident
|
? t("import.config.detectionBank", {
|
||||||
? "import.config.detectionRecognized"
|
// A bank name is a proper noun: it is interpolated, never
|
||||||
: "import.config.detectionUncertain",
|
// translated.
|
||||||
{
|
bank: bank.label,
|
||||||
read: detectionScore.readRows,
|
read: detectionScore.readRows,
|
||||||
total: detectionScore.totalRows,
|
total: detectionScore.totalRows,
|
||||||
}
|
})
|
||||||
)}
|
: t(
|
||||||
|
detectionScore.confident
|
||||||
|
? "import.config.detectionRecognized"
|
||||||
|
: "import.config.detectionUncertain",
|
||||||
|
{
|
||||||
|
read: detectionScore.readRows,
|
||||||
|
total: detectionScore.totalRows,
|
||||||
|
}
|
||||||
|
)}
|
||||||
</p>
|
</p>
|
||||||
{!detectionScore.confident && (
|
{!detectionScore.confident && (
|
||||||
<p className="text-xs text-[var(--muted-foreground)] mt-0.5">
|
<p className="text-xs text-[var(--muted-foreground)] mt-0.5">
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,12 @@ import {
|
||||||
detectImportFormat as runAutoDetect,
|
detectImportFormat as runAutoDetect,
|
||||||
type DetectionScore,
|
type DetectionScore,
|
||||||
} from "../utils/csvAutoDetect";
|
} from "../utils/csvAutoDetect";
|
||||||
|
import {
|
||||||
|
buildHeaderSignature,
|
||||||
|
detectHeaderDrift,
|
||||||
|
type BankSignatureId,
|
||||||
|
type HeaderDriftEntry,
|
||||||
|
} from "../utils/bankSignatures";
|
||||||
import {
|
import {
|
||||||
detectAmountSeparators,
|
detectAmountSeparators,
|
||||||
flipSignFormat,
|
flipSignFormat,
|
||||||
|
|
@ -85,6 +91,24 @@ interface WizardState {
|
||||||
* ran — a source opened on its stored format, which is never re-detected.
|
* ran — a source opened on its stored format, which is never re-detected.
|
||||||
*/
|
*/
|
||||||
detectionScore: DetectionScore | null;
|
detectionScore: DetectionScore | null;
|
||||||
|
/**
|
||||||
|
* Bank whose documented layout the last detection recognised (#330). Tied to
|
||||||
|
* `detectionScore`: it is set with it and cleared with it, so a badge naming
|
||||||
|
* a bank can never outlive the detection that named it.
|
||||||
|
*/
|
||||||
|
detectedBank: BankSignatureId | null;
|
||||||
|
/**
|
||||||
|
* How the header row of the files being imported differs from the one the
|
||||||
|
* source recorded at its last successful import — null when there is nothing
|
||||||
|
* to report, which includes every source that has no stored signature.
|
||||||
|
*/
|
||||||
|
formatDrift: HeaderDriftEntry[] | null;
|
||||||
|
/**
|
||||||
|
* The configuration re-detected on the drifted file, offered by the panel as
|
||||||
|
* "adopt". Null when detection could not read the new shape either: the drift
|
||||||
|
* is still worth reporting, there is just nothing to adopt.
|
||||||
|
*/
|
||||||
|
driftConfig: SourceConfig | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
type WizardAction =
|
type WizardAction =
|
||||||
|
|
@ -107,6 +131,11 @@ type WizardAction =
|
||||||
| { type: "SET_CONFIG_TEMPLATES"; payload: ImportConfigTemplate[] }
|
| { type: "SET_CONFIG_TEMPLATES"; payload: ImportConfigTemplate[] }
|
||||||
| { type: "SET_SELECTED_TEMPLATE_ID"; payload: number | null }
|
| { type: "SET_SELECTED_TEMPLATE_ID"; payload: number | null }
|
||||||
| { type: "SET_DETECTION_SCORE"; payload: DetectionScore | null }
|
| { type: "SET_DETECTION_SCORE"; payload: DetectionScore | null }
|
||||||
|
| { type: "SET_DETECTED_BANK"; payload: BankSignatureId | null }
|
||||||
|
| {
|
||||||
|
type: "SET_FORMAT_DRIFT";
|
||||||
|
payload: { entries: HeaderDriftEntry[]; config: SourceConfig | null } | null;
|
||||||
|
}
|
||||||
| { type: "RESET" };
|
| { type: "RESET" };
|
||||||
|
|
||||||
const defaultConfig: SourceConfig = {
|
const defaultConfig: SourceConfig = {
|
||||||
|
|
@ -142,6 +171,9 @@ const initialState: WizardState = {
|
||||||
configTemplates: [],
|
configTemplates: [],
|
||||||
selectedTemplateId: null,
|
selectedTemplateId: null,
|
||||||
detectionScore: null,
|
detectionScore: null,
|
||||||
|
detectedBank: null,
|
||||||
|
formatDrift: null,
|
||||||
|
driftConfig: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
function reducer(state: WizardState, action: WizardAction): WizardState {
|
function reducer(state: WizardState, action: WizardAction): WizardState {
|
||||||
|
|
@ -159,8 +191,17 @@ function reducer(state: WizardState, action: WizardAction): WizardState {
|
||||||
case "SET_SELECTED_SOURCE":
|
case "SET_SELECTED_SOURCE":
|
||||||
// The score belongs to the source it was measured on. Carrying it over
|
// The score belongs to the source it was measured on. Carrying it over
|
||||||
// would let a source opened on its stored format inherit the confidence
|
// would let a source opened on its stored format inherit the confidence
|
||||||
// of the previous one, which is worse than showing nothing.
|
// of the previous one, which is worse than showing nothing. The bank
|
||||||
return { ...state, selectedSource: action.payload, detectionScore: null };
|
// badge and the drift report are scoped the same way, for the same
|
||||||
|
// reason — both describe one source's files.
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
selectedSource: action.payload,
|
||||||
|
detectionScore: null,
|
||||||
|
detectedBank: null,
|
||||||
|
formatDrift: null,
|
||||||
|
driftConfig: null,
|
||||||
|
};
|
||||||
case "SET_SELECTED_FILES":
|
case "SET_SELECTED_FILES":
|
||||||
return { ...state, selectedFiles: action.payload };
|
return { ...state, selectedFiles: action.payload };
|
||||||
case "SET_SOURCE_CONFIG":
|
case "SET_SOURCE_CONFIG":
|
||||||
|
|
@ -212,7 +253,24 @@ function reducer(state: WizardState, action: WizardAction): WizardState {
|
||||||
case "SET_SELECTED_TEMPLATE_ID":
|
case "SET_SELECTED_TEMPLATE_ID":
|
||||||
return { ...state, selectedTemplateId: action.payload };
|
return { ...state, selectedTemplateId: action.payload };
|
||||||
case "SET_DETECTION_SCORE":
|
case "SET_DETECTION_SCORE":
|
||||||
return { ...state, detectionScore: action.payload };
|
// Clearing the score clears the bank with it, in ONE place: the three
|
||||||
|
// sites that drop the score (a hand edit, a template, a sign flip) then
|
||||||
|
// cannot leave "Format Desjardins reconnu" standing over a format
|
||||||
|
// detection never produced. A bank is only ever set by dispatching
|
||||||
|
// `SET_DETECTED_BANK` right after a non-null score.
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
detectionScore: action.payload,
|
||||||
|
detectedBank: action.payload === null ? null : state.detectedBank,
|
||||||
|
};
|
||||||
|
case "SET_DETECTED_BANK":
|
||||||
|
return { ...state, detectedBank: action.payload };
|
||||||
|
case "SET_FORMAT_DRIFT":
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
formatDrift: action.payload?.entries ?? null,
|
||||||
|
driftConfig: action.payload?.config ?? null,
|
||||||
|
};
|
||||||
case "RESET":
|
case "RESET":
|
||||||
return {
|
return {
|
||||||
...initialState,
|
...initialState,
|
||||||
|
|
@ -346,6 +404,9 @@ export function useImportWizard() {
|
||||||
}
|
}
|
||||||
|
|
||||||
dispatch({ type: "SET_DETECTION_SCORE", payload: outcome.score });
|
dispatch({ type: "SET_DETECTION_SCORE", payload: outcome.score });
|
||||||
|
// Always dispatched, including the `null` of an unknown file: a bank left
|
||||||
|
// over from the previously detected file would name the wrong bank.
|
||||||
|
dispatch({ type: "SET_DETECTED_BANK", payload: outcome.bank });
|
||||||
return { ...base, ...outcome.config };
|
return { ...base, ...outcome.config };
|
||||||
},
|
},
|
||||||
[]
|
[]
|
||||||
|
|
@ -672,6 +733,48 @@ export function useImportWizard() {
|
||||||
type: "SET_PARSED_PREVIEW",
|
type: "SET_PARSED_PREVIEW",
|
||||||
payload: result,
|
payload: result,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Format drift (#330). Compared HERE and not at the configuration step
|
||||||
|
// because this is where the header of the files actually being imported
|
||||||
|
// is known — `previewHeaders` at the configuration step is read from the
|
||||||
|
// first file alone, under a delimiter the user may still be editing.
|
||||||
|
//
|
||||||
|
// `hasHeader` gates the whole thing: a headerless file has `Col 0`,
|
||||||
|
// `Col 1` … for headers, which is not a signature and must never be
|
||||||
|
// compared to one.
|
||||||
|
const drift = state.sourceConfig.hasHeader
|
||||||
|
? detectHeaderDrift(
|
||||||
|
state.existingSource?.header_signature,
|
||||||
|
result.headers
|
||||||
|
)
|
||||||
|
: null;
|
||||||
|
|
||||||
|
if (drift) {
|
||||||
|
// Re-detect on the drifted file so the panel has something to offer.
|
||||||
|
// This is the THIRD caller of the single detection path, and it goes
|
||||||
|
// through it rather than around it on purpose: a second detector is the
|
||||||
|
// divergence class this whole chantier removes. A file the re-detection
|
||||||
|
// cannot read leaves `null` — the drift is still reported, with nothing
|
||||||
|
// to adopt.
|
||||||
|
const reDetected = await detectFormatForFile(
|
||||||
|
state.selectedFiles[0].file_path,
|
||||||
|
state.sourceConfig
|
||||||
|
);
|
||||||
|
// That run measured the RE-DETECTED format, which is not the one the
|
||||||
|
// wizard is using — it is only what the panel offers. Leaving its score
|
||||||
|
// and its bank badge behind would put "Format reconnu — 6 of 6 rows
|
||||||
|
// read" next to the stored mapping the moment the user steps back to
|
||||||
|
// the configuration. Same rule as the sign flip: a badge describes the
|
||||||
|
// format in use or it describes nothing.
|
||||||
|
dispatch({ type: "SET_DETECTION_SCORE", payload: null });
|
||||||
|
dispatch({
|
||||||
|
type: "SET_FORMAT_DRIFT",
|
||||||
|
payload: { entries: drift, config: reDetected },
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
dispatch({ type: "SET_FORMAT_DRIFT", payload: null });
|
||||||
|
}
|
||||||
|
|
||||||
dispatch({ type: "SET_STEP", payload: "file-preview" });
|
dispatch({ type: "SET_STEP", payload: "file-preview" });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
dispatch({
|
dispatch({
|
||||||
|
|
@ -679,7 +782,13 @@ export function useImportWizard() {
|
||||||
payload: e instanceof Error ? e.message : String(e),
|
payload: e instanceof Error ? e.message : String(e),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}, [state.selectedFiles, parseFilesInternal]);
|
}, [
|
||||||
|
state.selectedFiles,
|
||||||
|
state.sourceConfig,
|
||||||
|
state.existingSource,
|
||||||
|
parseFilesInternal,
|
||||||
|
detectFormatForFile,
|
||||||
|
]);
|
||||||
|
|
||||||
// Internal helper: runs duplicate checking against parsed rows.
|
// Internal helper: runs duplicate checking against parsed rows.
|
||||||
//
|
//
|
||||||
|
|
@ -836,18 +945,31 @@ export function useImportWizard() {
|
||||||
// behind, and it goes through `formatToRow` so no field can be dropped.
|
// behind, and it goes through `formatToRow` so no field can be dropped.
|
||||||
// It has to precede the file records, which carry a `source_id` FK.
|
// It has to precede the file records, which carry a `source_id` FK.
|
||||||
const formatRow = formatToRow(config);
|
const formatRow = formatToRow(config);
|
||||||
|
|
||||||
|
// The header row this import actually read, recorded so the NEXT one can
|
||||||
|
// be compared against it (#330). It is not a format field and does not go
|
||||||
|
// through the codec: it says what the file looked like, not how to read
|
||||||
|
// it. A headerless file writes null — `previewHeaders` holds `Col 0`,
|
||||||
|
// `Col 1` … there, and storing that would make every later import look
|
||||||
|
// like drift.
|
||||||
|
const headerSignature = buildHeaderSignature(
|
||||||
|
config.hasHeader ? state.previewHeaders : null
|
||||||
|
);
|
||||||
|
|
||||||
let sourceId: number;
|
let sourceId: number;
|
||||||
if (state.existingSource) {
|
if (state.existingSource) {
|
||||||
sourceId = state.existingSource.id;
|
sourceId = state.existingSource.id;
|
||||||
await updateSource(sourceId, {
|
await updateSource(sourceId, {
|
||||||
name: config.name,
|
name: config.name,
|
||||||
...formatRow,
|
...formatRow,
|
||||||
|
header_signature: headerSignature,
|
||||||
template_id: state.selectedTemplateId,
|
template_id: state.selectedTemplateId,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
sourceId = await createSource({
|
sourceId = await createSource({
|
||||||
name: config.name,
|
name: config.name,
|
||||||
...formatRow,
|
...formatRow,
|
||||||
|
header_signature: headerSignature,
|
||||||
template_id: state.selectedTemplateId,
|
template_id: state.selectedTemplateId,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -993,6 +1115,50 @@ export function useImportWizard() {
|
||||||
dispatch({ type: "RESET" });
|
dispatch({ type: "RESET" });
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Take the format re-detected on the drifted file, and re-read the preview
|
||||||
|
* under it (#330).
|
||||||
|
*
|
||||||
|
* Same shape as the sign flip, and for the same reason: the adopted format is
|
||||||
|
* PASSED to the parse instead of being read back from `state`, which has not
|
||||||
|
* re-rendered yet. Reading the stale one would redisplay the table the user
|
||||||
|
* just chose to replace.
|
||||||
|
*
|
||||||
|
* Nothing is written here. The adopted format reaches `import_sources` at
|
||||||
|
* `executeImport` like every other configuration — an import abandoned on
|
||||||
|
* this screen leaves the stored format exactly as it was.
|
||||||
|
*/
|
||||||
|
const adoptDriftFormat = useCallback(async () => {
|
||||||
|
const adopted = state.driftConfig;
|
||||||
|
if (!adopted) return;
|
||||||
|
|
||||||
|
dispatch({ type: "SET_SOURCE_CONFIG", payload: adopted });
|
||||||
|
dispatch({ type: "SET_FORMAT_DRIFT", payload: null });
|
||||||
|
dispatch({ type: "SET_LOADING", payload: true });
|
||||||
|
dispatch({ type: "SET_ERROR", payload: null });
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await parseFilesInternal(adopted);
|
||||||
|
dispatch({ type: "SET_PARSED_PREVIEW", payload: result });
|
||||||
|
} catch (e) {
|
||||||
|
dispatch({
|
||||||
|
type: "SET_ERROR",
|
||||||
|
payload: e instanceof Error ? e.message : String(e),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}, [state.driftConfig, parseFilesInternal]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keep the stored configuration and dismiss the panel.
|
||||||
|
*
|
||||||
|
* The rows on screen were already parsed under that configuration, so there
|
||||||
|
* is nothing to re-read. The stored signature is still refreshed at import
|
||||||
|
* time: the new header IS what this import read, whichever format read it.
|
||||||
|
*/
|
||||||
|
const keepCurrentFormat = useCallback(() => {
|
||||||
|
dispatch({ type: "SET_FORMAT_DRIFT", payload: null });
|
||||||
|
}, []);
|
||||||
|
|
||||||
const autoDetectConfig = useCallback(async () => {
|
const autoDetectConfig = useCallback(async () => {
|
||||||
if (state.selectedFiles.length === 0) return;
|
if (state.selectedFiles.length === 0) return;
|
||||||
|
|
||||||
|
|
@ -1108,6 +1274,8 @@ export function useImportWizard() {
|
||||||
executeImport,
|
executeImport,
|
||||||
goToStep,
|
goToStep,
|
||||||
reset,
|
reset,
|
||||||
|
adoptDriftFormat,
|
||||||
|
keepCurrentFormat,
|
||||||
autoDetectConfig,
|
autoDetectConfig,
|
||||||
saveConfigAsTemplate,
|
saveConfigAsTemplate,
|
||||||
applyConfigTemplate,
|
applyConfigTemplate,
|
||||||
|
|
|
||||||
|
|
@ -116,7 +116,24 @@
|
||||||
"updateTemplate": "Update template",
|
"updateTemplate": "Update template",
|
||||||
"detectionRecognized": "Format recognized — {{read}} of {{total}} rows read",
|
"detectionRecognized": "Format recognized — {{read}} of {{total}} rows read",
|
||||||
"detectionUncertain": "Uncertain format — only {{read}} of {{total}} rows read",
|
"detectionUncertain": "Uncertain format — only {{read}} of {{total}} rows read",
|
||||||
"detectionUncertainHint": "Check the column mapping and the date format, then look at the preview before importing."
|
"detectionUncertainHint": "Check the column mapping and the date format, then look at the preview before importing.",
|
||||||
|
"detectionBank": "{{bank}} format recognized — {{read}} of {{total}} rows read"
|
||||||
|
},
|
||||||
|
"drift": {
|
||||||
|
"title": "This source's format has changed",
|
||||||
|
"intro": "This file's header row no longer matches the one recorded at the last successful import. The stored columns may no longer point at the right data.",
|
||||||
|
"columnMoved": "{{label}}: column {{from}} → {{to}}",
|
||||||
|
"columnAdded": "{{label}}: new column, at position {{to}}",
|
||||||
|
"columnRemoved": "{{label}}: column gone, it was at position {{from}}",
|
||||||
|
"adopt": "Adopt the re-detected format",
|
||||||
|
"adoptHint": "Reads the file the way it looks today. The new configuration is saved at import time.",
|
||||||
|
"adoptUnavailable": "Detection could not read this new shape: fix the mapping at the previous step.",
|
||||||
|
"keep": "Keep the current configuration",
|
||||||
|
"keepHint": "Keeps the stored mapping. Use this when the header changed but the columns did not move."
|
||||||
|
},
|
||||||
|
"repairPath": {
|
||||||
|
"title": "Repairing an import that was already written wrong",
|
||||||
|
"body": "Do not re-import a file you have already imported in order to correct it: duplicates are matched on the date, the description AND the amount, so a row with a corrected amount does not match the faulty row — it is added next to it. A flipped sign also produces two mirror rows that cancel each other out in every report. Delete the faulty import from the import history first, then replay the file."
|
||||||
},
|
},
|
||||||
"preview": {
|
"preview": {
|
||||||
"title": "Data Preview",
|
"title": "Data Preview",
|
||||||
|
|
|
||||||
|
|
@ -116,7 +116,24 @@
|
||||||
"updateTemplate": "Mettre à jour le modèle",
|
"updateTemplate": "Mettre à jour le modèle",
|
||||||
"detectionRecognized": "Format reconnu — {{read}} des {{total}} lignes lues",
|
"detectionRecognized": "Format reconnu — {{read}} des {{total}} lignes lues",
|
||||||
"detectionUncertain": "Format incertain — {{read}} des {{total}} lignes lues seulement",
|
"detectionUncertain": "Format incertain — {{read}} des {{total}} lignes lues seulement",
|
||||||
"detectionUncertainHint": "Vérifiez le mapping des colonnes et le format de date, puis regardez l'aperçu avant d'importer."
|
"detectionUncertainHint": "Vérifiez le mapping des colonnes et le format de date, puis regardez l'aperçu avant d'importer.",
|
||||||
|
"detectionBank": "Format {{bank}} reconnu — {{read}} des {{total}} lignes lues"
|
||||||
|
},
|
||||||
|
"drift": {
|
||||||
|
"title": "Le format de cette source a changé",
|
||||||
|
"intro": "L'en-tête de ce fichier ne correspond plus à celui du dernier import réussi. Les colonnes mémorisées ne pointent peut-être plus sur les bonnes données.",
|
||||||
|
"columnMoved": "{{label}} : colonne {{from}} → {{to}}",
|
||||||
|
"columnAdded": "{{label}} : nouvelle colonne, en position {{to}}",
|
||||||
|
"columnRemoved": "{{label}} : colonne disparue, elle était en position {{from}}",
|
||||||
|
"adopt": "Adopter le format re-détecté",
|
||||||
|
"adoptHint": "Relit le fichier tel qu'il se présente aujourd'hui. La nouvelle configuration est mémorisée à l'import.",
|
||||||
|
"adoptUnavailable": "La détection n'a pas su lire cette nouvelle forme : corrigez le mapping à l'étape précédente.",
|
||||||
|
"keep": "Conserver la configuration actuelle",
|
||||||
|
"keepHint": "Garde le mapping mémorisé. À utiliser si l'en-tête a changé sans que les colonnes bougent."
|
||||||
|
},
|
||||||
|
"repairPath": {
|
||||||
|
"title": "Réparer un import déjà écrit de travers",
|
||||||
|
"body": "Ne réimportez pas un fichier déjà importé pour le corriger : les doublons sont repérés sur la date, la description ET le montant, donc une ligne au montant corrigé ne s'apparie pas à la ligne fautive — elle s'ajoute. Une inversion de signe crée en prime deux lignes miroir qui s'annulent dans les rapports. Supprimez d'abord l'import fautif dans l'historique des imports, puis rejouez le fichier."
|
||||||
},
|
},
|
||||||
"preview": {
|
"preview": {
|
||||||
"title": "Aperçu des données",
|
"title": "Aperçu des données",
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import ImportFolderConfig from "../components/import/ImportFolderConfig";
|
||||||
import SourceList from "../components/import/SourceList";
|
import SourceList from "../components/import/SourceList";
|
||||||
import SourceConfigPanel from "../components/import/SourceConfigPanel";
|
import SourceConfigPanel from "../components/import/SourceConfigPanel";
|
||||||
import FilePreviewTable from "../components/import/FilePreviewTable";
|
import FilePreviewTable from "../components/import/FilePreviewTable";
|
||||||
|
import FormatDriftPanel from "../components/import/FormatDriftPanel";
|
||||||
import DuplicateCheckPanel from "../components/import/DuplicateCheckPanel";
|
import DuplicateCheckPanel from "../components/import/DuplicateCheckPanel";
|
||||||
import ImportConfirmation from "../components/import/ImportConfirmation";
|
import ImportConfirmation from "../components/import/ImportConfirmation";
|
||||||
import ImportProgress from "../components/import/ImportProgress";
|
import ImportProgress from "../components/import/ImportProgress";
|
||||||
|
|
@ -29,6 +30,8 @@ export default function ImportPage() {
|
||||||
executeImport,
|
executeImport,
|
||||||
goToStep,
|
goToStep,
|
||||||
reset,
|
reset,
|
||||||
|
adoptDriftFormat,
|
||||||
|
keepCurrentFormat,
|
||||||
autoDetectConfig,
|
autoDetectConfig,
|
||||||
saveConfigAsTemplate,
|
saveConfigAsTemplate,
|
||||||
applyConfigTemplate,
|
applyConfigTemplate,
|
||||||
|
|
@ -102,6 +105,7 @@ export default function ImportPage() {
|
||||||
onDeleteTemplate={deleteConfigTemplate}
|
onDeleteTemplate={deleteConfigTemplate}
|
||||||
selectedTemplateId={state.selectedTemplateId}
|
selectedTemplateId={state.selectedTemplateId}
|
||||||
detectionScore={state.detectionScore}
|
detectionScore={state.detectionScore}
|
||||||
|
detectedBank={state.detectedBank}
|
||||||
isLoading={state.isLoading}
|
isLoading={state.isLoading}
|
||||||
/>
|
/>
|
||||||
{/*
|
{/*
|
||||||
|
|
@ -122,6 +126,20 @@ export default function ImportPage() {
|
||||||
|
|
||||||
{state.step === "file-preview" && (
|
{state.step === "file-preview" && (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
|
{/*
|
||||||
|
Format drift (#330) — ABOVE the table, because it tells the user
|
||||||
|
whether the table below is worth reading. Present only when the
|
||||||
|
header row moved since the last successful import of this source.
|
||||||
|
*/}
|
||||||
|
{state.formatDrift && (
|
||||||
|
<FormatDriftPanel
|
||||||
|
entries={state.formatDrift}
|
||||||
|
canAdopt={state.driftConfig !== null}
|
||||||
|
onAdopt={adoptDriftFormat}
|
||||||
|
onKeep={keepCurrentFormat}
|
||||||
|
isBusy={state.isLoading}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
<FilePreviewTable
|
<FilePreviewTable
|
||||||
rows={state.parsedPreview}
|
rows={state.parsedPreview}
|
||||||
onFlipSigns={flipSignConvention}
|
onFlipSigns={flipSignConvention}
|
||||||
|
|
|
||||||
|
|
@ -52,8 +52,8 @@ export async function createSource(
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
const db = await getDb();
|
const db = await getDb();
|
||||||
const result = await db.execute(
|
const result = await db.execute(
|
||||||
`INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention, template_id)
|
`INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention, header_signature, template_id)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||||
ON CONFLICT(name) DO UPDATE SET
|
ON CONFLICT(name) DO UPDATE SET
|
||||||
description = excluded.description,
|
description = excluded.description,
|
||||||
date_format = excluded.date_format,
|
date_format = excluded.date_format,
|
||||||
|
|
@ -64,6 +64,7 @@ export async function createSource(
|
||||||
has_header = excluded.has_header,
|
has_header = excluded.has_header,
|
||||||
amount_mode = excluded.amount_mode,
|
amount_mode = excluded.amount_mode,
|
||||||
sign_convention = excluded.sign_convention,
|
sign_convention = excluded.sign_convention,
|
||||||
|
header_signature = excluded.header_signature,
|
||||||
template_id = excluded.template_id,
|
template_id = excluded.template_id,
|
||||||
updated_at = CURRENT_TIMESTAMP`,
|
updated_at = CURRENT_TIMESTAMP`,
|
||||||
[
|
[
|
||||||
|
|
@ -77,6 +78,10 @@ export async function createSource(
|
||||||
source.has_header,
|
source.has_header,
|
||||||
source.amount_mode,
|
source.amount_mode,
|
||||||
source.sign_convention,
|
source.sign_convention,
|
||||||
|
// Explicitly null for a headerless file: drift detection is inoperative
|
||||||
|
// there and a stale signature from an earlier shape would be worse than
|
||||||
|
// none (#330).
|
||||||
|
source.header_signature ?? null,
|
||||||
source.template_id ?? null,
|
source.template_id ?? null,
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
|
||||||
695
src/utils/bankSignatures.test.ts
Normal file
695
src/utils/bankSignatures.test.ts
Normal file
|
|
@ -0,0 +1,695 @@
|
||||||
|
// Bank signatures and format drift (#330).
|
||||||
|
//
|
||||||
|
// Two contracts are frozen here.
|
||||||
|
//
|
||||||
|
// THE SIGNATURES. Each of the four banks has a fixture carrying its documented
|
||||||
|
// header layout. The test that matters is not "the signature matches" — it is
|
||||||
|
// the pair: the same file WITH its bank header and WITHOUT it. Three of the
|
||||||
|
// four layouts are read wrong by the generic dictionary, and the counterfactual
|
||||||
|
// is what proves the signature earns its place instead of merely agreeing with
|
||||||
|
// the heuristic. The other half of that pair is the fall-back: every fixture of
|
||||||
|
// the #326 corpus must still be detected with no bank at all.
|
||||||
|
//
|
||||||
|
// THE DRIFT. `header_signature` records the labels of the last successful
|
||||||
|
// import; a later file whose header normalizes differently is drift. The
|
||||||
|
// interesting cases are the ones that must NOT fire: no stored signature, a
|
||||||
|
// headerless file, a stored value that will not parse, and a cosmetic rename
|
||||||
|
// (`Montant` -> `MONTANT ($)`) that normalizes to the same label.
|
||||||
|
|
||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
import { readFileSync } from "fs";
|
||||||
|
import { resolve } from "path";
|
||||||
|
import {
|
||||||
|
BANK_SIGNATURES,
|
||||||
|
MIN_SIGNATURE_LABELS,
|
||||||
|
bankSignatureById,
|
||||||
|
buildHeaderSignature,
|
||||||
|
detectHeaderDrift,
|
||||||
|
matchBankSignature,
|
||||||
|
parseHeaderSignature,
|
||||||
|
type BankSignatureId,
|
||||||
|
} from "./bankSignatures";
|
||||||
|
import { normalizeHeaderCell } from "./headerDictionary";
|
||||||
|
import { detectImportFormat } from "./csvAutoDetect";
|
||||||
|
import { CSV_FIXTURE_NAMES, readCsvFixture } from "../__fixtures__/csv";
|
||||||
|
import fr from "../i18n/locales/fr.json";
|
||||||
|
import en from "../i18n/locales/en.json";
|
||||||
|
|
||||||
|
/** The bank a file is recognised as, or null. Throws on a file detection refuses. */
|
||||||
|
function bankOf(content: string): BankSignatureId | null {
|
||||||
|
const outcome = detectImportFormat(content);
|
||||||
|
if (outcome.status !== "ok") {
|
||||||
|
throw new Error(`detection returned ${outcome.status}`);
|
||||||
|
}
|
||||||
|
return outcome.bank;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The configuration detection settles on. Throws on a file it refuses. */
|
||||||
|
function configOf(content: string) {
|
||||||
|
const outcome = detectImportFormat(content);
|
||||||
|
if (outcome.status !== "ok") {
|
||||||
|
throw new Error(`detection returned ${outcome.status}`);
|
||||||
|
}
|
||||||
|
return outcome.config;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The same file with its header row replaced — every data row untouched. */
|
||||||
|
function withHeader(raw: string, header: string): string {
|
||||||
|
const lines = raw.split("\n");
|
||||||
|
lines[0] = header;
|
||||||
|
return lines.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("the signature table is well formed (#330)", () => {
|
||||||
|
it("gives every bank a distinct id", () => {
|
||||||
|
const ids = BANK_SIGNATURES.map((s) => s.id);
|
||||||
|
expect(new Set(ids).size).toBe(ids.length);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("declares every label already normalized", () => {
|
||||||
|
// A label written `Montant` would never match: the header cell is
|
||||||
|
// normalized before the comparison and the table's side is not.
|
||||||
|
for (const signature of BANK_SIGNATURES) {
|
||||||
|
for (const variant of signature.variants) {
|
||||||
|
for (const label of variant.labels) {
|
||||||
|
expect(normalizeHeaderCell(label), `${signature.id}: ${label}`).toBe(
|
||||||
|
label
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps every role's label inside the fingerprint it projects", () => {
|
||||||
|
// A role naming a label absent from `labels` would resolve to a column the
|
||||||
|
// match never checked for.
|
||||||
|
for (const signature of BANK_SIGNATURES) {
|
||||||
|
for (const variant of signature.variants) {
|
||||||
|
for (const [role, label] of Object.entries(variant.roles)) {
|
||||||
|
expect(variant.labels, `${signature.id}.${role}`).toContain(label);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses a fingerprint too small to identify anyone", () => {
|
||||||
|
// `Date;Description;Montant` is the shape of half the corpus and of any
|
||||||
|
// hand-made export. A three-label variant would put a bank's name over
|
||||||
|
// files nobody can attribute to it.
|
||||||
|
for (const signature of BANK_SIGNATURES) {
|
||||||
|
for (const variant of signature.variants) {
|
||||||
|
expect(
|
||||||
|
variant.labels.length,
|
||||||
|
`${signature.id}: ${variant.labels.join(",")}`
|
||||||
|
).toBeGreaterThanOrEqual(MIN_SIGNATURE_LABELS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves a bank from its id, and nothing from an unknown one", () => {
|
||||||
|
expect(bankSignatureById("desjardins")?.label).toBe("Desjardins");
|
||||||
|
expect(bankSignatureById(null)).toBeNull();
|
||||||
|
expect(bankSignatureById(undefined)).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("each bank is recognised on its own fixture (#330)", () => {
|
||||||
|
it("reads the Desjardins layout and names it", () => {
|
||||||
|
expect(bankOf(readCsvFixture("bank-desjardins"))).toBe("desjardins");
|
||||||
|
expect(configOf(readCsvFixture("bank-desjardins"))).toEqual({
|
||||||
|
delimiter: ";",
|
||||||
|
hasHeader: true,
|
||||||
|
skipLines: 0,
|
||||||
|
dateFormat: "DD/MM/YYYY",
|
||||||
|
// Solde (column 3) is the balance the signature names, and it stays out
|
||||||
|
// of the amount candidates.
|
||||||
|
columnMapping: { date: 0, description: 1, amount: 2 },
|
||||||
|
amountMode: "single",
|
||||||
|
signConvention: "negative_expense",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the English Desjardins header too", () => {
|
||||||
|
const content =
|
||||||
|
"Date;Description;Amount;Balance\n" +
|
||||||
|
"05/01/2025;EPICERIE METRO;-84,32;915,68\n" +
|
||||||
|
"15/01/2025;DEPOT PAIE;1250,00;2165,68\n" +
|
||||||
|
"18/01/2025;HYDRO QUEBEC;-142,18;2023,50\n" +
|
||||||
|
"22/01/2025;RESTAURANT;-56,75;1966,75\n";
|
||||||
|
expect(bankOf(content)).toBe("desjardins");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads a whole-line-quoted Desjardins export", () => {
|
||||||
|
// `preprocessQuotedCSV` unwraps it into a comma-separated file; the
|
||||||
|
// signature declares the quirk and the comma, so the match survives it.
|
||||||
|
const content =
|
||||||
|
'"Date,""Description"",Montant,Solde"\n' +
|
||||||
|
'"05/01/2025,""EPICERIE METRO"",-84.32,915.68"\n' +
|
||||||
|
'"15/01/2025,""DEPOT PAIE"",1250.00,2165.68"\n' +
|
||||||
|
'"18/01/2025,""HYDRO QUEBEC"",-142.18,2023.50"\n' +
|
||||||
|
'"22/01/2025,""RESTAURANT"",-56.75,1966.75"\n';
|
||||||
|
expect(bankOf(content)).toBe("desjardins");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the RBC layout and maps CAD$ as the amount", () => {
|
||||||
|
expect(bankOf(readCsvFixture("bank-rbc"))).toBe("rbc");
|
||||||
|
expect(configOf(readCsvFixture("bank-rbc"))).toEqual({
|
||||||
|
delimiter: ",",
|
||||||
|
hasHeader: true,
|
||||||
|
skipLines: 0,
|
||||||
|
dateFormat: "DD/MM/YYYY",
|
||||||
|
columnMapping: { date: 2, description: 4, amount: 6 },
|
||||||
|
amountMode: "single",
|
||||||
|
signConvention: "negative_expense",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the Banque Nationale layout as a debit/credit pair", () => {
|
||||||
|
expect(bankOf(readCsvFixture("bank-bnc"))).toBe("bnc");
|
||||||
|
expect(configOf(readCsvFixture("bank-bnc"))).toEqual({
|
||||||
|
delimiter: ";",
|
||||||
|
hasHeader: true,
|
||||||
|
skipLines: 0,
|
||||||
|
dateFormat: "DD/MM/YYYY",
|
||||||
|
columnMapping: {
|
||||||
|
date: 0,
|
||||||
|
description: 1,
|
||||||
|
debitAmount: 3,
|
||||||
|
creditAmount: 4,
|
||||||
|
},
|
||||||
|
amountMode: "debit_credit",
|
||||||
|
signConvention: "negative_expense",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the Tangerine layout and maps Name as the description", () => {
|
||||||
|
expect(bankOf(readCsvFixture("bank-tangerine"))).toBe("tangerine");
|
||||||
|
expect(configOf(readCsvFixture("bank-tangerine"))).toEqual({
|
||||||
|
delimiter: ",",
|
||||||
|
hasHeader: true,
|
||||||
|
skipLines: 0,
|
||||||
|
dateFormat: "DD/MM/YYYY",
|
||||||
|
columnMapping: { date: 0, description: 2, amount: 4 },
|
||||||
|
amountMode: "single",
|
||||||
|
signConvention: "negative_expense",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("what the signatures actually buy (#330)", () => {
|
||||||
|
it("keeps RBC's cheque number out of the amounts", () => {
|
||||||
|
// The SAME rows, under a header no signature knows. `CAD$` and
|
||||||
|
// `Cheque Number` are sparse-complementary — one cheque number in six rows
|
||||||
|
// — so the shape scan pairs them as debit/credit and the row carrying a
|
||||||
|
// cheque number imports as -247.95 instead of -6.95. Nothing in the file
|
||||||
|
// can tell that apart from a genuine debit/credit pair; a documented layout
|
||||||
|
// can.
|
||||||
|
const anonymised = withHeader(
|
||||||
|
readCsvFixture("bank-rbc"),
|
||||||
|
"Type,Numero,Date,Cheque,Libelle,Note,Montant,Devise"
|
||||||
|
);
|
||||||
|
expect(bankOf(anonymised)).toBeNull();
|
||||||
|
expect(configOf(anonymised).amountMode).toBe("debit_credit");
|
||||||
|
expect(configOf(anonymised).columnMapping).toEqual({
|
||||||
|
date: 2,
|
||||||
|
description: 4,
|
||||||
|
debitAmount: 3,
|
||||||
|
creditAmount: 6,
|
||||||
|
});
|
||||||
|
|
||||||
|
// With the header the bank actually prints, the same rows read as one
|
||||||
|
// signed column.
|
||||||
|
expect(configOf(readCsvFixture("bank-rbc")).amountMode).toBe("single");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps Tangerine's direction column out of the description", () => {
|
||||||
|
// `Transaction` is a description keyword of the generic dictionary, and
|
||||||
|
// Tangerine's `Transaction` column holds DEBIT / CREDIT. Read as the
|
||||||
|
// description, every transaction of the file is labelled `DEBIT`.
|
||||||
|
const anonymised = withHeader(
|
||||||
|
readCsvFixture("bank-tangerine"),
|
||||||
|
"Date,Transaction,Nom,Note,Montant"
|
||||||
|
);
|
||||||
|
expect(bankOf(anonymised)).toBeNull();
|
||||||
|
expect(configOf(anonymised).columnMapping.description).toBe(1);
|
||||||
|
|
||||||
|
expect(configOf(readCsvFixture("bank-tangerine")).columnMapping.description).toBe(
|
||||||
|
2
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("an unknown file falls back to the generic dictionary (#330)", () => {
|
||||||
|
it("names no bank on any of the shape fixtures", () => {
|
||||||
|
// The corpus frozen by #326 is synthetic and belongs to no bank. A single
|
||||||
|
// false positive here is a banner claiming a bank the file is not from.
|
||||||
|
for (const name of CSV_FIXTURE_NAMES) {
|
||||||
|
if (name.startsWith("bank-")) continue;
|
||||||
|
if (name === "absolute-indicator") continue; // refused before any bank
|
||||||
|
expect(bankOf(readCsvFixture(name)), name).toBeNull();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves the pre-#330 configurations untouched", () => {
|
||||||
|
// The signatures must not have moved a single mapping of the corpus.
|
||||||
|
expect(configOf(readCsvFixture("signed-amount"))).toEqual({
|
||||||
|
delimiter: ";",
|
||||||
|
hasHeader: true,
|
||||||
|
skipLines: 0,
|
||||||
|
dateFormat: "DD/MM/YYYY",
|
||||||
|
columnMapping: { date: 0, description: 1, amount: 2 },
|
||||||
|
amountMode: "single",
|
||||||
|
signConvention: "negative_expense",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("the preconditions of a match (#330)", () => {
|
||||||
|
const rows =
|
||||||
|
"05/01/2025;EPICERIE METRO;-84,32;915,68\n" +
|
||||||
|
"15/01/2025;DEPOT PAIE;1250,00;2165,68\n" +
|
||||||
|
"18/01/2025;HYDRO QUEBEC;-142,18;2023,50\n" +
|
||||||
|
"22/01/2025;RESTAURANT;-56,75;1966,75\n";
|
||||||
|
|
||||||
|
it("refuses a delimiter the bank does not use", () => {
|
||||||
|
const tabbed = ("Date;Description;Montant;Solde\n" + rows).replace(
|
||||||
|
/;/g,
|
||||||
|
"\t"
|
||||||
|
);
|
||||||
|
expect(bankOf(tabbed)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses an export carrying more preamble than the bank prints", () => {
|
||||||
|
const content =
|
||||||
|
"RELEVE DE COMPTE\nCompte 12345\nDate;Description;Montant;Solde\n" + rows;
|
||||||
|
expect(bankOf(content)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses a whole-line-quoted file for a bank that does not quote", () => {
|
||||||
|
// Tangerine's labels, wrapped the way Desjardins wraps its exports. The
|
||||||
|
// quirk is what says this is not a Tangerine file.
|
||||||
|
const content =
|
||||||
|
'"Date,""Transaction"",Name,Memo,Amount"\n' +
|
||||||
|
'"05/01/2025,""DEBIT"",EPICERIE,,-84.32"\n' +
|
||||||
|
'"15/01/2025,""CREDIT"",PAIE,,1250.00"\n' +
|
||||||
|
'"18/01/2025,""DEBIT"",HYDRO,,-142.18"\n' +
|
||||||
|
'"22/01/2025,""DEBIT"",RESTO,,-56.75"\n';
|
||||||
|
expect(bankOf(content)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("names no bank for a headerless file, whatever its shape", () => {
|
||||||
|
// There is no label to read. This is the same boundary that leaves
|
||||||
|
// `header_signature` null on those sources.
|
||||||
|
expect(bankOf(readCsvFixture("no-header"))).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches on the whole label, never on a substring", () => {
|
||||||
|
// `matchHeaderColumn` matches substrings — that is the generic dictionary's
|
||||||
|
// rule and the reason it mis-reads these layouts. A fingerprint compared
|
||||||
|
// loosely would inherit the same problem.
|
||||||
|
expect(
|
||||||
|
matchBankSignature({
|
||||||
|
headerRow: ["Date", "Description", "Montant net", "Solde"],
|
||||||
|
delimiter: ";",
|
||||||
|
skipLines: 0,
|
||||||
|
wholeLineQuoted: false,
|
||||||
|
})
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves the roles to the columns of THIS file, not to the declared order", () => {
|
||||||
|
const match = matchBankSignature({
|
||||||
|
headerRow: ["Solde", "Montant", "Description", "Date"],
|
||||||
|
delimiter: ";",
|
||||||
|
skipLines: 0,
|
||||||
|
wholeLineQuoted: false,
|
||||||
|
});
|
||||||
|
expect(match?.signature.id).toBe("desjardins");
|
||||||
|
expect(match?.roles).toEqual({
|
||||||
|
date: 3,
|
||||||
|
description: 2,
|
||||||
|
amount: 1,
|
||||||
|
debit: null,
|
||||||
|
credit: null,
|
||||||
|
balance: 0,
|
||||||
|
roleCount: 4,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("the stored signature (#330)", () => {
|
||||||
|
it("stores the normalized labels, in order", () => {
|
||||||
|
expect(buildHeaderSignature(["Date", "Description", "Montant", "Solde"])).toBe(
|
||||||
|
'["date","description","montant","solde"]'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stores nothing for a file with no header row", () => {
|
||||||
|
// The whole point of the boundary: `Col 0`, `Col 1` … is not a signature,
|
||||||
|
// and inventing one from the data would fire on every change of content.
|
||||||
|
expect(buildHeaderSignature(null)).toBeNull();
|
||||||
|
expect(buildHeaderSignature([])).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads back what it wrote", () => {
|
||||||
|
const stored = buildHeaderSignature(["Date", "Montant"]);
|
||||||
|
expect(parseHeaderSignature(stored)).toEqual(["date", "montant"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null rather than throwing on anything it did not write", () => {
|
||||||
|
// A hash left by an older build, a truncated row, hand-edited SQL: drift
|
||||||
|
// detection switches off, an import never blows up.
|
||||||
|
expect(parseHeaderSignature(null)).toBeNull();
|
||||||
|
expect(parseHeaderSignature("")).toBeNull();
|
||||||
|
expect(parseHeaderSignature("d41d8cd98f00b204")).toBeNull();
|
||||||
|
expect(parseHeaderSignature("{}")).toBeNull();
|
||||||
|
expect(parseHeaderSignature("[1,2,3]")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("drift detection stays silent when it should (#330)", () => {
|
||||||
|
const stored = buildHeaderSignature(["Date", "Description", "Montant"]);
|
||||||
|
|
||||||
|
it("says nothing about a source that never recorded a signature", () => {
|
||||||
|
expect(detectHeaderDrift(null, ["Date", "Solde"])).toBeNull();
|
||||||
|
expect(detectHeaderDrift(undefined, ["Date", "Solde"])).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("says nothing about a headerless file", () => {
|
||||||
|
expect(detectHeaderDrift(stored, null)).toBeNull();
|
||||||
|
expect(detectHeaderDrift(stored, [])).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("says nothing about an identical header", () => {
|
||||||
|
expect(
|
||||||
|
detectHeaderDrift(stored, ["Date", "Description", "Montant"])
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("says nothing about a cosmetic rename", () => {
|
||||||
|
// Accents, case, punctuation and spacing are stripped before the
|
||||||
|
// comparison: none of them changes which column holds what.
|
||||||
|
expect(
|
||||||
|
detectHeaderDrift(stored, ["DATE", "Description ", "MONTANT ($)"])
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("says nothing when the stored value cannot be read", () => {
|
||||||
|
expect(detectHeaderDrift("not json", ["Date"])).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("drift detection names the columns that moved (#330)", () => {
|
||||||
|
it("reports a column that changed position", () => {
|
||||||
|
const stored = buildHeaderSignature([
|
||||||
|
"Date",
|
||||||
|
"Description",
|
||||||
|
"Montant",
|
||||||
|
"Solde",
|
||||||
|
]);
|
||||||
|
expect(
|
||||||
|
detectHeaderDrift(stored, ["Date", "Description", "Solde", "Montant"])
|
||||||
|
).toEqual([
|
||||||
|
{
|
||||||
|
kind: "moved",
|
||||||
|
normalizedLabel: "solde",
|
||||||
|
label: "Solde",
|
||||||
|
previousIndex: 3,
|
||||||
|
currentIndex: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
kind: "moved",
|
||||||
|
normalizedLabel: "montant",
|
||||||
|
label: "Montant",
|
||||||
|
previousIndex: 2,
|
||||||
|
currentIndex: 3,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a new column and the shift it causes", () => {
|
||||||
|
const stored = buildHeaderSignature(["Date", "Description", "Montant"]);
|
||||||
|
expect(
|
||||||
|
detectHeaderDrift(stored, ["Date", "Devise", "Description", "Montant"])
|
||||||
|
).toEqual([
|
||||||
|
{
|
||||||
|
kind: "added",
|
||||||
|
normalizedLabel: "devise",
|
||||||
|
label: "Devise",
|
||||||
|
previousIndex: null,
|
||||||
|
currentIndex: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
kind: "moved",
|
||||||
|
normalizedLabel: "description",
|
||||||
|
label: "Description",
|
||||||
|
previousIndex: 1,
|
||||||
|
currentIndex: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
kind: "moved",
|
||||||
|
normalizedLabel: "montant",
|
||||||
|
label: "Montant",
|
||||||
|
previousIndex: 2,
|
||||||
|
currentIndex: 3,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a column that disappeared, under its stored label", () => {
|
||||||
|
// The raw spelling of a removed column is recorded nowhere — only the
|
||||||
|
// normalized label survives, and that is what is shown.
|
||||||
|
const stored = buildHeaderSignature(["Date", "Description", "Solde"]);
|
||||||
|
expect(detectHeaderDrift(stored, ["Date", "Description"])).toEqual([
|
||||||
|
{
|
||||||
|
kind: "removed",
|
||||||
|
normalizedLabel: "solde",
|
||||||
|
label: "solde",
|
||||||
|
previousIndex: 2,
|
||||||
|
currentIndex: null,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("survives a bank swapping its amount column for a debit/credit pair", () => {
|
||||||
|
const stored = buildHeaderSignature([
|
||||||
|
"Date",
|
||||||
|
"Description",
|
||||||
|
"Montant",
|
||||||
|
"Solde",
|
||||||
|
]);
|
||||||
|
const drift = detectHeaderDrift(stored, [
|
||||||
|
"Date",
|
||||||
|
"Description",
|
||||||
|
"Débit",
|
||||||
|
"Crédit",
|
||||||
|
"Solde",
|
||||||
|
])!;
|
||||||
|
expect(drift.map((e) => [e.kind, e.normalizedLabel])).toEqual([
|
||||||
|
["added", "debit"],
|
||||||
|
["added", "credit"],
|
||||||
|
["moved", "solde"],
|
||||||
|
["removed", "montant"],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Static guards. The wiring lives in a React hook and two components, and the
|
||||||
|
// repository has no jsdom — same technique as the guards of #324 to #329.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const WIZARD_SRC = readFileSync(
|
||||||
|
resolve(import.meta.dirname, "..", "hooks", "useImportWizard.ts"),
|
||||||
|
"utf-8"
|
||||||
|
);
|
||||||
|
const PAGE_SRC = readFileSync(
|
||||||
|
resolve(import.meta.dirname, "..", "pages", "ImportPage.tsx"),
|
||||||
|
"utf-8"
|
||||||
|
);
|
||||||
|
const componentSrc = (name: string) =>
|
||||||
|
readFileSync(
|
||||||
|
resolve(import.meta.dirname, "..", "components", "import", name),
|
||||||
|
"utf-8"
|
||||||
|
);
|
||||||
|
|
||||||
|
describe("the signature is written at every successful import (#330)", () => {
|
||||||
|
it("builds it from the header row the import actually read", () => {
|
||||||
|
expect(WIZARD_SRC).toContain("buildHeaderSignature(");
|
||||||
|
expect(WIZARD_SRC).toContain(
|
||||||
|
"config.hasHeader ? state.previewHeaders : null"
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes it on both arms of the single write point", () => {
|
||||||
|
const body = WIZARD_SRC.slice(
|
||||||
|
WIZARD_SRC.indexOf("const executeImport ="),
|
||||||
|
WIZARD_SRC.indexOf("const goToStep =")
|
||||||
|
);
|
||||||
|
expect(body.match(/header_signature: headerSignature/g)).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps it out of the format codec", () => {
|
||||||
|
// It is drift metadata, not one of the eight fields that decide how a row
|
||||||
|
// is read. `formatToRow` naming it would put it back in the format.
|
||||||
|
const CODEC = readFileSync(
|
||||||
|
resolve(import.meta.dirname, "importFormat.ts"),
|
||||||
|
"utf-8"
|
||||||
|
);
|
||||||
|
expect(CODEC).not.toContain("header_signature");
|
||||||
|
expect(CODEC).not.toContain("headerSignature");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("drift is computed on the way to the preview (#330)", () => {
|
||||||
|
const body = () =>
|
||||||
|
WIZARD_SRC.slice(
|
||||||
|
WIZARD_SRC.indexOf("const parseAndPreview ="),
|
||||||
|
WIZARD_SRC.indexOf("const checkDuplicatesInternal =")
|
||||||
|
);
|
||||||
|
|
||||||
|
it("compares the stored signature to the headers just parsed", () => {
|
||||||
|
expect(body()).toContain("detectHeaderDrift(");
|
||||||
|
expect(body()).toContain("state.existingSource?.header_signature");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not compare anything on a headerless file", () => {
|
||||||
|
expect(body()).toContain("state.sourceConfig.hasHeader");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still reaches the preview step", () => {
|
||||||
|
expect(body()).toContain(
|
||||||
|
'dispatch({ type: "SET_STEP", payload: "file-preview" })'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops the score the re-detection produced, which describes another format", () => {
|
||||||
|
// The re-detection measures the format the PANEL offers, not the one in
|
||||||
|
// use. Keeping its score would show "Format reconnu — 6 of 6 rows read"
|
||||||
|
// beside the stored mapping as soon as the user steps back.
|
||||||
|
expect(body()).toContain(
|
||||||
|
'dispatch({ type: "SET_DETECTION_SCORE", payload: null })'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("the drift panel offers two outcomes and writes neither (#330)", () => {
|
||||||
|
const PANEL = componentSrc("FormatDriftPanel.tsx");
|
||||||
|
|
||||||
|
it("is rendered on the preview step, above the table", () => {
|
||||||
|
expect(PAGE_SRC).toContain("{state.formatDrift && (");
|
||||||
|
expect(PAGE_SRC).toContain("<FormatDriftPanel");
|
||||||
|
expect(PAGE_SRC).toContain("onAdopt={adoptDriftFormat}");
|
||||||
|
expect(PAGE_SRC).toContain("onKeep={keepCurrentFormat}");
|
||||||
|
expect(PAGE_SRC.indexOf("<FormatDriftPanel")).toBeLessThan(
|
||||||
|
PAGE_SRC.indexOf("<FilePreviewTable")
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-parses under the adopted format instead of re-reading state", () => {
|
||||||
|
// `state` has not re-rendered when the parse starts, exactly as for the
|
||||||
|
// sign flip: reading it back would redisplay the table just replaced.
|
||||||
|
const body = WIZARD_SRC.slice(
|
||||||
|
WIZARD_SRC.indexOf("const adoptDriftFormat ="),
|
||||||
|
WIZARD_SRC.indexOf("const keepCurrentFormat =")
|
||||||
|
);
|
||||||
|
expect(body).toContain("parseFilesInternal(adopted)");
|
||||||
|
expect(body).not.toContain("createSource(");
|
||||||
|
expect(body).not.toContain("updateSource(");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders the three kinds of change through i18n", () => {
|
||||||
|
for (const key of ["columnMoved", "columnAdded", "columnRemoved"]) {
|
||||||
|
expect(PANEL, key).toContain(`import.drift.${key}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("the repair path is stated in the interface (#330)", () => {
|
||||||
|
const NOTICE = componentSrc("RepairPathNotice.tsx");
|
||||||
|
|
||||||
|
it("names the history deletion rather than a vague warning", () => {
|
||||||
|
expect(NOTICE).toContain("import.repairPath.title");
|
||||||
|
expect(NOTICE).toContain("import.repairPath.body");
|
||||||
|
for (const locale of [fr, en]) {
|
||||||
|
expect(locale.import.repairPath.body.length).toBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
// The two facts a user needs: duplicates are matched on the amount, and the
|
||||||
|
// faulty import has to go first.
|
||||||
|
expect(fr.import.repairPath.body).toContain("historique");
|
||||||
|
expect(en.import.repairPath.body).toContain("history");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("appears in the preview and in the drift panel, not in one of the two", () => {
|
||||||
|
expect(componentSrc("FilePreviewTable.tsx")).toContain("<RepairPathNotice />");
|
||||||
|
expect(componentSrc("FormatDriftPanel.tsx")).toContain("<RepairPathNotice />");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("the recognised-bank banner (#330)", () => {
|
||||||
|
const PANEL = componentSrc("SourceConfigPanel.tsx");
|
||||||
|
|
||||||
|
it("names the bank only when the rows actually read", () => {
|
||||||
|
// A bank label over a file two thirds of whose rows fail is a claim the app
|
||||||
|
// cannot back; the uncertain wording is the one that helps there.
|
||||||
|
expect(PANEL).toContain("detectionScore?.confident");
|
||||||
|
expect(PANEL).toContain("import.config.detectionBank");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("interpolates the bank name instead of translating it", () => {
|
||||||
|
// A bank is a proper noun. `Desjardins` is `Desjardins` in both languages.
|
||||||
|
for (const locale of [fr, en]) {
|
||||||
|
expect(locale.import.config.detectionBank).toContain("{{bank}}");
|
||||||
|
expect(locale.import.config.detectionBank).toContain("{{read}}");
|
||||||
|
expect(locale.import.config.detectionBank).toContain("{{total}}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clears the bank with the score, in one place", () => {
|
||||||
|
// Three sites drop the score (a hand edit, a template, a sign flip). The
|
||||||
|
// reducer clearing the bank alongside it is what keeps a fourth from
|
||||||
|
// forgetting.
|
||||||
|
const reducerCase = WIZARD_SRC.slice(
|
||||||
|
WIZARD_SRC.indexOf('case "SET_DETECTION_SCORE":'),
|
||||||
|
WIZARD_SRC.indexOf('case "SET_DETECTED_BANK":')
|
||||||
|
);
|
||||||
|
expect(reducerCase).toContain("detectedBank: action.payload === null");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("every new string exists in both languages (#330)", () => {
|
||||||
|
it("carries the drift panel", () => {
|
||||||
|
for (const key of [
|
||||||
|
"title",
|
||||||
|
"intro",
|
||||||
|
"columnMoved",
|
||||||
|
"columnAdded",
|
||||||
|
"columnRemoved",
|
||||||
|
"adopt",
|
||||||
|
"adoptHint",
|
||||||
|
"adoptUnavailable",
|
||||||
|
"keep",
|
||||||
|
"keepHint",
|
||||||
|
] as const) {
|
||||||
|
expect(fr.import.drift[key].length, `fr.${key}`).toBeGreaterThan(0);
|
||||||
|
expect(en.import.drift[key].length, `en.${key}`).toBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("interpolates the column positions in both languages", () => {
|
||||||
|
for (const locale of [fr, en]) {
|
||||||
|
expect(locale.import.drift.columnMoved).toContain("{{label}}");
|
||||||
|
expect(locale.import.drift.columnMoved).toContain("{{from}}");
|
||||||
|
expect(locale.import.drift.columnMoved).toContain("{{to}}");
|
||||||
|
expect(locale.import.drift.columnAdded).toContain("{{to}}");
|
||||||
|
expect(locale.import.drift.columnRemoved).toContain("{{from}}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("carries the repair path", () => {
|
||||||
|
for (const key of ["title", "body"] as const) {
|
||||||
|
expect(fr.import.repairPath[key].length, `fr.${key}`).toBeGreaterThan(0);
|
||||||
|
expect(en.import.repairPath[key].length, `en.${key}`).toBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
439
src/utils/bankSignatures.ts
Normal file
439
src/utils/bankSignatures.ts
Normal file
|
|
@ -0,0 +1,439 @@
|
||||||
|
/**
|
||||||
|
* Bank signatures and header-signature drift (#330).
|
||||||
|
*
|
||||||
|
* TWO THINGS LIVE HERE, and they are the same thing seen twice: a header row
|
||||||
|
* reduced to its normalized labels.
|
||||||
|
*
|
||||||
|
* - A BANK SIGNATURE is that label list written down in advance, per bank, so
|
||||||
|
* a known export is recognised by name instead of being guessed at.
|
||||||
|
* - A STORED SIGNATURE is that same label list recorded on the source at the
|
||||||
|
* last successful import (`import_sources.header_signature`), so the next
|
||||||
|
* file from the same bank can be compared against it column by column.
|
||||||
|
*
|
||||||
|
* WHY SIGNATURES RUN BEFORE THE GENERIC DICTIONARY. `headerDictionary.ts`
|
||||||
|
* matches keywords as SUBSTRINGS, one role at a time, and has no notion of a
|
||||||
|
* best match — the first column containing the keyword wins. That is the right
|
||||||
|
* rule for an unknown file and the wrong one for a known layout:
|
||||||
|
* - Tangerine writes `Date,Transaction,Name,Memo,Amount`. The dictionary reads
|
||||||
|
* `Transaction` as the description (it is a description keyword) and maps
|
||||||
|
* the transaction TYPE column — `DEBIT`/`CREDIT` — as the label of every
|
||||||
|
* row. The signature names `Name`.
|
||||||
|
* - RBC writes its amount column `CAD$`, which contains neither `montant` nor
|
||||||
|
* `amount`, so the dictionary finds no amount column at all and the shape
|
||||||
|
* heuristic pairs `CAD$` with the (usually empty) `USD$` as a debit/credit
|
||||||
|
* pair. The signature names `CAD$` as a single signed amount.
|
||||||
|
*
|
||||||
|
* WHY A FAILING SIGNATURE CANNOT BREAK ANYTHING. These signatures are written
|
||||||
|
* from documented export layouts, WITHOUT real statements — the app is
|
||||||
|
* privacy-first and no real statement lands in this repository. So a signature
|
||||||
|
* is only ever allowed to do what the generic dictionary already does: hand
|
||||||
|
* `detectImportFormat` a set of PREFERENCES. Every one of them is dropped the
|
||||||
|
* moment the data contradicts it (a date column nothing parses as a date, an
|
||||||
|
* amount column the shape scan never proposed). A file no signature recognises
|
||||||
|
* falls through to the dictionary, unchanged. Failing degrades; it never breaks.
|
||||||
|
*
|
||||||
|
* WHY THE STORED SIGNATURE IS A LABEL LIST AND NOT A HASH. The drift panel has
|
||||||
|
* to name the columns that moved ("Montant : 3 → 4"), which a hash forbids.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
normalizeHeaderCell,
|
||||||
|
type LexicalHeaderMap,
|
||||||
|
} from "./headerDictionary";
|
||||||
|
|
||||||
|
/** Every bank the table knows, and the i18n-free proper noun it is shown as. */
|
||||||
|
export type BankSignatureId = "desjardins" | "rbc" | "bnc" | "tangerine";
|
||||||
|
|
||||||
|
/** The transaction roles a signature can name — `LexicalHeaderMap` minus its count. */
|
||||||
|
export type HeaderRole = keyof Omit<LexicalHeaderMap, "roleCount">;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One documented layout of one bank.
|
||||||
|
*
|
||||||
|
* `labels` is the FINGERPRINT: every one of them must appear in the header row,
|
||||||
|
* compared on the whole normalized cell (not as a substring). `roles` is a
|
||||||
|
* projection of that fingerprint onto the columns detection cares about —
|
||||||
|
* labels carrying no role (`Catégorie`, `Memo`, `N° de chèque`) stay in
|
||||||
|
* `labels` precisely because they are what makes the fingerprint distinctive.
|
||||||
|
*/
|
||||||
|
export interface BankHeaderVariant {
|
||||||
|
readonly labels: readonly string[];
|
||||||
|
readonly roles: Readonly<Partial<Record<HeaderRole, string>>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BankSignature {
|
||||||
|
readonly id: BankSignatureId;
|
||||||
|
/** Proper noun. Displayed as-is in every language, never translated. */
|
||||||
|
readonly label: string;
|
||||||
|
/** Delimiters this bank's exports use. A precondition of the match, never an override. */
|
||||||
|
readonly delimiters: readonly string[];
|
||||||
|
/** Preamble lines this export prints before its header row. */
|
||||||
|
readonly maxPreambleLines: number;
|
||||||
|
/** The export wraps whole lines in quotes — see `preprocessQuotedCSV`. */
|
||||||
|
readonly wholeLineQuoted: boolean;
|
||||||
|
readonly variants: readonly BankHeaderVariant[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Labels a variant must carry before it is allowed to claim a file.
|
||||||
|
*
|
||||||
|
* Three is not enough: `Date;Description;Montant` is the shape of half the
|
||||||
|
* synthetic corpus and of any hand-made export, so a three-label Desjardins
|
||||||
|
* variant would put "Format Desjardins reconnu" over files no one can attribute
|
||||||
|
* to Desjardins. Announcing the wrong bank is worse than announcing none — the
|
||||||
|
* generic path reads those files correctly already.
|
||||||
|
*/
|
||||||
|
export const MIN_SIGNATURE_LABELS = 4;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The table. Written from the banks' documented export layouts; none of it has
|
||||||
|
* been verified against a real statement (see the file header). Adding a bank
|
||||||
|
* is adding an entry — there is no code to write.
|
||||||
|
*/
|
||||||
|
export const BANK_SIGNATURES: readonly BankSignature[] = [
|
||||||
|
{
|
||||||
|
// AccèsD exports Date / Description / Montant / Solde, semicolon-separated,
|
||||||
|
// with comma decimals, in French or in English. Some credit-card exports
|
||||||
|
// wrap every line in quotes (handled upstream by `preprocessQuotedCSV`,
|
||||||
|
// which leaves a comma-separated file behind) and some carry NO header row
|
||||||
|
// at all — those cannot be signed, by construction.
|
||||||
|
id: "desjardins",
|
||||||
|
label: "Desjardins",
|
||||||
|
delimiters: [";", ","],
|
||||||
|
maxPreambleLines: 0,
|
||||||
|
wholeLineQuoted: true,
|
||||||
|
variants: [
|
||||||
|
{
|
||||||
|
labels: ["date", "description", "montant", "solde"],
|
||||||
|
roles: {
|
||||||
|
date: "date",
|
||||||
|
description: "description",
|
||||||
|
amount: "montant",
|
||||||
|
balance: "solde",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
labels: ["date", "description", "amount", "balance"],
|
||||||
|
roles: {
|
||||||
|
date: "date",
|
||||||
|
description: "description",
|
||||||
|
amount: "amount",
|
||||||
|
balance: "balance",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// RBC exports one comma-separated file for every account, amounts in a
|
||||||
|
// currency-named column. `CAD$` normalizes to `cad`, which no generic
|
||||||
|
// amount keyword matches — this variant is the only thing that maps it.
|
||||||
|
id: "rbc",
|
||||||
|
label: "RBC",
|
||||||
|
delimiters: [","],
|
||||||
|
maxPreambleLines: 0,
|
||||||
|
wholeLineQuoted: false,
|
||||||
|
variants: [
|
||||||
|
{
|
||||||
|
labels: [
|
||||||
|
"accounttype",
|
||||||
|
"accountnumber",
|
||||||
|
"transactiondate",
|
||||||
|
"chequenumber",
|
||||||
|
"description1",
|
||||||
|
"cad",
|
||||||
|
],
|
||||||
|
roles: {
|
||||||
|
date: "transactiondate",
|
||||||
|
description: "description1",
|
||||||
|
amount: "cad",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Banque Nationale exports a debit/credit pair next to a category column,
|
||||||
|
// semicolon-separated in French.
|
||||||
|
id: "bnc",
|
||||||
|
label: "Banque Nationale",
|
||||||
|
delimiters: [";", ","],
|
||||||
|
maxPreambleLines: 0,
|
||||||
|
wholeLineQuoted: false,
|
||||||
|
variants: [
|
||||||
|
{
|
||||||
|
labels: ["date", "description", "categorie", "debit", "credit", "solde"],
|
||||||
|
roles: {
|
||||||
|
date: "date",
|
||||||
|
description: "description",
|
||||||
|
debit: "debit",
|
||||||
|
credit: "credit",
|
||||||
|
balance: "solde",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
labels: ["date", "description", "category", "debit", "credit", "balance"],
|
||||||
|
roles: {
|
||||||
|
date: "date",
|
||||||
|
description: "description",
|
||||||
|
debit: "debit",
|
||||||
|
credit: "credit",
|
||||||
|
balance: "balance",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Tangerine exports Date,Transaction,Name,Memo,Amount — comma-separated,
|
||||||
|
// signed amounts. `Transaction` holds the direction word, NOT the label of
|
||||||
|
// the operation; reading it as the description makes every row read
|
||||||
|
// `DEBIT` or `CREDIT`.
|
||||||
|
id: "tangerine",
|
||||||
|
label: "Tangerine",
|
||||||
|
delimiters: [","],
|
||||||
|
maxPreambleLines: 0,
|
||||||
|
wholeLineQuoted: false,
|
||||||
|
variants: [
|
||||||
|
{
|
||||||
|
labels: ["date", "transaction", "name", "memo", "amount"],
|
||||||
|
roles: { date: "date", description: "name", amount: "amount" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
/** What the caller needs to know before a signature may claim a file. */
|
||||||
|
export interface BankSignatureInput {
|
||||||
|
/** The header row as parsed, raw cells. */
|
||||||
|
readonly headerRow: readonly string[];
|
||||||
|
/** Delimiter detection settled on. */
|
||||||
|
readonly delimiter: string;
|
||||||
|
/** Preamble lines detection had to skip. */
|
||||||
|
readonly skipLines: number;
|
||||||
|
/** True when `preprocessQuotedCSV` actually unwrapped the file. */
|
||||||
|
readonly wholeLineQuoted: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BankSignatureMatch {
|
||||||
|
readonly signature: BankSignature;
|
||||||
|
readonly variant: BankHeaderVariant;
|
||||||
|
/** The variant's roles, resolved to column indices of THIS file. */
|
||||||
|
readonly roles: LexicalHeaderMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* First-occurrence index of every normalized label of a header row.
|
||||||
|
*
|
||||||
|
* First occurrence, not last: a file repeating a label (`Montant;…;Montant`)
|
||||||
|
* is degenerate either way, and taking the leftmost keeps the result stable
|
||||||
|
* between the matcher and the drift diff, which both read this map.
|
||||||
|
*/
|
||||||
|
function labelIndex(headerRow: readonly string[]): Map<string, number> {
|
||||||
|
const index = new Map<string, number>();
|
||||||
|
headerRow.forEach((cell, i) => {
|
||||||
|
const label = normalizeHeaderCell(cell);
|
||||||
|
if (label && !index.has(label)) index.set(label, i);
|
||||||
|
});
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rolesOf(
|
||||||
|
variant: BankHeaderVariant,
|
||||||
|
index: ReadonlyMap<string, number>
|
||||||
|
): LexicalHeaderMap {
|
||||||
|
const columnOf = (role: HeaderRole): number | null => {
|
||||||
|
const label = variant.roles[role];
|
||||||
|
if (label === undefined) return null;
|
||||||
|
return index.get(label) ?? null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const date = columnOf("date");
|
||||||
|
const description = columnOf("description");
|
||||||
|
const amount = columnOf("amount");
|
||||||
|
const debit = columnOf("debit");
|
||||||
|
const credit = columnOf("credit");
|
||||||
|
const balance = columnOf("balance");
|
||||||
|
|
||||||
|
return {
|
||||||
|
date,
|
||||||
|
description,
|
||||||
|
amount,
|
||||||
|
debit,
|
||||||
|
credit,
|
||||||
|
balance,
|
||||||
|
roleCount: [date, description, amount, debit, credit, balance].filter(
|
||||||
|
(c) => c !== null
|
||||||
|
).length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the bank whose documented layout this header row is.
|
||||||
|
*
|
||||||
|
* Three preconditions guard every match, each one a way the file says it is not
|
||||||
|
* this export: a delimiter the bank does not use, more preamble than the bank
|
||||||
|
* prints, or a whole-line-quoted file claiming to be a bank that does not quote.
|
||||||
|
* They can only ever turn a match into a fall-back to the generic dictionary.
|
||||||
|
*
|
||||||
|
* Returns null for an unknown file — which is the common case and not an error.
|
||||||
|
*/
|
||||||
|
export function matchBankSignature(
|
||||||
|
input: BankSignatureInput
|
||||||
|
): BankSignatureMatch | null {
|
||||||
|
const index = labelIndex(input.headerRow);
|
||||||
|
if (index.size === 0) return null;
|
||||||
|
|
||||||
|
for (const signature of BANK_SIGNATURES) {
|
||||||
|
if (!signature.delimiters.includes(input.delimiter)) continue;
|
||||||
|
if (input.skipLines > signature.maxPreambleLines) continue;
|
||||||
|
if (input.wholeLineQuoted && !signature.wholeLineQuoted) continue;
|
||||||
|
|
||||||
|
for (const variant of signature.variants) {
|
||||||
|
if (!variant.labels.every((label) => index.has(label))) continue;
|
||||||
|
return { signature, variant, roles: rolesOf(variant, index) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The bank behind an id, for the banner. Null for an id the table lost. */
|
||||||
|
export function bankSignatureById(
|
||||||
|
id: BankSignatureId | null | undefined
|
||||||
|
): BankSignature | null {
|
||||||
|
if (!id) return null;
|
||||||
|
return BANK_SIGNATURES.find((s) => s.id === id) ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// The stored signature and the drift it detects
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The header row of a successful import, as `import_sources.header_signature`
|
||||||
|
* stores it: a JSON array of normalized labels.
|
||||||
|
*
|
||||||
|
* Returns null for a file with NO header row — and that null is written to the
|
||||||
|
* column as-is. A headerless source has no signature to compare against and
|
||||||
|
* drift detection is inoperative on it, deliberately: a signature invented from
|
||||||
|
* the data would fire on every change of content, which is every import.
|
||||||
|
*/
|
||||||
|
export function buildHeaderSignature(
|
||||||
|
headers: readonly string[] | null | undefined
|
||||||
|
): string | null {
|
||||||
|
if (!headers || headers.length === 0) return null;
|
||||||
|
return JSON.stringify(headers.map(normalizeHeaderCell));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read back a stored signature. Anything that is not an array of strings — a
|
||||||
|
* hash written by an older build, a truncated row, hand-edited SQL — comes back
|
||||||
|
* null, which switches drift detection off rather than throwing inside an
|
||||||
|
* import. Same rule as everywhere else here: degrade, never break.
|
||||||
|
*/
|
||||||
|
export function parseHeaderSignature(
|
||||||
|
stored: string | null | undefined
|
||||||
|
): string[] | null {
|
||||||
|
if (!stored) return null;
|
||||||
|
try {
|
||||||
|
const parsed: unknown = JSON.parse(stored);
|
||||||
|
if (!Array.isArray(parsed)) return null;
|
||||||
|
if (!parsed.every((label) => typeof label === "string")) return null;
|
||||||
|
return parsed as string[];
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type HeaderDriftKind = "moved" | "added" | "removed";
|
||||||
|
|
||||||
|
export interface HeaderDriftEntry {
|
||||||
|
readonly kind: HeaderDriftKind;
|
||||||
|
/** The normalized label — the key the two signatures are compared on. */
|
||||||
|
readonly normalizedLabel: string;
|
||||||
|
/**
|
||||||
|
* What to show. The file's own spelling for a column that still exists
|
||||||
|
* (`Montant`), the normalized label for one that disappeared — the raw
|
||||||
|
* spelling of a removed column is not recorded anywhere.
|
||||||
|
*/
|
||||||
|
readonly label: string;
|
||||||
|
/** Column index at the last successful import; null for a new column. */
|
||||||
|
readonly previousIndex: number | null;
|
||||||
|
/** Column index in the file being imported; null for a column that vanished. */
|
||||||
|
readonly currentIndex: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compare the header row of the file being imported against the one the last
|
||||||
|
* successful import recorded.
|
||||||
|
*
|
||||||
|
* Returns null when there is nothing to say — no stored signature, a headerless
|
||||||
|
* file, an unreadable stored value, or a header that normalizes to exactly the
|
||||||
|
* same labels in the same order. `Montant` becoming `MONTANT ($)` is NOT drift:
|
||||||
|
* it normalizes identically and changes nothing about how the file is read.
|
||||||
|
*
|
||||||
|
* A non-null result is always a non-empty list, so the caller can treat it as
|
||||||
|
* "show the panel".
|
||||||
|
*/
|
||||||
|
export function detectHeaderDrift(
|
||||||
|
stored: string | null | undefined,
|
||||||
|
currentHeaders: readonly string[] | null | undefined
|
||||||
|
): HeaderDriftEntry[] | null {
|
||||||
|
const previous = parseHeaderSignature(stored);
|
||||||
|
if (!previous || previous.length === 0) return null;
|
||||||
|
if (!currentHeaders || currentHeaders.length === 0) return null;
|
||||||
|
|
||||||
|
const current = currentHeaders.map(normalizeHeaderCell);
|
||||||
|
if (
|
||||||
|
previous.length === current.length &&
|
||||||
|
previous.every((label, i) => label === current[i])
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const previousIndex = new Map<string, number>();
|
||||||
|
previous.forEach((label, i) => {
|
||||||
|
if (label && !previousIndex.has(label)) previousIndex.set(label, i);
|
||||||
|
});
|
||||||
|
const currentIndex = labelIndex(currentHeaders);
|
||||||
|
|
||||||
|
const entries: HeaderDriftEntry[] = [];
|
||||||
|
|
||||||
|
// Walk the file being imported first, so the panel reads in its column order.
|
||||||
|
current.forEach((label, i) => {
|
||||||
|
if (!label) return;
|
||||||
|
if (currentIndex.get(label) !== i) return; // duplicate label, already handled
|
||||||
|
const before = previousIndex.get(label);
|
||||||
|
if (before === undefined) {
|
||||||
|
entries.push({
|
||||||
|
kind: "added",
|
||||||
|
normalizedLabel: label,
|
||||||
|
label: (currentHeaders[i] ?? "").trim() || label,
|
||||||
|
previousIndex: null,
|
||||||
|
currentIndex: i,
|
||||||
|
});
|
||||||
|
} else if (before !== i) {
|
||||||
|
entries.push({
|
||||||
|
kind: "moved",
|
||||||
|
normalizedLabel: label,
|
||||||
|
label: (currentHeaders[i] ?? "").trim() || label,
|
||||||
|
previousIndex: before,
|
||||||
|
currentIndex: i,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Then the columns that are simply gone, in the order they used to be in.
|
||||||
|
previous.forEach((label, i) => {
|
||||||
|
if (!label) return;
|
||||||
|
if (previousIndex.get(label) !== i) return;
|
||||||
|
if (currentIndex.has(label)) return;
|
||||||
|
entries.push({
|
||||||
|
kind: "removed",
|
||||||
|
normalizedLabel: label,
|
||||||
|
label,
|
||||||
|
previousIndex: i,
|
||||||
|
currentIndex: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return entries.length > 0 ? entries : null;
|
||||||
|
}
|
||||||
|
|
@ -260,7 +260,8 @@ const REFERENCE_AMOUNTS = [-84.32, 1250, -142.18, -56.75, 300, -6.95];
|
||||||
|
|
||||||
describe("corpus integrity (#326)", () => {
|
describe("corpus integrity (#326)", () => {
|
||||||
it("exposes every declared fixture as readable, non-empty content", () => {
|
it("exposes every declared fixture as readable, non-empty content", () => {
|
||||||
expect(CSV_FIXTURE_NAMES).toHaveLength(11);
|
// 11 shape cases (#326) + the 4 bank layouts (#330).
|
||||||
|
expect(CSV_FIXTURE_NAMES).toHaveLength(15);
|
||||||
for (const name of CSV_FIXTURE_NAMES) {
|
for (const name of CSV_FIXTURE_NAMES) {
|
||||||
expect(readCsvFixture(name).trim().length).toBeGreaterThan(0);
|
expect(readCsvFixture(name).trim().length).toBeGreaterThan(0);
|
||||||
}
|
}
|
||||||
|
|
@ -1126,10 +1127,14 @@ describe("useImportWizard — detection fires on its own (#328)", () => {
|
||||||
expect(restoreArm).not.toContain("runAutoDetect");
|
expect(restoreArm).not.toContain("runAutoDetect");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps a single detection path for the button and the automatic run", () => {
|
it("keeps a single detection path for every caller", () => {
|
||||||
// One detector, two callers: the wand button and `selectSource`.
|
// ONE detector. The count of callers is deliberate and rises when a
|
||||||
|
// feature earns it: the wand button and `selectSource` (#328), then the
|
||||||
|
// format-drift re-detection in `parseAndPreview` (#330), which goes through
|
||||||
|
// this path rather than calling `runAutoDetect` a second time — a second
|
||||||
|
// detector is the divergence class this chantier removes.
|
||||||
expect(WIZARD.match(/runAutoDetect\(/g)).toHaveLength(1);
|
expect(WIZARD.match(/runAutoDetect\(/g)).toHaveLength(1);
|
||||||
expect(WIZARD.match(/await detectFormatForFile\(/g)).toHaveLength(2);
|
expect(WIZARD.match(/await detectFormatForFile\(/g)).toHaveLength(3);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("drops the score as soon as the format is edited by hand", () => {
|
it("drops the score as soon as the format is edited by hand", () => {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,11 @@
|
||||||
import Papa from "papaparse";
|
import Papa from "papaparse";
|
||||||
import { parseDate } from "./dateParser";
|
import { parseDate } from "./dateParser";
|
||||||
import { parseFrenchAmount } from "./amountParser";
|
import { parseFrenchAmount } from "./amountParser";
|
||||||
|
import {
|
||||||
|
matchBankSignature,
|
||||||
|
type BankSignatureId,
|
||||||
|
type BankSignatureMatch,
|
||||||
|
} from "./bankSignatures";
|
||||||
import {
|
import {
|
||||||
CREDIT_INDICATOR_TOKENS,
|
CREDIT_INDICATOR_TOKENS,
|
||||||
DEBIT_INDICATOR_TOKENS,
|
DEBIT_INDICATOR_TOKENS,
|
||||||
|
|
@ -78,7 +83,19 @@ export const CONFIDENCE_THRESHOLD = 0.9;
|
||||||
* backwards" call for different messages. `detectImportFormat` separates them.
|
* backwards" call for different messages. `detectImportFormat` separates them.
|
||||||
*/
|
*/
|
||||||
export type AutoDetectOutcome =
|
export type AutoDetectOutcome =
|
||||||
| { status: "ok"; config: AutoDetectResult; score: DetectionScore }
|
| {
|
||||||
|
status: "ok";
|
||||||
|
config: AutoDetectResult;
|
||||||
|
score: DetectionScore;
|
||||||
|
/**
|
||||||
|
* The bank whose documented layout this file matched, or null when the
|
||||||
|
* generic dictionary read the header (#330). Deliberately NOT part of
|
||||||
|
* `config`: it says where the configuration came from, it is not one of
|
||||||
|
* the fields that decide how a row is read, and it is never persisted as
|
||||||
|
* format.
|
||||||
|
*/
|
||||||
|
bank: BankSignatureId | null;
|
||||||
|
}
|
||||||
| { status: "rejected"; reason: AutoDetectRejectionKey }
|
| { status: "rejected"; reason: AutoDetectRejectionKey }
|
||||||
| { status: "failed" };
|
| { status: "failed" };
|
||||||
|
|
||||||
|
|
@ -148,6 +165,13 @@ export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
|
||||||
* naming a column the data contradicts (a "Date" column nothing parses as a
|
* naming a column the data contradicts (a "Date" column nothing parses as a
|
||||||
* date, a "Solde" column that is the only amount candidate left) is dropped and
|
* date, a "Solde" column that is the only amount candidate left) is dropped and
|
||||||
* the shape heuristic decides, exactly as it did before #327.
|
* the shape heuristic decides, exactly as it did before #327.
|
||||||
|
*
|
||||||
|
* In FRONT of that lexical layer sits the bank-signature table (#330): a header
|
||||||
|
* row matching a documented export layout gets its roles from that layout and
|
||||||
|
* the generic dictionary is not consulted at all. The hints it produces are the
|
||||||
|
* same kind of preference — the file is reported as recognised, not read
|
||||||
|
* differently on trust. An unknown header falls straight through to the
|
||||||
|
* dictionary, which is what every file did before.
|
||||||
*/
|
*/
|
||||||
export function detectImportFormat(rawContent: string): AutoDetectOutcome {
|
export function detectImportFormat(rawContent: string): AutoDetectOutcome {
|
||||||
const failed: AutoDetectOutcome = { status: "failed" };
|
const failed: AutoDetectOutcome = { status: "failed" };
|
||||||
|
|
@ -193,9 +217,28 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
|
||||||
// Step 2: Detect header
|
// Step 2: Detect header
|
||||||
const hasHeader = detectHeader(effectiveData[0]);
|
const hasHeader = detectHeader(effectiveData[0]);
|
||||||
|
|
||||||
// Step 2b: Read the header labels. A headerless file yields no map at all,
|
// Step 2b: Try the known banks first (#330). A signature is claimed on the
|
||||||
// which is the explicit fall-back: every step below then runs on shape only.
|
// WHOLE normalized label, delimiter and preamble included, so an unknown file
|
||||||
const lexical = hasHeader ? matchTransactionHeaders(effectiveData[0]) : null;
|
// simply does not match and the generic dictionary reads it as before. A
|
||||||
|
// headerless file matches nothing by construction — there is no label to read.
|
||||||
|
const signature = hasHeader
|
||||||
|
? matchBankSignature({
|
||||||
|
headerRow: effectiveData[0],
|
||||||
|
delimiter,
|
||||||
|
skipLines,
|
||||||
|
// `preprocessQuotedCSV` returns its input untouched when the file is not
|
||||||
|
// whole-line-quoted, so this comparison IS the quirk.
|
||||||
|
wholeLineQuoted: content !== rawContent,
|
||||||
|
})
|
||||||
|
: null;
|
||||||
|
|
||||||
|
// Step 2c: Read the header labels. A signature that matched has already named
|
||||||
|
// the roles; otherwise the generic dictionary does. A headerless file yields
|
||||||
|
// no map at all, which is the explicit fall-back: every step below then runs
|
||||||
|
// on shape only.
|
||||||
|
const lexical = hasHeader
|
||||||
|
? (signature?.roles ?? matchTransactionHeaders(effectiveData[0]))
|
||||||
|
: null;
|
||||||
|
|
||||||
const dataStartIdx = hasHeader ? 1 : 0;
|
const dataStartIdx = hasHeader ? 1 : 0;
|
||||||
const sampleRows = effectiveData.slice(dataStartIdx, dataStartIdx + 20);
|
const sampleRows = effectiveData.slice(dataStartIdx, dataStartIdx + 20);
|
||||||
|
|
@ -254,7 +297,12 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
|
||||||
);
|
);
|
||||||
|
|
||||||
// Step 7: Determine amount mode
|
// Step 7: Determine amount mode
|
||||||
const amountResult = detectAmountMode(sampleRows, amountCandidates, lexical);
|
const amountResult = detectAmountMode(
|
||||||
|
sampleRows,
|
||||||
|
amountCandidates,
|
||||||
|
lexical,
|
||||||
|
signature
|
||||||
|
);
|
||||||
if (!amountResult) return failed;
|
if (!amountResult) return failed;
|
||||||
|
|
||||||
// Step 7b: Refuse the third amount format — unsigned magnitudes plus a
|
// Step 7b: Refuse the third amount format — unsigned magnitudes plus a
|
||||||
|
|
@ -303,7 +351,12 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
|
||||||
// Step 8: Replay what we just decided, over the WHOLE file. The sample above
|
// Step 8: Replay what we just decided, over the WHOLE file. The sample above
|
||||||
// is 20 rows because that is enough to decide a shape; the score is what the
|
// is 20 rows because that is enough to decide a shape; the score is what the
|
||||||
// user is told, so it has to describe the actual file.
|
// user is told, so it has to describe the actual file.
|
||||||
return { status: "ok", config, score: scoreConfig(config, data) };
|
return {
|
||||||
|
status: "ok",
|
||||||
|
config,
|
||||||
|
score: scoreConfig(config, data),
|
||||||
|
bank: signature?.signature.id ?? null,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -711,13 +764,44 @@ interface DebitCreditResult {
|
||||||
|
|
||||||
type AmountModeResult = SingleAmountResult | DebitCreditResult;
|
type AmountModeResult = SingleAmountResult | DebitCreditResult;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decide the amount mode, and which column(s) carry it.
|
||||||
|
*
|
||||||
|
* `signature` is the one hint that outranks the sparse-complementary scan, and
|
||||||
|
* only because that scan cannot be told apart from the truth by shape alone:
|
||||||
|
* RBC's `CAD$` and `USD$` ARE complementary — the USD column is empty on a
|
||||||
|
* Canadian account — so a file whose amounts are one signed column reads as a
|
||||||
|
* debit/credit pair, with every credit imported as an expense. A bank that
|
||||||
|
* documents its layout settles that; nothing else in the file can.
|
||||||
|
*
|
||||||
|
* It stays a preference all the same: the columns it names must be candidates
|
||||||
|
* the shape scan itself proposed. A signature naming a column that parses as
|
||||||
|
* nothing numeric is dropped here and the generic path decides, exactly like a
|
||||||
|
* mismatched label.
|
||||||
|
*/
|
||||||
function detectAmountMode(
|
function detectAmountMode(
|
||||||
rows: string[][],
|
rows: string[][],
|
||||||
amountCandidates: number[],
|
amountCandidates: number[],
|
||||||
lexical: LexicalHeaderMap | null
|
lexical: LexicalHeaderMap | null,
|
||||||
|
signature: BankSignatureMatch | null
|
||||||
): AmountModeResult | null {
|
): AmountModeResult | null {
|
||||||
if (amountCandidates.length === 0) return null;
|
if (amountCandidates.length === 0) return null;
|
||||||
|
|
||||||
|
if (signature) {
|
||||||
|
const { debit, credit, amount } = signature.roles;
|
||||||
|
if (
|
||||||
|
debit !== null &&
|
||||||
|
credit !== null &&
|
||||||
|
amountCandidates.includes(debit) &&
|
||||||
|
amountCandidates.includes(credit)
|
||||||
|
) {
|
||||||
|
return { mode: "debit_credit", debitCol: debit, creditCol: credit };
|
||||||
|
}
|
||||||
|
if (amount !== null && amountCandidates.includes(amount)) {
|
||||||
|
return detectSingleAmount(rows, amount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (amountCandidates.length === 1) {
|
if (amountCandidates.length === 1) {
|
||||||
return detectSingleAmount(rows, amountCandidates[0]);
|
return detectSingleAmount(rows, amountCandidates[0]);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue