("read_file_content", {
- filePath: state.selectedFiles[0].file_path,
- encoding: state.sourceConfig.encoding,
- });
+ const filePath = state.selectedFiles[0].file_path;
+ // Same path the automatic run takes, so the button REPLAYS detection
+ // rather than running a second, drifting variant of it.
+ const newConfig = await detectFormatForFile(filePath, state.sourceConfig);
+ if (!newConfig) return; // reason already dispatched, loading already off
- const outcome = runAutoDetect(content);
+ dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig });
+ dispatch({ type: "SET_LOADING", payload: false });
- if (outcome.status === "ok") {
- const result = outcome.config;
- const newConfig = {
- ...state.sourceConfig,
- delimiter: result.delimiter,
- hasHeader: result.hasHeader,
- skipLines: result.skipLines,
- dateFormat: result.dateFormat,
- columnMapping: result.columnMapping,
- amountMode: result.amountMode,
- signConvention: result.signConvention,
- };
- dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig });
- dispatch({ type: "SET_LOADING", payload: false });
-
- // Refresh column headers with new config
- await loadHeadersWithConfig(
- state.selectedFiles[0].file_path,
- newConfig.delimiter,
- newConfig.encoding,
- newConfig.skipLines,
- newConfig.hasHeader
- );
- } else {
- // A refused format states WHY (#327): "I cannot read this file" and
- // "this file is a shape this version would import backwards" are not
- // the same news, and only the second one is the app's own limitation.
- // `SET_ERROR` clears the loading flag on its own.
- dispatch({
- type: "SET_ERROR",
- payload:
- outcome.status === "rejected"
- ? outcome.reason
- : "import.errors.autoDetectFailed",
- });
- }
+ // Refresh column headers with new config
+ await loadHeadersWithConfig(
+ filePath,
+ newConfig.delimiter,
+ newConfig.encoding,
+ newConfig.skipLines,
+ newConfig.hasHeader
+ );
} catch (e) {
dispatch({
type: "SET_ERROR",
payload: e instanceof Error ? e.message : String(e),
});
}
- }, [state.selectedFiles, state.sourceConfig, loadHeadersWithConfig]);
+ }, [
+ state.selectedFiles,
+ state.sourceConfig,
+ detectFormatForFile,
+ loadHeadersWithConfig,
+ ]);
const saveConfigAsTemplate = useCallback(async (name: string) => {
await createTemplate({ name, ...formatToRow(state.sourceConfig) });
@@ -939,6 +1016,9 @@ export function useImportWizard() {
// Applying a template COPIES its format onto the source. The id recorded
// here is provenance only — the copy is what the next import reads.
dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: templateId });
+ // The format is the template's now, so a score measured on the detected
+ // one would be vouching for a configuration it never saw.
+ dispatch({ type: "SET_DETECTION_SCORE", payload: null });
// Reload headers with new config
if (state.selectedFiles.length > 0) {
diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json
index 3a55f8f..2e303e9 100644
--- a/src/i18n/locales/en.json
+++ b/src/i18n/locales/en.json
@@ -113,7 +113,10 @@
"templateSaved": "Template saved",
"deleteTemplate": "Delete template",
"noTemplates": "No templates saved",
- "updateTemplate": "Update template"
+ "updateTemplate": "Update template",
+ "detectionRecognized": "Format recognized — {{read}} of {{total}} rows read",
+ "detectionUncertain": "Uncertain format — only {{read}} of {{total}} rows read",
+ "detectionUncertainHint": "Check the column mapping and the date format, then look at the preview before importing."
},
"preview": {
"title": "Data Preview",
diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json
index 370a021..8e76eca 100644
--- a/src/i18n/locales/fr.json
+++ b/src/i18n/locales/fr.json
@@ -113,7 +113,10 @@
"templateSaved": "Modèle sauvegardé",
"deleteTemplate": "Supprimer le modèle",
"noTemplates": "Aucun modèle sauvegardé",
- "updateTemplate": "Mettre à jour le modèle"
+ "updateTemplate": "Mettre à jour le modèle",
+ "detectionRecognized": "Format reconnu — {{read}} des {{total}} lignes lues",
+ "detectionUncertain": "Format incertain — {{read}} des {{total}} lignes lues seulement",
+ "detectionUncertainHint": "Vérifiez le mapping des colonnes et le format de date, puis regardez l'aperçu avant d'importer."
},
"preview": {
"title": "Aperçu des données",
diff --git a/src/pages/ImportPage.tsx b/src/pages/ImportPage.tsx
index 211526a..056238e 100644
--- a/src/pages/ImportPage.tsx
+++ b/src/pages/ImportPage.tsx
@@ -108,6 +108,7 @@ export default function ImportPage() {
onUpdateTemplate={updateConfigTemplate}
onDeleteTemplate={deleteConfigTemplate}
selectedTemplateId={state.selectedTemplateId}
+ detectionScore={state.detectionScore}
isLoading={state.isLoading}
/>
diff --git a/src/utils/csvAutoDetect.test.ts b/src/utils/csvAutoDetect.test.ts
index 821ef40..87571ec 100644
--- a/src/utils/csvAutoDetect.test.ts
+++ b/src/utils/csvAutoDetect.test.ts
@@ -18,6 +18,7 @@ import {
autoDetectHoldingColumns,
analyzeHoldingsCsv,
autoDetectConfig,
+ CONFIDENCE_THRESHOLD,
detectImportFormat,
preprocessQuotedCSV,
} from "./csvAutoDetect";
@@ -826,3 +827,260 @@ describe("autoDetectConfig — degenerate input (#326)", () => {
).toBeNull();
});
});
+
+// =============================================================================
+// Issue #328 — the confidence score, and detection running on its own
+// =============================================================================
+//
+// Detection used to hand back a configuration it had never tested. These cases
+// pin the replay: a measured rate, computed by the production row rule, over the
+// rows of the real file.
+
+/** The score of a fixture, or a failure the test can name. */
+function scoreOf(name: CsvFixtureName) {
+ const outcome = detectImportFormat(readCsvFixture(name));
+ if (outcome.status !== "ok") {
+ throw new Error(`${name}: detection returned ${outcome.status}`);
+ }
+ return outcome.score;
+}
+
+describe("detectImportFormat — confidence score (#328)", () => {
+ it("scores every fixture the corpus expects a configuration for", () => {
+ for (const name of CSV_FIXTURE_NAMES) {
+ // `absolute-indicator` is REFUSED (#327), so it carries no score at all.
+ if (name === "absolute-indicator") continue;
+ const score = scoreOf(name);
+ expect(score.totalRows, name).toBeGreaterThan(0);
+ expect(score.readRows, name).toBe(score.totalRows);
+ expect(score.ratio, name).toBe(1);
+ expect(score.confident, name).toBe(true);
+ }
+ });
+
+ it("counts data rows only — header and preamble excluded", () => {
+ // Both files carry the same six transactions; `preamble` puts three lines
+ // of statement banner in front of the header. A score that counted them
+ // would report 6/10 on a file it reads perfectly.
+ expect(scoreOf("signed-amount").totalRows).toBe(6);
+ expect(scoreOf("preamble").totalRows).toBe(6);
+ expect(scoreOf("no-header").totalRows).toBe(6);
+ });
+
+ it("says nothing about the SIGN of what it read (#329)", () => {
+ // `all-positive` is the corpus' known defect: unsigned magnitudes, detected
+ // as `negative_expense`, so every credit imports as an expense. Every row
+ // still PARSES, so the score is a perfect 100 % — which is precisely why the
+ // threshold blocks nothing and the preview step stays mandatory.
+ const score = scoreOf("all-positive");
+ expect(score.ratio).toBe(1);
+ expect(score.confident).toBe(true);
+ });
+
+ it("carries no score on a refused format", () => {
+ const outcome = detectImportFormat(readCsvFixture("absolute-indicator"));
+ expect(outcome.status).toBe("rejected");
+ expect(outcome).not.toHaveProperty("score");
+ });
+});
+
+/**
+ * Ten rows, `unreadable` of which carry an empty date cell.
+ *
+ * An empty cell is skipped by the date-column scan (it divides on non-empty
+ * cells), so the column is still detected at a perfect rate while `mapRow`
+ * rejects those rows — which is what lets these cases move the score without
+ * moving the configuration.
+ */
+function withUnreadableRows(unreadable: number): string {
+ const rows: string[] = [];
+ for (let i = 0; i < 10; i++) {
+ const day = String(i + 1).padStart(2, "0");
+ rows.push(
+ i < unreadable
+ ? `;ACHAT DIVERS ${i};-1${i},00`
+ : `${day}/03/2025;ACHAT DIVERS ${i};-1${i},00`
+ );
+ }
+ return withHeader("Date;Description;Montant", rows);
+}
+
+describe("detectImportFormat — the 90 % threshold (#328)", () => {
+ it("reports the real count of rows it could not read", () => {
+ const outcome = detectImportFormat(withUnreadableRows(3));
+ if (outcome.status !== "ok") throw new Error(outcome.status);
+ expect(outcome.score.totalRows).toBe(10);
+ expect(outcome.score.readRows).toBe(7);
+ expect(outcome.score.ratio).toBeCloseTo(0.7);
+ expect(outcome.score.confident).toBe(false);
+ });
+
+ it("is confident AT the threshold, not merely above it", () => {
+ const outcome = detectImportFormat(withUnreadableRows(1));
+ if (outcome.status !== "ok") throw new Error(outcome.status);
+ expect(outcome.score.ratio).toBeCloseTo(CONFIDENCE_THRESHOLD);
+ expect(outcome.score.confident).toBe(true);
+ });
+
+ it("drops below the threshold one row further down", () => {
+ const outcome = detectImportFormat(withUnreadableRows(2));
+ if (outcome.status !== "ok") throw new Error(outcome.status);
+ expect(outcome.score.confident).toBe(false);
+ });
+
+ it("ignores a trailing blank line rather than counting it unread", () => {
+ const source = withHeader("Date;Description;Montant", SIGNED_ROWS);
+ const clean = detectImportFormat(source);
+ const trailing = detectImportFormat(source + "\n\n");
+ if (clean.status !== "ok" || trailing.status !== "ok") {
+ throw new Error("detection failed on a well-formed file");
+ }
+ expect(trailing.score).toEqual(clean.score);
+ });
+});
+
+describe("the score runs the production row rule, not a copy (#328)", () => {
+ it("delegates to `mapRow` and its column-level separator arbitration", () => {
+ // The revision of #328 is explicit: the score must consume `mapRow`. Two
+ // mappers is the divergence class this chantier removes — the score would
+ // read 100 % while the import wrote different amounts.
+ const SRC = readFileSync(
+ resolve(import.meta.dirname, "csvAutoDetect.ts"),
+ "utf-8"
+ );
+ expect(SRC).toContain(
+ 'import { detectAmountSeparators, mapRow } from "./importFormat"'
+ );
+ expect(SRC).toContain("mapRow(raw, format, { decimalSeparators })");
+ });
+
+ it("agrees row for row with an end-to-end parse of the same file", () => {
+ // Two paths over the same file: the score's internal replay, and the corpus
+ // helper that reproduces `parseFilesInternal`. They must not disagree.
+ for (const name of CSV_FIXTURE_NAMES) {
+ if (name === "absolute-indicator") continue;
+ const endToEnd = parseFixtureEndToEnd(name);
+ const read = endToEnd.rows.filter((r) => !("error" in r)).length;
+ const score = scoreOf(name);
+ expect(score.readRows, name).toBe(read);
+ expect(score.totalRows, name).toBe(endToEnd.rows.length);
+ }
+ });
+});
+
+// -----------------------------------------------------------------------------
+// Static guards: detection runs on its own, and ONLY on an unconfigured source.
+//
+// Both live inside `useCallback`s of a React hook and the repository has no
+// jsdom, so the wiring is asserted on the source — same technique as the guards
+// above and in importFormat.test.ts.
+// -----------------------------------------------------------------------------
+
+const WIZARD = readFileSync(
+ resolve(import.meta.dirname, "..", "hooks", "useImportWizard.ts"),
+ "utf-8"
+);
+
+/** The body of `selectSource`, up to the next `useCallback` declaration. */
+function selectSourceBody(): string {
+ const start = WIZARD.indexOf("const selectSource =");
+ const end = WIZARD.indexOf("const loadHeadersWithConfig =", start);
+ expect(start).toBeGreaterThan(-1);
+ expect(end).toBeGreaterThan(start);
+ return WIZARD.slice(start, end);
+}
+
+describe("useImportWizard — detection fires on its own (#328)", () => {
+ it("runs detection while opening a source, not only on the button", () => {
+ expect(selectSourceBody()).toContain("detectFormatForFile(");
+ });
+
+ it("guards that run on the ABSENCE of a stored source", () => {
+ // `!existing`, never `!restored`: a stored format that will not decode
+ // already reports its own error, and re-detecting over it would replace
+ // that message with a silent guess.
+ expect(selectSourceBody()).toContain(
+ "if (!existing && source.files.length > 0)"
+ );
+ });
+
+ it("never detects on the arm that restored a stored format", () => {
+ const body = selectSourceBody();
+ const armStart = body.indexOf("if (restored) {");
+ const restoreArm = body.slice(armStart, body.indexOf("} else {", armStart));
+ expect(restoreArm.length).toBeGreaterThan(0);
+ expect(restoreArm).not.toContain("detectFormatForFile");
+ expect(restoreArm).not.toContain("runAutoDetect");
+ });
+
+ it("keeps a single detection path for the button and the automatic run", () => {
+ // One detector, two callers: the wand button and `selectSource`.
+ expect(WIZARD.match(/runAutoDetect\(/g)).toHaveLength(1);
+ expect(WIZARD.match(/await detectFormatForFile\(/g)).toHaveLength(2);
+ });
+
+ it("drops the score as soon as the format is edited by hand", () => {
+ // A banner still claiming "147 of 150 rows" over a mapping nobody measured
+ // is the same misinformation the score exists to remove. Compared through
+ // the codec, so renaming the source — which changes nothing about reading —
+ // keeps it.
+ const body = WIZARD.slice(
+ WIZARD.indexOf("const updateConfig ="),
+ WIZARD.indexOf("const toggleFile =")
+ );
+ expect(body).toContain("JSON.stringify(formatToRow(config))");
+ expect(body).toContain('dispatch({ type: "SET_DETECTION_SCORE", payload: null })');
+ });
+
+ it("drops the score when a template overwrites the format", () => {
+ const body = WIZARD.slice(
+ WIZARD.indexOf("const applyConfigTemplate ="),
+ WIZARD.indexOf("const updateConfigTemplate =")
+ );
+ expect(body).toContain('dispatch({ type: "SET_DETECTION_SCORE", payload: null })');
+ });
+});
+
+describe("SourceConfigPanel — banner and sign convention (#328)", () => {
+ const PANEL = readFileSync(
+ resolve(
+ import.meta.dirname,
+ "..",
+ "components",
+ "import",
+ "SourceConfigPanel.tsx"
+ ),
+ "utf-8"
+ );
+
+ it("hides the sign convention in debit/credit mode", () => {
+ // `mapRow` computes `credit - debit` on magnitudes there and never reads
+ // `signConvention` — the selector changed nothing.
+ expect(PANEL).toContain('config.amountMode === "single" && (');
+ });
+
+ it("renders both arms of the score through i18n", () => {
+ expect(PANEL).toContain("import.config.detectionRecognized");
+ expect(PANEL).toContain("import.config.detectionUncertain");
+ });
+
+ it("carries every new string in both languages", () => {
+ for (const key of [
+ "detectionRecognized",
+ "detectionUncertain",
+ "detectionUncertainHint",
+ ] as const) {
+ expect(fr.import.config[key].length, `fr.${key}`).toBeGreaterThan(0);
+ expect(en.import.config[key].length, `en.${key}`).toBeGreaterThan(0);
+ }
+ });
+
+ it("interpolates the detailed count in both languages", () => {
+ for (const key of ["detectionRecognized", "detectionUncertain"] as const) {
+ expect(fr.import.config[key], `fr.${key}`).toContain("{{read}}");
+ expect(fr.import.config[key], `fr.${key}`).toContain("{{total}}");
+ expect(en.import.config[key], `en.${key}`).toContain("{{read}}");
+ expect(en.import.config[key], `en.${key}`).toContain("{{total}}");
+ }
+ });
+});
diff --git a/src/utils/csvAutoDetect.ts b/src/utils/csvAutoDetect.ts
index 79a5c5f..3f91cd7 100644
--- a/src/utils/csvAutoDetect.ts
+++ b/src/utils/csvAutoDetect.ts
@@ -10,7 +10,13 @@ import {
normalizeHeaderCell,
type LexicalHeaderMap,
} from "./headerDictionary";
-import type { ColumnMapping, AmountMode, SignConvention } from "../shared/types";
+import { detectAmountSeparators, mapRow } from "./importFormat";
+import type {
+ ColumnMapping,
+ AmountMode,
+ ImportFormat,
+ SignConvention,
+} from "../shared/types";
export interface AutoDetectResult {
delimiter: string;
@@ -29,6 +35,40 @@ export interface AutoDetectResult {
*/
export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat";
+/**
+ * How well the detected configuration reads the file it was detected from.
+ *
+ * Detection used to return a configuration without ever testing it: a plausible
+ * delimiter, a plausible date column and a plausible amount column produced a
+ * result whether or not a single row survived them. The score closes that hole
+ * by REPLAYING the configuration — the confidence reported is measured, not
+ * asserted.
+ *
+ * What it does NOT measure is whether the amounts carry the right SIGN. A file
+ * of unsigned magnitudes (`all-positive` in the corpus) scores 100 % while every
+ * expense imports as income, because every row parses. That is why the threshold
+ * only colours the banner and blocks nothing: the preview step (#329) is the
+ * real net, and it is traversed at every import whatever the score says.
+ */
+export interface DetectionScore {
+ /** Rows whose date AND amount the configuration reads. */
+ readRows: number;
+ /** Data rows the replay ran on — header and skipped preamble excluded. */
+ totalRows: number;
+ /** `readRows / totalRows`, or 0 when the file carries no data row. */
+ ratio: number;
+ /** `ratio >= CONFIDENCE_THRESHOLD`. */
+ confident: boolean;
+}
+
+/**
+ * Share of rows that must be read for the result to be announced as recognised
+ * rather than doubtful (decided in planning). A statement mixing a few unusable
+ * lines into an otherwise clean file is common enough that a stricter bar would
+ * cry wolf; anything below this is a mapping worth a second look.
+ */
+export const CONFIDENCE_THRESHOLD = 0.9;
+
/**
* The full-fidelity outcome of detection.
*
@@ -38,7 +78,7 @@ export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat";
* backwards" call for different messages. `detectImportFormat` separates them.
*/
export type AutoDetectOutcome =
- | { status: "ok"; config: AutoDetectResult }
+ | { status: "ok"; config: AutoDetectResult; score: DetectionScore }
| { status: "rejected"; reason: AutoDetectRejectionKey }
| { status: "failed" };
@@ -250,17 +290,70 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
signConvention = amountResult.signConvention;
}
+ const config: AutoDetectResult = {
+ delimiter,
+ hasHeader,
+ skipLines,
+ dateFormat: dateResult.format,
+ columnMapping: mapping,
+ amountMode: amountResult.mode,
+ signConvention,
+ };
+
+ // Step 8: Replay what we just decided, over the WHOLE file. The sample above
+ // is 20 rows because that is enough to decide a shape; the score is what the
+ // user is told, so it has to describe the actual file.
+ return { status: "ok", config, score: scoreConfig(config, data) };
+}
+
+/**
+ * Replay a detected configuration over the file's data rows and count how many
+ * of them it reads.
+ *
+ * Deliberately NOT a rule of its own: the row is mapped by `mapRow`, the same
+ * pure function `parseFilesInternal` runs at import time, under the same
+ * column-level decimal arbitration. Two mappers would be the exact class of
+ * divergence this chantier exists to remove — a score could then read 100 %
+ * while the import wrote different amounts.
+ *
+ * The row selection mirrors `parseFilesInternal` line for line, including the
+ * lone-empty-cell skip: a trailing blank line must not count as an unread row.
+ */
+function scoreConfig(
+ config: AutoDetectResult,
+ data: string[][]
+): DetectionScore {
+ // `mapRow` reads no byte and therefore never looks at `encoding` — the
+ // content reached us already decoded. Naming it here only satisfies the type.
+ const format: ImportFormat = { ...config, encoding: "utf-8" };
+
+ const dataRows: string[][] = [];
+ const startIdx = config.skipLines + (config.hasHeader ? 1 : 0);
+ for (let i = startIdx; i < data.length; i++) {
+ const raw = data[i];
+ if (raw.length <= 1 && raw[0]?.trim() === "") continue;
+ dataRows.push(raw);
+ }
+
+ const decimalSeparators = detectAmountSeparators(dataRows, format);
+
+ let readRows = 0;
+ for (const raw of dataRows) {
+ try {
+ if (mapRow(raw, format, { decimalSeparators }).parsed) readRows++;
+ } catch {
+ // `mapRow` is documented not to throw; if it ever did, that is one
+ // unreadable row, not a detection that collapses on the whole file.
+ }
+ }
+
+ const totalRows = dataRows.length;
+ const ratio = totalRows === 0 ? 0 : readRows / totalRows;
return {
- status: "ok",
- config: {
- delimiter,
- hasHeader,
- skipLines,
- dateFormat: dateResult.format,
- columnMapping: mapping,
- amountMode: amountResult.mode,
- signConvention,
- },
+ readRows,
+ totalRows,
+ ratio,
+ confident: totalRows > 0 && ratio >= CONFIDENCE_THRESHOLD,
};
}