feat(import): score the detected format and run detection on its own #338

Closed
maximus wants to merge 1 commit from issue-328-confidence-score into issue-327-lexical-header-detection
7 changed files with 597 additions and 99 deletions
Showing only changes of commit bf608b9d67 - Show all commits

View file

@ -1,6 +1,6 @@
import { useState } from "react"; import { useState } from "react";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { Wand2, Check, Save, X } from "lucide-react"; import { Wand2, Check, Save, X, AlertTriangle } from "lucide-react";
import type { import type {
ScannedSource, ScannedSource,
ScannedFile, ScannedFile,
@ -9,6 +9,7 @@ import type {
ColumnMapping, ColumnMapping,
ImportConfigTemplate, ImportConfigTemplate,
} from "../../shared/types"; } from "../../shared/types";
import type { DetectionScore } from "../../utils/csvAutoDetect";
import ColumnMappingEditor from "./ColumnMappingEditor"; import ColumnMappingEditor from "./ColumnMappingEditor";
interface SourceConfigPanelProps { interface SourceConfigPanelProps {
@ -27,6 +28,8 @@ interface SourceConfigPanelProps {
onUpdateTemplate: () => void; onUpdateTemplate: () => void;
onDeleteTemplate: (id: number) => void; onDeleteTemplate: (id: number) => void;
selectedTemplateId: number | null; selectedTemplateId: number | null;
/** Result of the last detection run, or null when none ran on this source. */
detectionScore?: DetectionScore | null;
isLoading?: boolean; isLoading?: boolean;
} }
@ -46,6 +49,7 @@ export default function SourceConfigPanel({
onUpdateTemplate, onUpdateTemplate,
onDeleteTemplate, onDeleteTemplate,
selectedTemplateId, selectedTemplateId,
detectionScore,
isLoading, isLoading,
}: SourceConfigPanelProps) { }: SourceConfigPanelProps) {
const { t } = useTranslation(); const { t } = useTranslation();
@ -76,6 +80,52 @@ export default function SourceConfigPanel({
</button> </button>
</div> </div>
{/*
Detection result. Present only when a detection actually ran on THIS
source a source opened on its stored format shows nothing, because
that format was chosen once and is not being re-guessed.
The threshold colours the banner and nothing else: both arms leave every
control editable and every button enabled. A low score is a reason to
look at the preview, not a refusal to import.
*/}
{detectionScore && (
<div
className={`p-3 rounded-xl bg-[var(--card)] border-2 flex items-start gap-2 ${
detectionScore.confident
? "border-[var(--border)]"
: "border-[var(--accent)]"
}`}
>
{detectionScore.confident ? (
<Check size={16} className="text-[var(--positive)] shrink-0 mt-0.5" />
) : (
<AlertTriangle
size={16}
className="text-[var(--accent)] shrink-0 mt-0.5"
/>
)}
<div>
<p className="text-sm text-[var(--foreground)]">
{t(
detectionScore.confident
? "import.config.detectionRecognized"
: "import.config.detectionUncertain",
{
read: detectionScore.readRows,
total: detectionScore.totalRows,
}
)}
</p>
{!detectionScore.confident && (
<p className="text-xs text-[var(--muted-foreground)] mt-0.5">
{t("import.config.detectionUncertainHint")}
</p>
)}
</div>
</div>
)}
{/* Template row */} {/* Template row */}
<div className="flex items-center gap-3 flex-wrap"> <div className="flex items-center gap-3 flex-wrap">
<div className="flex items-center gap-2 flex-1 min-w-[200px]"> <div className="flex items-center gap-2 flex-1 min-w-[200px]">
@ -270,7 +320,16 @@ export default function SourceConfigPanel({
</div> </div>
</div> </div>
{/* Sign convention */} {/*
Sign convention single-amount mode ONLY.
`mapRow` computes `credit - debit` on magnitudes in debit/credit mode and
never reads `signConvention` there; the direction is carried by which
column holds the value. Showing the selector anyway offered a control
that changed nothing, which reads as "the app ignored my setting".
The stored value is left untouched: hidden, not reset.
*/}
{config.amountMode === "single" && (
<div> <div>
<label className="block text-sm text-[var(--muted-foreground)] mb-1"> <label className="block text-sm text-[var(--muted-foreground)] mb-1">
{t("import.config.signConvention")} {t("import.config.signConvention")}
@ -304,6 +363,7 @@ export default function SourceConfigPanel({
</label> </label>
</div> </div>
</div> </div>
)}
{/* Column mapping */} {/* Column mapping */}
{headers.length > 0 && ( {headers.length > 0 && (

View file

@ -42,6 +42,7 @@ import {
import { import {
preprocessQuotedCSV, preprocessQuotedCSV,
detectImportFormat as runAutoDetect, detectImportFormat as runAutoDetect,
type DetectionScore,
} from "../utils/csvAutoDetect"; } from "../utils/csvAutoDetect";
import { import {
detectAmountSeparators, detectAmountSeparators,
@ -78,6 +79,11 @@ interface WizardState {
importedFilesBySource: Map<string, Set<string>>; importedFilesBySource: Map<string, Set<string>>;
configTemplates: ImportConfigTemplate[]; configTemplates: ImportConfigTemplate[];
selectedTemplateId: number | null; selectedTemplateId: number | null;
/**
* Result of the LAST detection run on the selected source, or null when none
* ran a source opened on its stored format, which is never re-detected.
*/
detectionScore: DetectionScore | null;
} }
type WizardAction = type WizardAction =
@ -99,6 +105,7 @@ type WizardAction =
| { type: "SET_CONFIGURED_SOURCES"; payload: { names: Set<string>; files: Map<string, Set<string>> } } | { type: "SET_CONFIGURED_SOURCES"; payload: { names: Set<string>; files: Map<string, Set<string>> } }
| { type: "SET_CONFIG_TEMPLATES"; payload: ImportConfigTemplate[] } | { type: "SET_CONFIG_TEMPLATES"; payload: ImportConfigTemplate[] }
| { type: "SET_SELECTED_TEMPLATE_ID"; payload: number | null } | { type: "SET_SELECTED_TEMPLATE_ID"; payload: number | null }
| { type: "SET_DETECTION_SCORE"; payload: DetectionScore | null }
| { type: "RESET" }; | { type: "RESET" };
const defaultConfig: SourceConfig = { const defaultConfig: SourceConfig = {
@ -133,6 +140,7 @@ const initialState: WizardState = {
importedFilesBySource: new Map(), importedFilesBySource: new Map(),
configTemplates: [], configTemplates: [],
selectedTemplateId: null, selectedTemplateId: null,
detectionScore: null,
}; };
function reducer(state: WizardState, action: WizardAction): WizardState { function reducer(state: WizardState, action: WizardAction): WizardState {
@ -148,7 +156,10 @@ function reducer(state: WizardState, action: WizardAction): WizardState {
case "SET_SCANNED_SOURCES": case "SET_SCANNED_SOURCES":
return { ...state, scannedSources: action.payload, isLoading: false }; return { ...state, scannedSources: action.payload, isLoading: false };
case "SET_SELECTED_SOURCE": case "SET_SELECTED_SOURCE":
return { ...state, selectedSource: action.payload }; // The score belongs to the source it was measured on. Carrying it over
// would let a source opened on its stored format inherit the confidence
// of the previous one, which is worse than showing nothing.
return { ...state, selectedSource: action.payload, detectionScore: null };
case "SET_SELECTED_FILES": case "SET_SELECTED_FILES":
return { ...state, selectedFiles: action.payload }; return { ...state, selectedFiles: action.payload };
case "SET_SOURCE_CONFIG": case "SET_SOURCE_CONFIG":
@ -199,6 +210,8 @@ function reducer(state: WizardState, action: WizardAction): WizardState {
return { ...state, configTemplates: action.payload }; return { ...state, configTemplates: action.payload };
case "SET_SELECTED_TEMPLATE_ID": case "SET_SELECTED_TEMPLATE_ID":
return { ...state, selectedTemplateId: action.payload }; return { ...state, selectedTemplateId: action.payload };
case "SET_DETECTION_SCORE":
return { ...state, detectionScore: action.payload };
case "RESET": case "RESET":
return { return {
...initialState, ...initialState,
@ -291,8 +304,59 @@ export function useImportWizard() {
} }
}, [state.importFolder, scanFolderInternal]); }, [state.importFolder, scanFolderInternal]);
/**
* Run detection on one file and report the outcome. Returns the detected
* format merged onto `base`, or null when detection produced no usable
* configuration (the reason is dispatched as the page error).
*
* THE single detection path, shared by the two things that start one: the
* magic-wand button and the automatic run on a source that has never been
* configured. Everything it needs is a parameter, so it never reads a piece of
* `state` its caller has just dispatched but not yet re-rendered which is
* exactly why the automatic run could not simply call the button's callback.
*
* `base` keeps the fields detection does not decide (the source name, the
* encoding), and the spread of `outcome.config` writes every field it does
* naming them one by one is how a field gets silently dropped (#324).
*/
const detectFormatForFile = useCallback(
async (filePath: string, base: SourceConfig): Promise<SourceConfig | null> => {
const content = await invoke<string>("read_file_content", {
filePath,
encoding: base.encoding,
});
const outcome = runAutoDetect(content);
if (outcome.status !== "ok") {
// A refused format states WHY (#327): "I cannot read this file" and
// "this file is a shape this version would import backwards" are not
// the same news, and only the second one is the app's own limitation.
// `SET_ERROR` clears the loading flag on its own.
dispatch({ type: "SET_DETECTION_SCORE", payload: null });
dispatch({
type: "SET_ERROR",
payload:
outcome.status === "rejected"
? outcome.reason
: "import.errors.autoDetectFailed",
});
return null;
}
dispatch({ type: "SET_DETECTION_SCORE", payload: outcome.score });
return { ...base, ...outcome.config };
},
[]
);
const selectSource = useCallback( const selectSource = useCallback(
async (source: ScannedSource) => { async (source: ScannedSource) => {
// The banner is page-wide: an error left by the previous source must not
// survive into this one. Cleared here, before anything that may set a new
// one — a stored format that will not decode, a detection that refuses.
dispatch({ type: "SET_ERROR", payload: null });
// Sort files: new files first, then already-imported // Sort files: new files first, then already-imported
const importedNames = state.importedFilesBySource.get(source.folder_name); const importedNames = state.importedFilesBySource.get(source.folder_name);
const sorted = [...source.files].sort((a, b) => { const sorted = [...source.files].sort((a, b) => {
@ -362,14 +426,35 @@ export function useImportWizard() {
} }
} }
dispatch({ const fresh: SourceConfig = {
type: "SET_SOURCE_CONFIG",
payload: {
...defaultConfig, ...defaultConfig,
name: source.folder_name, name: source.folder_name,
encoding: activeEncoding, encoding: activeEncoding,
}, };
});
// Detection fires on its own HERE and nowhere else: the arm reached
// when the source has no stored format at all. `defaultConfig` is
// `;` + `DD/MM/YYYY` + columns 0/1/2 — plausible enough to import a
// whole file wrong rather than fail, so leaving a never-configured
// source sitting on it is the defect (#328).
//
// The condition is `!existing`, not `!restored`: a source whose stored
// format will not decode already carries an explicit error, and
// re-detecting over it would replace that message with a silent guess.
// Its wand button is still there.
let active = fresh;
if (!existing && source.files.length > 0) {
const detected = await detectFormatForFile(
source.files[0].file_path,
fresh
);
if (detected) active = detected;
}
dispatch({ type: "SET_SOURCE_CONFIG", payload: active });
activeDelimiter = active.delimiter;
activeSkipLines = active.skipLines;
activeHasHeader = active.hasHeader;
} }
// Load preview headers from first file // Load preview headers from first file
@ -433,6 +518,19 @@ export function useImportWizard() {
(config: SourceConfig) => { (config: SourceConfig) => {
dispatch({ type: "SET_SOURCE_CONFIG", payload: config }); dispatch({ type: "SET_SOURCE_CONFIG", payload: config });
// The score measures the format DETECTION produced. The moment any of
// those eight fields is edited by hand, the number stops describing what
// the wizard would read — a banner still claiming "147 of 150 rows" over
// a mapping nobody measured is the same misinformation this chantier is
// removing. Comparing through the codec keeps a rename (the ninth field,
// which changes nothing about reading) from clearing it.
if (
JSON.stringify(formatToRow(config)) !==
JSON.stringify(formatToRow(state.sourceConfig))
) {
dispatch({ type: "SET_DETECTION_SCORE", payload: null });
}
// Reload headers when delimiter, encoding, skipLines, or hasHeader changes // Reload headers when delimiter, encoding, skipLines, or hasHeader changes
if (state.selectedFiles.length > 0) { if (state.selectedFiles.length > 0) {
loadHeadersWithConfig( loadHeadersWithConfig(
@ -444,7 +542,7 @@ export function useImportWizard() {
); );
} }
}, },
[state.selectedFiles, loadHeadersWithConfig] [state.selectedFiles, state.sourceConfig, loadHeadersWithConfig]
); );
const toggleFile = useCallback( const toggleFile = useCallback(
@ -863,56 +961,35 @@ export function useImportWizard() {
dispatch({ type: "SET_ERROR", payload: null }); dispatch({ type: "SET_ERROR", payload: null });
try { try {
const content = await invoke<string>("read_file_content", { const filePath = state.selectedFiles[0].file_path;
filePath: state.selectedFiles[0].file_path, // Same path the automatic run takes, so the button REPLAYS detection
encoding: state.sourceConfig.encoding, // rather than running a second, drifting variant of it.
}); const newConfig = await detectFormatForFile(filePath, state.sourceConfig);
if (!newConfig) return; // reason already dispatched, loading already off
const outcome = runAutoDetect(content);
if (outcome.status === "ok") {
const result = outcome.config;
const newConfig = {
...state.sourceConfig,
delimiter: result.delimiter,
hasHeader: result.hasHeader,
skipLines: result.skipLines,
dateFormat: result.dateFormat,
columnMapping: result.columnMapping,
amountMode: result.amountMode,
signConvention: result.signConvention,
};
dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig }); dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig });
dispatch({ type: "SET_LOADING", payload: false }); dispatch({ type: "SET_LOADING", payload: false });
// Refresh column headers with new config // Refresh column headers with new config
await loadHeadersWithConfig( await loadHeadersWithConfig(
state.selectedFiles[0].file_path, filePath,
newConfig.delimiter, newConfig.delimiter,
newConfig.encoding, newConfig.encoding,
newConfig.skipLines, newConfig.skipLines,
newConfig.hasHeader newConfig.hasHeader
); );
} else {
// A refused format states WHY (#327): "I cannot read this file" and
// "this file is a shape this version would import backwards" are not
// the same news, and only the second one is the app's own limitation.
// `SET_ERROR` clears the loading flag on its own.
dispatch({
type: "SET_ERROR",
payload:
outcome.status === "rejected"
? outcome.reason
: "import.errors.autoDetectFailed",
});
}
} catch (e) { } catch (e) {
dispatch({ dispatch({
type: "SET_ERROR", type: "SET_ERROR",
payload: e instanceof Error ? e.message : String(e), payload: e instanceof Error ? e.message : String(e),
}); });
} }
}, [state.selectedFiles, state.sourceConfig, loadHeadersWithConfig]); }, [
state.selectedFiles,
state.sourceConfig,
detectFormatForFile,
loadHeadersWithConfig,
]);
const saveConfigAsTemplate = useCallback(async (name: string) => { const saveConfigAsTemplate = useCallback(async (name: string) => {
await createTemplate({ name, ...formatToRow(state.sourceConfig) }); await createTemplate({ name, ...formatToRow(state.sourceConfig) });
@ -939,6 +1016,9 @@ export function useImportWizard() {
// Applying a template COPIES its format onto the source. The id recorded // Applying a template COPIES its format onto the source. The id recorded
// here is provenance only — the copy is what the next import reads. // here is provenance only — the copy is what the next import reads.
dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: templateId }); dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: templateId });
// The format is the template's now, so a score measured on the detected
// one would be vouching for a configuration it never saw.
dispatch({ type: "SET_DETECTION_SCORE", payload: null });
// Reload headers with new config // Reload headers with new config
if (state.selectedFiles.length > 0) { if (state.selectedFiles.length > 0) {

View file

@ -113,7 +113,10 @@
"templateSaved": "Template saved", "templateSaved": "Template saved",
"deleteTemplate": "Delete template", "deleteTemplate": "Delete template",
"noTemplates": "No templates saved", "noTemplates": "No templates saved",
"updateTemplate": "Update template" "updateTemplate": "Update template",
"detectionRecognized": "Format recognized — {{read}} of {{total}} rows read",
"detectionUncertain": "Uncertain format — only {{read}} of {{total}} rows read",
"detectionUncertainHint": "Check the column mapping and the date format, then look at the preview before importing."
}, },
"preview": { "preview": {
"title": "Data Preview", "title": "Data Preview",

View file

@ -113,7 +113,10 @@
"templateSaved": "Modèle sauvegardé", "templateSaved": "Modèle sauvegardé",
"deleteTemplate": "Supprimer le modèle", "deleteTemplate": "Supprimer le modèle",
"noTemplates": "Aucun modèle sauvegardé", "noTemplates": "Aucun modèle sauvegardé",
"updateTemplate": "Mettre à jour le modèle" "updateTemplate": "Mettre à jour le modèle",
"detectionRecognized": "Format reconnu — {{read}} des {{total}} lignes lues",
"detectionUncertain": "Format incertain — {{read}} des {{total}} lignes lues seulement",
"detectionUncertainHint": "Vérifiez le mapping des colonnes et le format de date, puis regardez l'aperçu avant d'importer."
}, },
"preview": { "preview": {
"title": "Aperçu des données", "title": "Aperçu des données",

View file

@ -108,6 +108,7 @@ export default function ImportPage() {
onUpdateTemplate={updateConfigTemplate} onUpdateTemplate={updateConfigTemplate}
onDeleteTemplate={deleteConfigTemplate} onDeleteTemplate={deleteConfigTemplate}
selectedTemplateId={state.selectedTemplateId} selectedTemplateId={state.selectedTemplateId}
detectionScore={state.detectionScore}
isLoading={state.isLoading} isLoading={state.isLoading}
/> />
<div className="flex items-center justify-between pt-6 border-t border-[var(--border)]"> <div className="flex items-center justify-between pt-6 border-t border-[var(--border)]">

View file

@ -18,6 +18,7 @@ import {
autoDetectHoldingColumns, autoDetectHoldingColumns,
analyzeHoldingsCsv, analyzeHoldingsCsv,
autoDetectConfig, autoDetectConfig,
CONFIDENCE_THRESHOLD,
detectImportFormat, detectImportFormat,
preprocessQuotedCSV, preprocessQuotedCSV,
} from "./csvAutoDetect"; } from "./csvAutoDetect";
@ -826,3 +827,260 @@ describe("autoDetectConfig — degenerate input (#326)", () => {
).toBeNull(); ).toBeNull();
}); });
}); });
// =============================================================================
// Issue #328 — the confidence score, and detection running on its own
// =============================================================================
//
// Detection used to hand back a configuration it had never tested. These cases
// pin the replay: a measured rate, computed by the production row rule, over the
// rows of the real file.
/** The score of a fixture, or a failure the test can name. */
function scoreOf(name: CsvFixtureName) {
const outcome = detectImportFormat(readCsvFixture(name));
if (outcome.status !== "ok") {
throw new Error(`${name}: detection returned ${outcome.status}`);
}
return outcome.score;
}
describe("detectImportFormat — confidence score (#328)", () => {
it("scores every fixture the corpus expects a configuration for", () => {
for (const name of CSV_FIXTURE_NAMES) {
// `absolute-indicator` is REFUSED (#327), so it carries no score at all.
if (name === "absolute-indicator") continue;
const score = scoreOf(name);
expect(score.totalRows, name).toBeGreaterThan(0);
expect(score.readRows, name).toBe(score.totalRows);
expect(score.ratio, name).toBe(1);
expect(score.confident, name).toBe(true);
}
});
it("counts data rows only — header and preamble excluded", () => {
// Both files carry the same six transactions; `preamble` puts three lines
// of statement banner in front of the header. A score that counted them
// would report 6/10 on a file it reads perfectly.
expect(scoreOf("signed-amount").totalRows).toBe(6);
expect(scoreOf("preamble").totalRows).toBe(6);
expect(scoreOf("no-header").totalRows).toBe(6);
});
it("says nothing about the SIGN of what it read (#329)", () => {
// `all-positive` is the corpus' known defect: unsigned magnitudes, detected
// as `negative_expense`, so every credit imports as an expense. Every row
// still PARSES, so the score is a perfect 100 % — which is precisely why the
// threshold blocks nothing and the preview step stays mandatory.
const score = scoreOf("all-positive");
expect(score.ratio).toBe(1);
expect(score.confident).toBe(true);
});
it("carries no score on a refused format", () => {
const outcome = detectImportFormat(readCsvFixture("absolute-indicator"));
expect(outcome.status).toBe("rejected");
expect(outcome).not.toHaveProperty("score");
});
});
/**
* Ten rows, `unreadable` of which carry an empty date cell.
*
* An empty cell is skipped by the date-column scan (it divides on non-empty
* cells), so the column is still detected at a perfect rate while `mapRow`
* rejects those rows which is what lets these cases move the score without
* moving the configuration.
*/
function withUnreadableRows(unreadable: number): string {
const rows: string[] = [];
for (let i = 0; i < 10; i++) {
const day = String(i + 1).padStart(2, "0");
rows.push(
i < unreadable
? `;ACHAT DIVERS ${i};-1${i},00`
: `${day}/03/2025;ACHAT DIVERS ${i};-1${i},00`
);
}
return withHeader("Date;Description;Montant", rows);
}
describe("detectImportFormat — the 90 % threshold (#328)", () => {
it("reports the real count of rows it could not read", () => {
const outcome = detectImportFormat(withUnreadableRows(3));
if (outcome.status !== "ok") throw new Error(outcome.status);
expect(outcome.score.totalRows).toBe(10);
expect(outcome.score.readRows).toBe(7);
expect(outcome.score.ratio).toBeCloseTo(0.7);
expect(outcome.score.confident).toBe(false);
});
it("is confident AT the threshold, not merely above it", () => {
const outcome = detectImportFormat(withUnreadableRows(1));
if (outcome.status !== "ok") throw new Error(outcome.status);
expect(outcome.score.ratio).toBeCloseTo(CONFIDENCE_THRESHOLD);
expect(outcome.score.confident).toBe(true);
});
it("drops below the threshold one row further down", () => {
const outcome = detectImportFormat(withUnreadableRows(2));
if (outcome.status !== "ok") throw new Error(outcome.status);
expect(outcome.score.confident).toBe(false);
});
it("ignores a trailing blank line rather than counting it unread", () => {
const source = withHeader("Date;Description;Montant", SIGNED_ROWS);
const clean = detectImportFormat(source);
const trailing = detectImportFormat(source + "\n\n");
if (clean.status !== "ok" || trailing.status !== "ok") {
throw new Error("detection failed on a well-formed file");
}
expect(trailing.score).toEqual(clean.score);
});
});
describe("the score runs the production row rule, not a copy (#328)", () => {
it("delegates to `mapRow` and its column-level separator arbitration", () => {
// The revision of #328 is explicit: the score must consume `mapRow`. Two
// mappers is the divergence class this chantier removes — the score would
// read 100 % while the import wrote different amounts.
const SRC = readFileSync(
resolve(import.meta.dirname, "csvAutoDetect.ts"),
"utf-8"
);
expect(SRC).toContain(
'import { detectAmountSeparators, mapRow } from "./importFormat"'
);
expect(SRC).toContain("mapRow(raw, format, { decimalSeparators })");
});
it("agrees row for row with an end-to-end parse of the same file", () => {
// Two paths over the same file: the score's internal replay, and the corpus
// helper that reproduces `parseFilesInternal`. They must not disagree.
for (const name of CSV_FIXTURE_NAMES) {
if (name === "absolute-indicator") continue;
const endToEnd = parseFixtureEndToEnd(name);
const read = endToEnd.rows.filter((r) => !("error" in r)).length;
const score = scoreOf(name);
expect(score.readRows, name).toBe(read);
expect(score.totalRows, name).toBe(endToEnd.rows.length);
}
});
});
// -----------------------------------------------------------------------------
// Static guards: detection runs on its own, and ONLY on an unconfigured source.
//
// Both live inside `useCallback`s of a React hook and the repository has no
// jsdom, so the wiring is asserted on the source — same technique as the guards
// above and in importFormat.test.ts.
// -----------------------------------------------------------------------------
const WIZARD = readFileSync(
resolve(import.meta.dirname, "..", "hooks", "useImportWizard.ts"),
"utf-8"
);
/** The body of `selectSource`, up to the next `useCallback` declaration. */
function selectSourceBody(): string {
const start = WIZARD.indexOf("const selectSource =");
const end = WIZARD.indexOf("const loadHeadersWithConfig =", start);
expect(start).toBeGreaterThan(-1);
expect(end).toBeGreaterThan(start);
return WIZARD.slice(start, end);
}
describe("useImportWizard — detection fires on its own (#328)", () => {
it("runs detection while opening a source, not only on the button", () => {
expect(selectSourceBody()).toContain("detectFormatForFile(");
});
it("guards that run on the ABSENCE of a stored source", () => {
// `!existing`, never `!restored`: a stored format that will not decode
// already reports its own error, and re-detecting over it would replace
// that message with a silent guess.
expect(selectSourceBody()).toContain(
"if (!existing && source.files.length > 0)"
);
});
it("never detects on the arm that restored a stored format", () => {
const body = selectSourceBody();
const armStart = body.indexOf("if (restored) {");
const restoreArm = body.slice(armStart, body.indexOf("} else {", armStart));
expect(restoreArm.length).toBeGreaterThan(0);
expect(restoreArm).not.toContain("detectFormatForFile");
expect(restoreArm).not.toContain("runAutoDetect");
});
it("keeps a single detection path for the button and the automatic run", () => {
// One detector, two callers: the wand button and `selectSource`.
expect(WIZARD.match(/runAutoDetect\(/g)).toHaveLength(1);
expect(WIZARD.match(/await detectFormatForFile\(/g)).toHaveLength(2);
});
it("drops the score as soon as the format is edited by hand", () => {
// A banner still claiming "147 of 150 rows" over a mapping nobody measured
// is the same misinformation the score exists to remove. Compared through
// the codec, so renaming the source — which changes nothing about reading —
// keeps it.
const body = WIZARD.slice(
WIZARD.indexOf("const updateConfig ="),
WIZARD.indexOf("const toggleFile =")
);
expect(body).toContain("JSON.stringify(formatToRow(config))");
expect(body).toContain('dispatch({ type: "SET_DETECTION_SCORE", payload: null })');
});
it("drops the score when a template overwrites the format", () => {
const body = WIZARD.slice(
WIZARD.indexOf("const applyConfigTemplate ="),
WIZARD.indexOf("const updateConfigTemplate =")
);
expect(body).toContain('dispatch({ type: "SET_DETECTION_SCORE", payload: null })');
});
});
describe("SourceConfigPanel — banner and sign convention (#328)", () => {
const PANEL = readFileSync(
resolve(
import.meta.dirname,
"..",
"components",
"import",
"SourceConfigPanel.tsx"
),
"utf-8"
);
it("hides the sign convention in debit/credit mode", () => {
// `mapRow` computes `credit - debit` on magnitudes there and never reads
// `signConvention` — the selector changed nothing.
expect(PANEL).toContain('config.amountMode === "single" && (');
});
it("renders both arms of the score through i18n", () => {
expect(PANEL).toContain("import.config.detectionRecognized");
expect(PANEL).toContain("import.config.detectionUncertain");
});
it("carries every new string in both languages", () => {
for (const key of [
"detectionRecognized",
"detectionUncertain",
"detectionUncertainHint",
] as const) {
expect(fr.import.config[key].length, `fr.${key}`).toBeGreaterThan(0);
expect(en.import.config[key].length, `en.${key}`).toBeGreaterThan(0);
}
});
it("interpolates the detailed count in both languages", () => {
for (const key of ["detectionRecognized", "detectionUncertain"] as const) {
expect(fr.import.config[key], `fr.${key}`).toContain("{{read}}");
expect(fr.import.config[key], `fr.${key}`).toContain("{{total}}");
expect(en.import.config[key], `en.${key}`).toContain("{{read}}");
expect(en.import.config[key], `en.${key}`).toContain("{{total}}");
}
});
});

View file

@ -10,7 +10,13 @@ import {
normalizeHeaderCell, normalizeHeaderCell,
type LexicalHeaderMap, type LexicalHeaderMap,
} from "./headerDictionary"; } from "./headerDictionary";
import type { ColumnMapping, AmountMode, SignConvention } from "../shared/types"; import { detectAmountSeparators, mapRow } from "./importFormat";
import type {
ColumnMapping,
AmountMode,
ImportFormat,
SignConvention,
} from "../shared/types";
export interface AutoDetectResult { export interface AutoDetectResult {
delimiter: string; delimiter: string;
@ -29,6 +35,40 @@ export interface AutoDetectResult {
*/ */
export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat"; export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat";
/**
* How well the detected configuration reads the file it was detected from.
*
* Detection used to return a configuration without ever testing it: a plausible
* delimiter, a plausible date column and a plausible amount column produced a
* result whether or not a single row survived them. The score closes that hole
* by REPLAYING the configuration the confidence reported is measured, not
* asserted.
*
* What it does NOT measure is whether the amounts carry the right SIGN. A file
* of unsigned magnitudes (`all-positive` in the corpus) scores 100 % while every
* expense imports as income, because every row parses. That is why the threshold
* only colours the banner and blocks nothing: the preview step (#329) is the
* real net, and it is traversed at every import whatever the score says.
*/
export interface DetectionScore {
/** Rows whose date AND amount the configuration reads. */
readRows: number;
/** Data rows the replay ran on — header and skipped preamble excluded. */
totalRows: number;
/** `readRows / totalRows`, or 0 when the file carries no data row. */
ratio: number;
/** `ratio >= CONFIDENCE_THRESHOLD`. */
confident: boolean;
}
/**
* Share of rows that must be read for the result to be announced as recognised
* rather than doubtful (decided in planning). A statement mixing a few unusable
* lines into an otherwise clean file is common enough that a stricter bar would
* cry wolf; anything below this is a mapping worth a second look.
*/
export const CONFIDENCE_THRESHOLD = 0.9;
/** /**
* The full-fidelity outcome of detection. * The full-fidelity outcome of detection.
* *
@ -38,7 +78,7 @@ export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat";
* backwards" call for different messages. `detectImportFormat` separates them. * backwards" call for different messages. `detectImportFormat` separates them.
*/ */
export type AutoDetectOutcome = export type AutoDetectOutcome =
| { status: "ok"; config: AutoDetectResult } | { status: "ok"; config: AutoDetectResult; score: DetectionScore }
| { status: "rejected"; reason: AutoDetectRejectionKey } | { status: "rejected"; reason: AutoDetectRejectionKey }
| { status: "failed" }; | { status: "failed" };
@ -250,9 +290,7 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
signConvention = amountResult.signConvention; signConvention = amountResult.signConvention;
} }
return { const config: AutoDetectResult = {
status: "ok",
config: {
delimiter, delimiter,
hasHeader, hasHeader,
skipLines, skipLines,
@ -260,7 +298,62 @@ export function detectImportFormat(rawContent: string): AutoDetectOutcome {
columnMapping: mapping, columnMapping: mapping,
amountMode: amountResult.mode, amountMode: amountResult.mode,
signConvention, signConvention,
}, };
// Step 8: Replay what we just decided, over the WHOLE file. The sample above
// is 20 rows because that is enough to decide a shape; the score is what the
// user is told, so it has to describe the actual file.
return { status: "ok", config, score: scoreConfig(config, data) };
}
/**
* Replay a detected configuration over the file's data rows and count how many
* of them it reads.
*
* Deliberately NOT a rule of its own: the row is mapped by `mapRow`, the same
* pure function `parseFilesInternal` runs at import time, under the same
* column-level decimal arbitration. Two mappers would be the exact class of
* divergence this chantier exists to remove a score could then read 100 %
* while the import wrote different amounts.
*
* The row selection mirrors `parseFilesInternal` line for line, including the
* lone-empty-cell skip: a trailing blank line must not count as an unread row.
*/
function scoreConfig(
config: AutoDetectResult,
data: string[][]
): DetectionScore {
// `mapRow` reads no byte and therefore never looks at `encoding` — the
// content reached us already decoded. Naming it here only satisfies the type.
const format: ImportFormat = { ...config, encoding: "utf-8" };
const dataRows: string[][] = [];
const startIdx = config.skipLines + (config.hasHeader ? 1 : 0);
for (let i = startIdx; i < data.length; i++) {
const raw = data[i];
if (raw.length <= 1 && raw[0]?.trim() === "") continue;
dataRows.push(raw);
}
const decimalSeparators = detectAmountSeparators(dataRows, format);
let readRows = 0;
for (const raw of dataRows) {
try {
if (mapRow(raw, format, { decimalSeparators }).parsed) readRows++;
} catch {
// `mapRow` is documented not to throw; if it ever did, that is one
// unreadable row, not a detection that collapses on the whole file.
}
}
const totalRows = dataRows.length;
const ratio = totalRows === 0 ? 0 : readRows / totalRows;
return {
readRows,
totalRows,
ratio,
confident: totalRows > 0 && ratio >= CONFIDENCE_THRESHOLD,
}; };
} }