import { useReducer, useCallback, useEffect } from "react"; import { invoke } from "@tauri-apps/api/core"; import Papa from "papaparse"; import type { ImportWizardStep, ImportFolderScan, ScannedSource, ScannedFile, SourceConfig, ParsedRow, DuplicateCheckResult, ImportReport, ImportSource, ImportConfigTemplate, } from "../shared/types"; import { getImportFolder, setImportFolder, } from "../services/userPreferenceService"; import { getAllSources, getSourceByName, createSource, updateSource, } from "../services/importSourceService"; import { existsByHash, createImportedFile, updateFileStatus, getFilesBySourceId, } from "../services/importedFileService"; import { insertBatch, findDuplicates, } from "../services/transactionService"; import { categorizeBatch } from "../services/categorizationService"; import { notifyDataChanged } from "../services/dataEvents"; import { getAllTemplates, createTemplate, updateTemplate, deleteTemplate as deleteTemplateService, } from "../services/importConfigTemplateService"; import { preprocessQuotedCSV, detectImportFormat as runAutoDetect, type DetectionScore, } from "../utils/csvAutoDetect"; import { buildHeaderSignature, detectHeaderDrift, type BankSignatureId, type HeaderDriftEntry, } from "../utils/bankSignatures"; import { detectAmountSeparators, flipSignFormat, formatFromRow, formatToRow, ImportFormatError, mapRow, ROW_ERROR_KEYS, } from "../utils/importFormat"; import { logInfo } from "../services/logService"; /** Error text for the banner: an i18n key when we have one, the raw message otherwise. */ function errorMessage(e: unknown): string { if (e instanceof ImportFormatError) return e.i18nKey; return e instanceof Error ? e.message : String(e); } export interface WizardState { step: ImportWizardStep; importFolder: string | null; /** * The stored import folder has been read at mount, and scanned when there is * one (#384, #398). Until then `importFolder` is null for lack of reading, * not for lack of a folder: the empty state and the step card wait for it, * or the "no folder" variant would flash. Set once, success or failure, and * kept by `RESET`, which the mount read never runs again to restore. */ importFolderLoaded: boolean; scannedSources: ScannedSource[]; /** * Names of the CSV/TXT files placed directly at the root of the import * folder, which no source reads (#427). Set with `scannedSources` by the * same action, so the warning and the source list always come from the same * scan; kept by `RESET` like them. Shown on screen only, never logged: a * statement name often carries an account number. */ rootFiles: string[]; selectedSource: ScannedSource | null; selectedFiles: ScannedFile[]; sourceConfig: SourceConfig; existingSource: ImportSource | null; parsedPreview: ParsedRow[]; previewHeaders: string[]; duplicateResult: DuplicateCheckResult | null; excludedDuplicateIndices: Set; importReport: ImportReport | null; importProgress: { current: number; total: number; file: string }; isLoading: boolean; error: string | null; configuredSourceNames: Set; importedFilesBySource: Map>; configTemplates: ImportConfigTemplate[]; selectedTemplateId: number | null; /** * Result of the LAST detection run on the selected source, or null when none * ran — a source opened on its stored format, which is never re-detected. */ detectionScore: DetectionScore | null; /** * Bank whose documented layout the last detection recognised (#330). Tied to * `detectionScore`: it is set with it and cleared with it, so a badge naming * a bank can never outlive the detection that named it. */ detectedBank: BankSignatureId | null; /** * How the header row of the files being imported differs from the one the * source recorded at its last successful import — null when there is nothing * to report, which includes every source that has no stored signature. */ formatDrift: HeaderDriftEntry[] | null; /** * The configuration re-detected on the drifted file, offered by the panel as * "adopt". Null when detection could not read the new shape either: the drift * is still worth reporting, there is just nothing to adopt. */ driftConfig: SourceConfig | null; /** * Names of the sample folders the last "create a sample folder" action wrote * (#356), shown in the success banner. Null when there is nothing to report: * initially, after the banner is dismissed, and whenever the import folder * changes — a banner naming folders of another folder would be a lie. */ sampleCreated: string[] | null; } export type WizardAction = | { type: "SET_LOADING"; payload: boolean } | { type: "SET_ERROR"; payload: string | null } | { type: "SET_STEP"; payload: ImportWizardStep } | { type: "SET_IMPORT_FOLDER"; payload: string | null } | { type: "SET_IMPORT_FOLDER_LOADED" } | { type: "SET_SCANNED_SOURCES"; payload: ImportFolderScan } | { type: "SET_SELECTED_SOURCE"; payload: ScannedSource } | { type: "SET_SELECTED_FILES"; payload: ScannedFile[] } | { type: "SET_SOURCE_CONFIG"; payload: SourceConfig } | { type: "SET_EXISTING_SOURCE"; payload: ImportSource | null } | { type: "SET_PARSED_PREVIEW"; payload: { rows: ParsedRow[]; headers: string[] } } | { type: "SET_DUPLICATE_RESULT"; payload: DuplicateCheckResult } | { type: "TOGGLE_DUPLICATE_ROW"; payload: number } | { type: "SET_SKIP_ALL_DUPLICATES"; payload: boolean } | { type: "SET_IMPORT_REPORT"; payload: ImportReport } | { type: "SET_IMPORT_PROGRESS"; payload: { current: number; total: number; file: string } } | { type: "SET_CONFIGURED_SOURCES"; payload: { names: Set; files: Map> } } | { type: "SET_CONFIG_TEMPLATES"; payload: ImportConfigTemplate[] } | { type: "SET_SELECTED_TEMPLATE_ID"; payload: number | null } | { type: "SET_DETECTION_SCORE"; payload: DetectionScore | null } | { type: "SET_DETECTED_BANK"; payload: BankSignatureId | null } | { type: "SET_FORMAT_DRIFT"; payload: { entries: HeaderDriftEntry[]; config: SourceConfig | null } | null; } | { type: "SET_SAMPLE_CREATED"; payload: string[] | null } | { type: "RESET" }; const defaultConfig: SourceConfig = { name: "", delimiter: ";", encoding: "utf-8", dateFormat: "DD/MM/YYYY", skipLines: 0, columnMapping: { date: 0, description: 1, amount: 2 }, amountMode: "single", signConvention: "negative_expense", hasHeader: true, }; const initialState: WizardState = { step: "source-list", importFolder: null, importFolderLoaded: false, scannedSources: [], rootFiles: [], selectedSource: null, selectedFiles: [], sourceConfig: { ...defaultConfig }, existingSource: null, parsedPreview: [], previewHeaders: [], duplicateResult: null, excludedDuplicateIndices: new Set(), importReport: null, importProgress: { current: 0, total: 0, file: "" }, isLoading: false, error: null, configuredSourceNames: new Set(), importedFilesBySource: new Map(), configTemplates: [], selectedTemplateId: null, detectionScore: null, detectedBank: null, formatDrift: null, driftConfig: null, sampleCreated: null, }; /** Exported for the unit tests (#356): the hook is driven through this reducer. */ export const initialWizardState: WizardState = initialState; export function wizardReducer(state: WizardState, action: WizardAction): WizardState { switch (action.type) { case "SET_LOADING": return { ...state, isLoading: action.payload }; case "SET_ERROR": return { ...state, error: action.payload, isLoading: false }; case "SET_STEP": return { ...state, step: action.payload }; case "SET_IMPORT_FOLDER": // The sample-folder banner describes the previous folder: drop it. return { ...state, importFolder: action.payload, sampleCreated: null }; case "SET_IMPORT_FOLDER_LOADED": return { ...state, importFolderLoaded: true }; case "SET_SCANNED_SOURCES": return { ...state, scannedSources: action.payload.sources, rootFiles: action.payload.root_files, isLoading: false, }; case "SET_SELECTED_SOURCE": // The score belongs to the source it was measured on. Carrying it over // would let a source opened on its stored format inherit the confidence // of the previous one, which is worse than showing nothing. The bank // badge and the drift report are scoped the same way, for the same // reason — both describe one source's files. return { ...state, selectedSource: action.payload, detectionScore: null, detectedBank: null, formatDrift: null, driftConfig: null, }; case "SET_SELECTED_FILES": return { ...state, selectedFiles: action.payload }; case "SET_SOURCE_CONFIG": return { ...state, sourceConfig: action.payload }; case "SET_EXISTING_SOURCE": return { ...state, existingSource: action.payload }; case "SET_PARSED_PREVIEW": return { ...state, parsedPreview: action.payload.rows, previewHeaders: action.payload.headers, isLoading: false, }; case "SET_DUPLICATE_RESULT": return { ...state, duplicateResult: action.payload, excludedDuplicateIndices: new Set(action.payload.duplicateRows.map((d) => d.rowIndex)), isLoading: false, }; case "TOGGLE_DUPLICATE_ROW": { const next = new Set(state.excludedDuplicateIndices); if (next.has(action.payload)) { next.delete(action.payload); } else { next.add(action.payload); } return { ...state, excludedDuplicateIndices: next }; } case "SET_SKIP_ALL_DUPLICATES": return { ...state, excludedDuplicateIndices: action.payload ? new Set(state.duplicateResult?.duplicateRows.map((d) => d.rowIndex) ?? []) : new Set(), }; case "SET_IMPORT_REPORT": return { ...state, importReport: action.payload, isLoading: false }; case "SET_IMPORT_PROGRESS": return { ...state, importProgress: action.payload }; case "SET_CONFIGURED_SOURCES": return { ...state, configuredSourceNames: action.payload.names, importedFilesBySource: action.payload.files, }; case "SET_CONFIG_TEMPLATES": return { ...state, configTemplates: action.payload }; case "SET_SELECTED_TEMPLATE_ID": return { ...state, selectedTemplateId: action.payload }; case "SET_DETECTION_SCORE": // Clearing the score clears the bank with it, in ONE place: the three // sites that drop the score (a hand edit, a template, a sign flip) then // cannot leave "Format Desjardins reconnu" standing over a format // detection never produced. A bank is only ever set by dispatching // `SET_DETECTED_BANK` right after a non-null score. return { ...state, detectionScore: action.payload, detectedBank: action.payload === null ? null : state.detectedBank, }; case "SET_DETECTED_BANK": return { ...state, detectedBank: action.payload }; case "SET_FORMAT_DRIFT": return { ...state, formatDrift: action.payload?.entries ?? null, driftConfig: action.payload?.config ?? null, }; case "SET_SAMPLE_CREATED": return { ...state, sampleCreated: action.payload }; case "RESET": // Rebuilt from `initialState`, minus what the page read once and keeps: // the folder, whether it was read (#384: the mount read does not run // again, so "Cancel" or "Done" would otherwise hide the empty state and // the step card for good), its scan (sources and root files, #427), the // configured sources and the templates. return { ...initialState, importFolder: state.importFolder, importFolderLoaded: state.importFolderLoaded, scannedSources: state.scannedSources, rootFiles: state.rootFiles, configuredSourceNames: state.configuredSourceNames, importedFilesBySource: state.importedFilesBySource, configTemplates: state.configTemplates, }; default: return state; } } /** * The webview's LOCAL calendar date as `YYYY-MM-DD` (#356). Built from the * local getters, never `toISOString()`, which is UTC and would hand the Rust * side yesterday's date to anyone west of Greenwich in the evening. */ export function formatLocalDate(date: Date): string { const y = date.getFullYear(); const m = String(date.getMonth() + 1).padStart(2, "0"); const d = String(date.getDate()).padStart(2, "0"); return `${y}-${m}-${d}`; } export function useImportWizard() { const [state, dispatch] = useReducer(wizardReducer, initialState); // Load import folder on mount useEffect(() => { (async () => { try { const folder = await getImportFolder(); dispatch({ type: "SET_IMPORT_FOLDER", payload: folder }); if (folder) { await scanFolderInternal(folder); } } catch { // No folder configured yet } finally { // Read, and scanned when there is a folder (#384): the empty state and // the step card can tell "no folder" from "not read yet". After the // scan, so the "no source" variant does not flash either. dispatch({ type: "SET_IMPORT_FOLDER_LOADED" }); } })(); }, []); // eslint-disable-line react-hooks/exhaustive-deps const loadConfiguredSources = useCallback(async () => { const sources = await getAllSources(); const names = new Set(sources.map((s) => s.name)); const files = new Map>(); for (const source of sources) { const imported = await getFilesBySourceId(source.id); files.set( source.name, new Set(imported.map((f) => f.filename)) ); } dispatch({ type: "SET_CONFIGURED_SOURCES", payload: { names, files } }); const templates = await getAllTemplates(); dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates }); }, []); const scanFolderInternal = useCallback( async (folder: string) => { dispatch({ type: "SET_LOADING", payload: true }); try { // The root file names stay out of the logs (#427): on screen only. const scan = await invoke("scan_import_folder", { folderPath: folder, }); dispatch({ type: "SET_SCANNED_SOURCES", payload: scan }); await loadConfiguredSources(); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [loadConfiguredSources] ); const browseFolder = useCallback(async () => { try { const folder = await invoke("pick_folder"); if (folder) { await setImportFolder(folder); dispatch({ type: "SET_IMPORT_FOLDER", payload: folder }); await scanFolderInternal(folder); } } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [scanFolderInternal]); const refreshFolder = useCallback(async () => { if (state.importFolder) { await scanFolderInternal(state.importFolder); } }, [state.importFolder, scanFolderInternal]); /** * Write the two sample bank folders into the import folder (#356), rescan * it so their cards show up without another click, then raise the success * banner. The Rust command refuses when either folder already exists and * writes nothing; its message lands in the regular error banner. Only the * folder NAMES reach the logs: the logs travel with a feedback report, the * absolute path of the import folder must not. */ const createSampleFolders = useCallback(async () => { const folder = state.importFolder; if (!folder) return; dispatch({ type: "SET_LOADING", payload: true }); dispatch({ type: "SET_ERROR", payload: null }); try { const names = await invoke("create_sample_import_folders", { importFolder: folder, today: formatLocalDate(new Date()), }); logInfo(`[import] Sample folders created: ${names.join(", ")}`); await scanFolderInternal(folder); dispatch({ type: "SET_SAMPLE_CREATED", payload: names }); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [state.importFolder, scanFolderInternal]); const dismissSampleCreated = useCallback(() => { dispatch({ type: "SET_SAMPLE_CREATED", payload: null }); }, []); /** * Run detection on one file and report the outcome. Returns the detected * format merged onto `base`, or null when detection produced no usable * configuration (the reason is dispatched as the page error). * * THE single detection path, shared by the two things that start one: the * magic-wand button and the automatic run on a source that has never been * configured. Everything it needs is a parameter, so it never reads a piece of * `state` its caller has just dispatched but not yet re-rendered — which is * exactly why the automatic run could not simply call the button's callback. * * `base` keeps the fields detection does not decide (the source name, the * encoding), and the spread of `outcome.config` writes every field it does — * naming them one by one is how a field gets silently dropped (#324). */ const detectFormatForFile = useCallback( async (filePath: string, base: SourceConfig): Promise => { const content = await invoke("read_file_content", { filePath, encoding: base.encoding, }); const outcome = runAutoDetect(content); if (outcome.status !== "ok") { // A refused format states WHY (#327): "I cannot read this file" and // "this file is a shape this version would import backwards" are not // the same news, and only the second one is the app's own limitation. // `SET_ERROR` clears the loading flag on its own. dispatch({ type: "SET_DETECTION_SCORE", payload: null }); dispatch({ type: "SET_ERROR", payload: outcome.status === "rejected" ? outcome.reason : "import.errors.autoDetectFailed", }); return null; } dispatch({ type: "SET_DETECTION_SCORE", payload: outcome.score }); // Always dispatched, including the `null` of an unknown file: a bank left // over from the previously detected file would name the wrong bank. dispatch({ type: "SET_DETECTED_BANK", payload: outcome.bank }); return { ...base, ...outcome.config }; }, [] ); const selectSource = useCallback( async (source: ScannedSource) => { // The banner is page-wide: an error left by the previous source must not // survive into this one. Cleared here, before anything that may set a new // one — a stored format that will not decode, a detection that refuses. dispatch({ type: "SET_ERROR", payload: null }); // Sort files: new files first, then already-imported const importedNames = state.importedFilesBySource.get(source.folder_name); const sorted = [...source.files].sort((a, b) => { const aImported = importedNames?.has(a.filename) ?? false; const bImported = importedNames?.has(b.filename) ?? false; if (aImported !== bImported) return aImported ? 1 : -1; return a.filename.localeCompare(b.filename); }); const sortedSource = { ...source, files: sorted }; // Pre-select only new files const newFiles = sorted.filter((f) => !importedNames?.has(f.filename)); dispatch({ type: "SET_SELECTED_SOURCE", payload: sortedSource }); dispatch({ type: "SET_SELECTED_FILES", payload: newFiles }); try { // Check if this source already has config in DB const existing = await getSourceByName(source.folder_name); dispatch({ type: "SET_EXISTING_SOURCE", payload: existing }); // Provenance of the stored format, not the format itself: the template // is never re-read, it is only shown as the source's origin. Blindly // resetting it to null used to make an already-linked source look // unconfigured. dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: existing?.template_id ?? null, }); let activeDelimiter = defaultConfig.delimiter; let activeEncoding = "utf-8"; let activeSkipLines = 0; let activeHasHeader = true; // Restore the format as it was SAVED. Nothing is re-inferred and // nothing is defaulted here — the amount mode and the sign convention // are columns now (v17), and reading them is the whole point of #324. let restored: SourceConfig | null = null; if (existing) { try { restored = { name: existing.name, ...formatFromRow(existing) }; } catch (e) { // A stored format we cannot decode is REPORTED, not quietly swapped // for a plausible default — that substitution is how amounts got // flipped. The wizard then opens on a fresh configuration, because // "reconfigure this source" has to be an action the user can take. dispatch({ type: "SET_ERROR", payload: errorMessage(e) }); } } if (restored) { dispatch({ type: "SET_SOURCE_CONFIG", payload: restored }); activeDelimiter = restored.delimiter; activeEncoding = restored.encoding; activeSkipLines = restored.skipLines; activeHasHeader = restored.hasHeader; } else { // Auto-detect encoding for first file if (source.files.length > 0) { try { activeEncoding = await invoke("detect_encoding", { filePath: source.files[0].file_path, }); } catch { // fallback to utf-8 } } const fresh: SourceConfig = { ...defaultConfig, name: source.folder_name, encoding: activeEncoding, }; // Detection fires on its own HERE and nowhere else: the arm reached // when the source has no stored format at all. `defaultConfig` is // `;` + `DD/MM/YYYY` + columns 0/1/2 — plausible enough to import a // whole file wrong rather than fail, so leaving a never-configured // source sitting on it is the defect (#328). // // The condition is `!existing`, not `!restored`: a source whose stored // format will not decode already carries an explicit error, and // re-detecting over it would replace that message with a silent guess. // Its wand button is still there. let active = fresh; if (!existing && source.files.length > 0) { const detected = await detectFormatForFile( source.files[0].file_path, fresh ); if (detected) active = detected; } dispatch({ type: "SET_SOURCE_CONFIG", payload: active }); activeDelimiter = active.delimiter; activeSkipLines = active.skipLines; activeHasHeader = active.hasHeader; } // Load preview headers from first file if (source.files.length > 0) { await loadHeadersWithConfig( source.files[0].file_path, activeDelimiter, activeEncoding, activeSkipLines, activeHasHeader ); } dispatch({ type: "SET_STEP", payload: "source-config" }); } catch (e) { dispatch({ type: "SET_ERROR", payload: errorMessage(e) }); } }, [state.importedFilesBySource] // eslint-disable-line react-hooks/exhaustive-deps ); const loadHeadersWithConfig = useCallback( async (filePath: string, delimiter: string, encoding: string, skipLines: number, hasHeader: boolean) => { try { const preview = await invoke("get_file_preview", { filePath, encoding, maxLines: skipLines + 5, }); const preprocessed = preprocessQuotedCSV(preview); const parsed = Papa.parse(preprocessed, { delimiter, skipEmptyLines: true }); const data = parsed.data as string[][]; const headerRow = hasHeader && data.length > skipLines ? skipLines : -1; if (headerRow >= 0 && data[headerRow]) { dispatch({ type: "SET_PARSED_PREVIEW", payload: { rows: [], headers: data[headerRow].map((h) => h.trim()), }, }); } else if (data.length > 0) { // No header row — generate column indices as headers const firstDataRow = data[skipLines] || data[0]; dispatch({ type: "SET_PARSED_PREVIEW", payload: { rows: [], headers: firstDataRow.map((_, i) => `Col ${i}`), }, }); } } catch { // ignore preview errors } }, [] ); const updateConfig = useCallback( (config: SourceConfig) => { dispatch({ type: "SET_SOURCE_CONFIG", payload: config }); // The score measures the format DETECTION produced. The moment any of // those eight fields is edited by hand, the number stops describing what // the wizard would read — a banner still claiming "147 of 150 rows" over // a mapping nobody measured is the same misinformation this chantier is // removing. Comparing through the codec keeps a rename (the ninth field, // which changes nothing about reading) from clearing it. if ( JSON.stringify(formatToRow(config)) !== JSON.stringify(formatToRow(state.sourceConfig)) ) { dispatch({ type: "SET_DETECTION_SCORE", payload: null }); } // Reload headers when delimiter, encoding, skipLines, or hasHeader changes if (state.selectedFiles.length > 0) { loadHeadersWithConfig( state.selectedFiles[0].file_path, config.delimiter, config.encoding, config.skipLines, config.hasHeader ); } }, [state.selectedFiles, state.sourceConfig, loadHeadersWithConfig] ); const toggleFile = useCallback( (file: ScannedFile) => { const exists = state.selectedFiles.some( (f) => f.file_path === file.file_path ); if (exists) { dispatch({ type: "SET_SELECTED_FILES", payload: state.selectedFiles.filter( (f) => f.file_path !== file.file_path ), }); } else { dispatch({ type: "SET_SELECTED_FILES", payload: [...state.selectedFiles, file], }); } }, [state.selectedFiles] ); const selectAllFiles = useCallback(() => { if (state.selectedSource) { const importedNames = state.importedFilesBySource.get(state.selectedSource.folder_name); const newFiles = importedNames ? state.selectedSource.files.filter((f) => !importedNames.has(f.filename)) : state.selectedSource.files; dispatch({ type: "SET_SELECTED_FILES", payload: newFiles, }); } }, [state.selectedSource, state.importedFilesBySource]); // Internal helper: parses selected files and returns rows + headers. // // `configOverride` exists for the sign flip: it re-parses under a format that // has just been dispatched and is therefore not yet readable in `state`. // Reading the stale one would redisplay the exact table the user asked to // correct, which reads as "the button did nothing". const parseFilesInternal = useCallback(async ( configOverride?: SourceConfig ): Promise<{ rows: ParsedRow[]; headers: string[] }> => { const config = configOverride ?? state.sourceConfig; const allRows: ParsedRow[] = []; let headers: string[] = []; for (const file of state.selectedFiles) { const content = await invoke("read_file_content", { filePath: file.file_path, encoding: config.encoding, }); const preprocessed = preprocessQuotedCSV(content); const parsed = Papa.parse(preprocessed, { delimiter: config.delimiter, skipEmptyLines: true, }); const data = parsed.data as string[][]; const startIdx = config.skipLines + (config.hasHeader ? 1 : 0); if (config.hasHeader && data.length > config.skipLines) { headers = data[config.skipLines].map((h) => h.trim()); } else if (!config.hasHeader && headers.length === 0 && data.length > config.skipLines) { const firstDataRow = data[config.skipLines]; headers = firstDataRow.map((_, i) => `Col ${i}`); } // Data rows of THIS file, kept aside so the decimal separator of each // amount column is arbitrated over the whole column before any row is // read. `1.234` alone is ambiguous; the column is not. const dataRows: string[][] = []; for (let i = startIdx; i < data.length; i++) { const raw = data[i]; if (raw.length <= 1 && raw[0]?.trim() === "") continue; dataRows.push(raw); } const decimalSeparators = detectAmountSeparators(dataRows, config); for (const raw of dataRows) { try { allRows.push( mapRow(raw, config, { rowIndex: allRows.length, sourceFilename: file.filename, decimalSeparators, }) ); } catch { allRows.push({ rowIndex: allRows.length, raw, parsed: null, error: ROW_ERROR_KEYS.parseError, sourceFilename: file.filename, }); } } } return { rows: allRows, headers }; }, [state.selectedFiles, state.sourceConfig]); // Parse the selected files and STOP at the preview. // // Every import goes through that step now (#329). It used to be an optional // modal nobody had to open, next to a button that went straight to the // duplicate check — so a file read under a wrong convention reached the // database without anyone ever seeing a total. The step is unconditional on // purpose: the detection score does not gate it, because a perfect score says // nothing about the sign of what was read. const parseAndPreview = useCallback(async () => { if (state.selectedFiles.length === 0) return; dispatch({ type: "SET_LOADING", payload: true }); dispatch({ type: "SET_ERROR", payload: null }); try { const result = await parseFilesInternal(); dispatch({ type: "SET_PARSED_PREVIEW", payload: result, }); // Format drift (#330). Compared HERE and not at the configuration step // because this is where the header of the files actually being imported // is known — `previewHeaders` at the configuration step is read from the // first file alone, under a delimiter the user may still be editing. // // `hasHeader` gates the whole thing: a headerless file has `Col 0`, // `Col 1` … for headers, which is not a signature and must never be // compared to one. const drift = state.sourceConfig.hasHeader ? detectHeaderDrift( state.existingSource?.header_signature, result.headers ) : null; if (drift) { // Re-detect on the drifted file so the panel has something to offer. // This is the THIRD caller of the single detection path, and it goes // through it rather than around it on purpose: a second detector is the // divergence class this whole chantier removes. A file the re-detection // cannot read leaves `null` — the drift is still reported, with nothing // to adopt. const reDetected = await detectFormatForFile( state.selectedFiles[0].file_path, state.sourceConfig ); // That run measured the RE-DETECTED format, which is not the one the // wizard is using — it is only what the panel offers. Leaving its score // and its bank badge behind would put "Format reconnu — 6 of 6 rows // read" next to the stored mapping the moment the user steps back to // the configuration. Same rule as the sign flip: a badge describes the // format in use or it describes nothing. dispatch({ type: "SET_DETECTION_SCORE", payload: null }); dispatch({ type: "SET_FORMAT_DRIFT", payload: { entries: drift, config: reDetected }, }); } else { dispatch({ type: "SET_FORMAT_DRIFT", payload: null }); } dispatch({ type: "SET_STEP", payload: "file-preview" }); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [ state.selectedFiles, state.sourceConfig, state.existingSource, parseFilesInternal, detectFormatForFile, ]); // Internal helper: runs duplicate checking against parsed rows. // // Deliberately writes NOTHING. The source config used to be saved here, so an // import abandoned at the duplicate step still left a — possibly wrong — // configuration behind. It is written by `executeImport` now (#324). const checkDuplicatesInternal = useCallback(async (parsedRows: ParsedRow[]) => { // Check file-level duplicates (check ALL selected files, not just the first) let fileAlreadyImported = false; let existingFileId: number | undefined; for (const file of state.selectedFiles) { const hash = await invoke("hash_file", { filePath: file.file_path, }); const existing = await existsByHash(hash); if (existing) { fileAlreadyImported = true; existingFileId = existing.id; break; } } // Check row-level duplicates against DB const validRows = parsedRows.filter((r) => r.parsed); const duplicateMatches = await findDuplicates( validRows.map((r) => ({ date: r.parsed!.date, description: r.parsed!.description, amount: r.parsed!.amount, })) ); const dbDuplicateIndices = new Set(duplicateMatches.map((d) => d.rowIndex)); const duplicateRows = duplicateMatches.map((d) => ({ rowIndex: d.rowIndex, date: d.date, description: d.description, amount: d.amount, existingTransactionId: d.existingTransactionId, })); // Cross-file duplicate detection: find rows that appear in multiple source files const seenKeys = new Map(); // key → first-seen validRows index for (let i = 0; i < validRows.length; i++) { if (dbDuplicateIndices.has(i)) continue; // already flagged as DB duplicate const row = validRows[i]; const key = `${row.parsed!.date}|${row.parsed!.description}|${row.parsed!.amount}`; const firstIdx = seenKeys.get(key); if (firstIdx !== undefined) { // Only flag as cross-file duplicate if rows come from different files if (validRows[firstIdx].sourceFilename !== row.sourceFilename) { duplicateRows.push({ rowIndex: i, date: row.parsed!.date, description: row.parsed!.description, amount: row.parsed!.amount, existingTransactionId: -1, // signals "within batch" in the UI }); dbDuplicateIndices.add(i); } } else { seenKeys.set(key, i); } } const newRows = validRows.filter( (_, i) => !dbDuplicateIndices.has(i) ); dispatch({ type: "SET_DUPLICATE_RESULT", payload: { fileAlreadyImported, existingFileId, duplicateRows, newRows, }, }); dispatch({ type: "SET_STEP", payload: "duplicate-check" }); }, [state.selectedFiles]); // Leave the preview for the duplicate check, on the rows it just displayed. // // Dead code until #329: the wizard jumped from the configuration straight to // the duplicates, re-parsing on the way. It is the preview step's "next" // button now, so the rows the user validated are the rows that get checked — // no second parse can quietly read the file differently in between. const checkDuplicates = useCallback(async () => { dispatch({ type: "SET_LOADING", payload: true }); dispatch({ type: "SET_ERROR", payload: null }); try { await checkDuplicatesInternal(state.parsedPreview); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [state.parsedPreview, checkDuplicatesInternal]); /** * Read the file the other way round, and remember it. * * The correction lands on the CONFIGURATION (`flipSignFormat`, which knows * that debit/credit flips by swapping columns rather than by toggling a * convention `mapRow` ignores there), so it is persisted with the source at * import time and the next file from that bank reads right on its own. * * Headers are deliberately NOT reloaded: a flip touches neither delimiter, * encoding, skipped lines nor header flag, and `loadHeadersWithConfig` * dispatches an empty row list — racing it against this re-parse is a coin * toss between the corrected table and a blank one. */ const flipSignConvention = useCallback(async () => { const flipped: SourceConfig = { ...state.sourceConfig, ...flipSignFormat(state.sourceConfig), }; dispatch({ type: "SET_SOURCE_CONFIG", payload: flipped }); // Detection did not produce this format. Keeping its badge would vouch for // a configuration it never measured — the same rule as a manual edit. dispatch({ type: "SET_DETECTION_SCORE", payload: null }); dispatch({ type: "SET_LOADING", payload: true }); dispatch({ type: "SET_ERROR", payload: null }); try { const result = await parseFilesInternal(flipped); dispatch({ type: "SET_PARSED_PREVIEW", payload: result, }); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [state.sourceConfig, parseFilesInternal]); const executeImport = useCallback(async () => { if (!state.duplicateResult || !state.selectedSource) return; dispatch({ type: "SET_STEP", payload: "importing" }); dispatch({ type: "SET_ERROR", payload: null }); try { const config = state.sourceConfig; // Persist the format — the ONLY write point. It happens here rather than // at the duplicate step so an abandoned import leaves no configuration // behind, and it goes through `formatToRow` so no field can be dropped. // It has to precede the file records, which carry a `source_id` FK. const formatRow = formatToRow(config); // The header row this import actually read, recorded so the NEXT one can // be compared against it (#330). It is not a format field and does not go // through the codec: it says what the file looked like, not how to read // it. A headerless file writes null — `previewHeaders` holds `Col 0`, // `Col 1` … there, and storing that would make every later import look // like drift. const headerSignature = buildHeaderSignature( config.hasHeader ? state.previewHeaders : null ); // A source is named after its subfolder (#426): the next import looks it // up, and its imported files, by `folder_name`, so a name typed during // configuration would make it look new. `config.name` is never written. const sourceName = state.selectedSource.folder_name; let sourceId: number; if (state.existingSource) { sourceId = state.existingSource.id; await updateSource(sourceId, { name: sourceName, ...formatRow, header_signature: headerSignature, template_id: state.selectedTemplateId, }); } else { sourceId = await createSource({ name: sourceName, ...formatRow, header_signature: headerSignature, template_id: state.selectedTemplateId, }); } // Determine rows to import: new rows + non-excluded duplicates const includedDuplicates = state.duplicateResult.duplicateRows .filter((d) => !state.excludedDuplicateIndices.has(d.rowIndex)); const rowsToImport = [ ...state.duplicateResult.newRows, ...state.parsedPreview.filter( (r) => r.parsed && includedDuplicates.some((d) => d.rowIndex === r.rowIndex) ), ]; const validRows = rowsToImport.filter((r) => r.parsed); const totalRows = validRows.length; dispatch({ type: "SET_IMPORT_PROGRESS", payload: { current: 0, total: totalRows, file: state.selectedFiles[0]?.filename || "" }, }); // Create one imported_files record per file const fileIdMap = new Map(); for (const file of state.selectedFiles) { const hash = await invoke("hash_file", { filePath: file.file_path, }); const rowCount = validRows.filter((r) => r.sourceFilename === file.filename).length; const fId = await createImportedFile({ source_id: sourceId, filename: file.filename, file_hash: hash, row_count: rowCount, status: "completed", }); fileIdMap.set(file.filename, fId); } // Auto-categorize const descriptions = validRows.map((r) => r.parsed!.description); const categorizations = await categorizeBatch(descriptions); let categorizedCount = 0; let uncategorizedCount = 0; const errors: Array<{ rowIndex: number; message: string }> = []; // Build transaction records const transactions = validRows.map((row, i) => { const cat = categorizations[i]; if (cat.category_id) { categorizedCount++; } else { uncategorizedCount++; } return { date: row.parsed!.date, description: row.parsed!.description, amount: row.parsed!.amount, source_id: sourceId, file_id: fileIdMap.get(row.sourceFilename || "") ?? 0, original_description: row.raw.join(config.delimiter), category_id: cat.category_id, supplier_id: cat.supplier_id, }; }); // Insert with progress let importedCount = 0; try { importedCount = await insertBatch(transactions, (inserted) => { const currentFile = validRows[inserted - 1]?.sourceFilename || ""; dispatch({ type: "SET_IMPORT_PROGRESS", payload: { current: inserted, total: totalRows, file: currentFile }, }); }); dispatch({ type: "SET_IMPORT_PROGRESS", payload: { current: importedCount, total: totalRows, file: "done" }, }); } catch (e) { // Update status on all file records on error for (const fId of fileIdMap.values()) { await updateFileStatus(fId, "error", 0, String(e)); } errors.push({ rowIndex: 0, message: e instanceof Error ? e.message : String(e), }); } // Count errors from parsing const parseErrors = state.parsedPreview.filter((r) => r.error); for (const err of parseErrors) { errors.push({ rowIndex: err.rowIndex, message: err.error || ROW_ERROR_KEYS.parseError, }); } const report: ImportReport = { totalRows: state.parsedPreview.length, importedCount, skippedDuplicates: state.excludedDuplicateIndices.size, errorCount: errors.length, categorizedCount, uncategorizedCount, errors, }; dispatch({ type: "SET_IMPORT_REPORT", payload: report }); dispatch({ type: "SET_STEP", payload: "report" }); // Refresh configured sources await loadConfiguredSources(); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); dispatch({ type: "SET_STEP", payload: "confirm" }); } finally { // One more announcement once the import is over, succeeded or not, so // the last re-read of the getting-started path sees its final state: // inserts done, file statuses settled (ADR 0022). notifyDataChanged(); } }, [ state.duplicateResult, state.selectedSource, state.sourceConfig, state.existingSource, state.selectedTemplateId, state.excludedDuplicateIndices, state.parsedPreview, state.selectedFiles, loadConfiguredSources, ]); const goToStep = useCallback((step: ImportWizardStep) => { dispatch({ type: "SET_STEP", payload: step }); }, []); const reset = useCallback(() => { dispatch({ type: "RESET" }); }, []); /** * Take the format re-detected on the drifted file, and re-read the preview * under it (#330). * * Same shape as the sign flip, and for the same reason: the adopted format is * PASSED to the parse instead of being read back from `state`, which has not * re-rendered yet. Reading the stale one would redisplay the table the user * just chose to replace. * * Nothing is written here. The adopted format reaches `import_sources` at * `executeImport` like every other configuration — an import abandoned on * this screen leaves the stored format exactly as it was. */ const adoptDriftFormat = useCallback(async () => { const adopted = state.driftConfig; if (!adopted) return; dispatch({ type: "SET_SOURCE_CONFIG", payload: adopted }); dispatch({ type: "SET_FORMAT_DRIFT", payload: null }); dispatch({ type: "SET_LOADING", payload: true }); dispatch({ type: "SET_ERROR", payload: null }); try { const result = await parseFilesInternal(adopted); dispatch({ type: "SET_PARSED_PREVIEW", payload: result }); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [state.driftConfig, parseFilesInternal]); /** * Keep the stored configuration and dismiss the panel. * * The rows on screen were already parsed under that configuration, so there * is nothing to re-read. The stored signature is still refreshed at import * time: the new header IS what this import read, whichever format read it. */ const keepCurrentFormat = useCallback(() => { dispatch({ type: "SET_FORMAT_DRIFT", payload: null }); }, []); const autoDetectConfig = useCallback(async () => { if (state.selectedFiles.length === 0) return; dispatch({ type: "SET_LOADING", payload: true }); dispatch({ type: "SET_ERROR", payload: null }); try { const filePath = state.selectedFiles[0].file_path; // Same path the automatic run takes, so the button REPLAYS detection // rather than running a second, drifting variant of it. const newConfig = await detectFormatForFile(filePath, state.sourceConfig); if (!newConfig) return; // reason already dispatched, loading already off dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig }); dispatch({ type: "SET_LOADING", payload: false }); // Refresh column headers with new config await loadHeadersWithConfig( filePath, newConfig.delimiter, newConfig.encoding, newConfig.skipLines, newConfig.hasHeader ); } catch (e) { dispatch({ type: "SET_ERROR", payload: e instanceof Error ? e.message : String(e), }); } }, [ state.selectedFiles, state.sourceConfig, detectFormatForFile, loadHeadersWithConfig, ]); const saveConfigAsTemplate = useCallback(async (name: string) => { await createTemplate({ name, ...formatToRow(state.sourceConfig) }); const templates = await getAllTemplates(); dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates }); }, [state.sourceConfig]); const applyConfigTemplate = useCallback((templateId: number) => { const template = state.configTemplates.find((t) => t.id === templateId); if (!template) return; let newConfig: SourceConfig; try { newConfig = { name: state.sourceConfig.name, ...formatFromRow(template), }; } catch (e) { dispatch({ type: "SET_ERROR", payload: errorMessage(e) }); return; } dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig }); // Applying a template COPIES its format onto the source. The id recorded // here is provenance only — the copy is what the next import reads. dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: templateId }); // The format is the template's now, so a score measured on the detected // one would be vouching for a configuration it never saw. dispatch({ type: "SET_DETECTION_SCORE", payload: null }); // Reload headers with new config if (state.selectedFiles.length > 0) { loadHeadersWithConfig( state.selectedFiles[0].file_path, newConfig.delimiter, newConfig.encoding, newConfig.skipLines, newConfig.hasHeader ); } }, [state.configTemplates, state.sourceConfig.name, state.selectedFiles, loadHeadersWithConfig]); const updateConfigTemplate = useCallback(async () => { if (!state.selectedTemplateId) return; const template = state.configTemplates.find((t) => t.id === state.selectedTemplateId); if (!template) return; // Writes to `import_config_templates` only: a source configured from this // template keeps its own eight columns and is untouched. await updateTemplate(state.selectedTemplateId, { name: template.name, ...formatToRow(state.sourceConfig), }); const templates = await getAllTemplates(); dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates }); }, [state.selectedTemplateId, state.configTemplates, state.sourceConfig]); const deleteConfigTemplate = useCallback(async (id: number) => { await deleteTemplateService(id); if (state.selectedTemplateId === id) { dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: null }); } const templates = await getAllTemplates(); dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates }); }, [state.selectedTemplateId]); return { state, browseFolder, refreshFolder, createSampleFolders, dismissSampleCreated, selectSource, updateConfig, toggleFile, selectAllFiles, parseAndPreview, checkDuplicates, flipSignConvention, executeImport, goToStep, reset, adoptDriftFormat, keepCurrentFormat, autoDetectConfig, saveConfigAsTemplate, applyConfigTemplate, updateConfigTemplate, deleteConfigTemplate, toggleDuplicateRow: (index: number) => dispatch({ type: "TOGGLE_DUPLICATE_ROW", payload: index }), setSkipAllDuplicates: (skipAll: boolean) => dispatch({ type: "SET_SKIP_ALL_DUPLICATES", payload: skipAll }), }; }