fix(import): persist the import format and restore it faithfully #335

Closed
maximus wants to merge 1 commit from issue-324-persist-import-format into issue-323-migration-v17
13 changed files with 1139 additions and 184 deletions

View file

@ -0,0 +1,262 @@
/**
* Import format save/reload integration (#324).
*
* The unit tests in `src/utils/importFormat.test.ts` prove the codec carries
* every field. This file proves the SQL on either side of it does too: a format
* written by `importSourceService` and read back is the same format, field by
* field. That is the test that would have caught the root bug before v17 the
* INSERT simply had no `amount_mode` / `sign_convention` column to write to, so
* a positive-expense source silently came back inverted on its second import.
*
* Like `balance-flow.test.ts`, real `tauri-plugin-sql` cannot be started outside
* the Tauri WebView, so the services run against an in-memory FakeDb that
* interprets the handful of statements they actually issue. It stores what it
* is given, so `has_header` lands as the 0/1 integer SQLite stores and comes
* back as one the exact shape the codec has to absorb.
*/
import { describe, it, expect, beforeEach, vi } from "vitest";
vi.mock("../services/db", () => {
const getDb = vi.fn();
return {
getDb,
withTransaction: vi.fn(async (fn: (db: unknown) => unknown) => fn(await getDb())),
};
});
import { getDb } from "../services/db";
import {
createSource,
getSourceByName,
updateSource,
} from "../services/importSourceService";
import {
createTemplate,
getAllTemplates,
updateTemplate,
} from "../services/importConfigTemplateService";
import { formatFromRow, formatToRow, FORMAT_FIELD_PAIRS } from "../utils/importFormat";
import type { ImportFormat, SourceConfig } from "../shared/types";
// ---------------------------------------------------------------------------
// FakeDb — a tiny interpreter for the statements these two services issue.
// ---------------------------------------------------------------------------
type Row = Record<string, unknown>;
function makeFakeDb() {
const tables: Record<string, Row[]> = {
import_sources: [],
import_config_templates: [],
};
let nextId = 1;
const insert = (sql: string, params: unknown[]) => {
const [, table, cols] =
/INSERT INTO (\w+) \(([^)]+)\)/.exec(sql) ?? [];
const columns = cols.split(",").map((c) => c.trim());
const row: Row = { id: nextId++ };
columns.forEach((c, i) => (row[c] = params[i]));
if (/ON CONFLICT\(name\) DO UPDATE/.test(sql)) {
const clash = tables[table].find((r) => r.name === row.name);
if (clash) {
// Mimic `excluded.*`: overwrite every listed column, keep the id.
columns.forEach((c, i) => (clash[c] = params[i]));
nextId--;
return { lastInsertId: 0, rowsAffected: 1 };
}
}
tables[table].push(row);
return { lastInsertId: row.id as number, rowsAffected: 1 };
};
const update = (sql: string, params: unknown[]) => {
const [, table, assignments] =
/UPDATE (\w+)\s+SET ([\s\S]+?)\s+WHERE id\s*=\s*\$(\d+)/.exec(sql) ?? [];
const id = params[params.length - 1];
const row = tables[table].find((r) => r.id === id);
if (!row) return { rowsAffected: 0 };
for (const [, column, index] of assignments.matchAll(
/(\w+)\s*=\s*\$(\d+)/g
)) {
row[column] = params[Number(index) - 1];
}
return { rowsAffected: 1 };
};
return {
tables,
execute: vi.fn(async (sql: string, params: unknown[] = []) => {
if (sql.trimStart().startsWith("INSERT")) return insert(sql, params);
if (sql.trimStart().startsWith("UPDATE")) return update(sql, params);
throw new Error(`FakeDb: unsupported statement ${sql.slice(0, 40)}`);
}),
select: vi.fn(async (sql: string, params: unknown[] = []) => {
const [, table] = /FROM (\w+)/.exec(sql) ?? [];
const rows = tables[table];
if (/WHERE name = \$1/.test(sql)) return rows.filter((r) => r.name === params[0]);
if (/WHERE id = \$1/.test(sql)) return rows.filter((r) => r.id === params[0]);
return [...rows].sort((a, b) => String(a.name).localeCompare(String(b.name)));
}),
};
}
let db: ReturnType<typeof makeFakeDb>;
beforeEach(() => {
db = makeFakeDb();
vi.mocked(getDb).mockResolvedValue(db as never);
});
// ---------------------------------------------------------------------------
// Fixtures — a credit-card statement: expenses are POSITIVE. This is the
// configuration the old code could not keep.
// ---------------------------------------------------------------------------
const CREDIT_CARD: SourceConfig = {
name: "Visa Desjardins",
delimiter: ",",
encoding: "windows-1252",
dateFormat: "YYYY-MM-DD",
skipLines: 2,
hasHeader: false,
columnMapping: { date: 1, description: 3, amount: 4 },
amountMode: "single",
signConvention: "positive_expense",
};
/** What `useImportWizard.selectSource` does on a stored source. */
async function reload(name: string): Promise<SourceConfig> {
const stored = await getSourceByName(name);
if (!stored) throw new Error(`source ${name} not found`);
return { name: stored.name, ...formatFromRow(stored) };
}
function expectSameFormat(actual: ImportFormat, expected: ImportFormat) {
for (const field of Object.keys(FORMAT_FIELD_PAIRS) as Array<keyof ImportFormat>) {
expect(actual[field], `field ${field}`).toEqual(expected[field]);
}
}
// ---------------------------------------------------------------------------
describe("configure -> save -> reload (#324)", () => {
it("returns the saved format field by field", async () => {
await createSource({ name: CREDIT_CARD.name, ...formatToRow(CREDIT_CARD) });
expectSameFormat(await reload(CREDIT_CARD.name), CREDIT_CARD);
});
it("keeps positive_expense on the second import and every one after", async () => {
await createSource({ name: CREDIT_CARD.name, ...formatToRow(CREDIT_CARD) });
// Three consecutive imports re-save what was reloaded, as the wizard does.
let current = await reload(CREDIT_CARD.name);
for (let i = 0; i < 3; i++) {
const id = (await getSourceByName(CREDIT_CARD.name))!.id;
await updateSource(id, { name: current.name, ...formatToRow(current) });
current = await reload(CREDIT_CARD.name);
}
expect(current.signConvention).toBe("positive_expense");
expectSameFormat(current, CREDIT_CARD);
});
it("stores has_header as the integer SQLite holds, not a JS boolean", async () => {
await createSource({ name: CREDIT_CARD.name, ...formatToRow(CREDIT_CARD) });
expect(db.tables.import_sources[0].has_header).toBe(0);
expect((await reload(CREDIT_CARD.name)).hasHeader).toBe(false);
});
it("keeps the chosen mode when the mapping does not betray it", async () => {
const debitCredit: SourceConfig = {
...CREDIT_CARD,
amountMode: "debit_credit",
// No debitAmount: the old restore re-inferred "single" from this shape.
columnMapping: { date: 1, description: 3, creditAmount: 5 },
};
await createSource({ name: debitCredit.name, ...formatToRow(debitCredit) });
expect((await reload(debitCredit.name)).amountMode).toBe("debit_credit");
});
it("updates the format in place when the source is re-configured", async () => {
const id = await createSource({
name: CREDIT_CARD.name,
...formatToRow(CREDIT_CARD),
});
const flipped: SourceConfig = {
...CREDIT_CARD,
signConvention: "negative_expense",
amountMode: "debit_credit",
columnMapping: { date: 1, description: 3, debitAmount: 4, creditAmount: 5 },
};
await updateSource(id, { name: flipped.name, ...formatToRow(flipped) });
expectSameFormat(await reload(CREDIT_CARD.name), flipped);
expect(db.tables.import_sources).toHaveLength(1);
});
});
describe("template_id is provenance, never format (#324)", () => {
async function seedLinkedPair() {
const templateId = await createTemplate({
name: "Desjardins carte",
...formatToRow(CREDIT_CARD),
});
await createSource({
name: CREDIT_CARD.name,
...formatToRow(CREDIT_CARD),
template_id: templateId,
});
return templateId;
}
it("records the template the source was configured from", async () => {
const templateId = await seedLinkedPair();
expect((await getSourceByName(CREDIT_CARD.name))!.template_id).toBe(templateId);
});
// The acceptance criterion: the eight source columns are authoritative, so a
// template edit cannot reach through the link and change how a source reads.
it("editing the template alters no linked source", async () => {
const templateId = await seedLinkedPair();
await updateTemplate(templateId, {
name: "Desjardins carte",
...formatToRow({
...CREDIT_CARD,
delimiter: "\t",
encoding: "utf-8",
dateFormat: "DD/MM/YYYY",
skipLines: 0,
hasHeader: true,
columnMapping: { date: 0, description: 1, debitAmount: 2, creditAmount: 3 },
amountMode: "debit_credit",
signConvention: "negative_expense",
}),
});
expectSameFormat(await reload(CREDIT_CARD.name), CREDIT_CARD);
// …and the template really did change, so the test is not vacuous.
const [stored] = await getAllTemplates();
expect(formatFromRow(stored).signConvention).toBe("negative_expense");
expect(formatFromRow(stored).amountMode).toBe("debit_credit");
});
it("survives a re-import that re-saves the format", async () => {
const templateId = await seedLinkedPair();
const id = (await getSourceByName(CREDIT_CARD.name))!.id;
const current = await reload(CREDIT_CARD.name);
await updateSource(id, {
name: current.name,
...formatToRow(current),
template_id: templateId,
});
expect((await getSourceByName(CREDIT_CARD.name))!.template_id).toBe(templateId);
});
});

View file

@ -1,12 +1,19 @@
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import type { ColumnMapping, AmountMode } from "../../shared/types"; import type { ColumnMapping, AmountMode } from "../../shared/types";
import { clearMappingForMode } from "../../utils/importFormat";
interface ColumnMappingEditorProps { interface ColumnMappingEditorProps {
headers: string[]; headers: string[];
mapping: ColumnMapping; mapping: ColumnMapping;
amountMode: AmountMode; amountMode: AmountMode;
onMappingChange: (mapping: ColumnMapping) => void; onMappingChange: (mapping: ColumnMapping) => void;
onAmountModeChange: (mode: AmountMode) => void; /**
* The mode carries the pruned mapping with it. Both have to land in a single
* state update: the parent's handlers each spread the same `config` prop, so
* two consecutive calls would see the same stale value and the second would
* overwrite the first.
*/
onAmountModeChange: (mode: AmountMode, mapping: ColumnMapping) => void;
} }
export default function ColumnMappingEditor({ export default function ColumnMappingEditor({
@ -18,6 +25,12 @@ export default function ColumnMappingEditor({
}: ColumnMappingEditorProps) { }: ColumnMappingEditorProps) {
const { t } = useTranslation(); const { t } = useTranslation();
// The mode is the source of truth for the mapping, not the reverse: leaving a
// mode drops its columns so a stale key cannot keep deciding how amounts are
// read (#324).
const selectMode = (mode: AmountMode) =>
onAmountModeChange(mode, clearMappingForMode(mapping, mode));
const columnOptions = headers.map((h, i) => ( const columnOptions = headers.map((h, i) => (
<option key={i} value={i}> <option key={i} value={i}>
{i}: {h} {i}: {h}
@ -79,7 +92,7 @@ export default function ColumnMappingEditor({
name="amountMode" name="amountMode"
value="single" value="single"
checked={amountMode === "single"} checked={amountMode === "single"}
onChange={() => onAmountModeChange("single")} onChange={() => selectMode("single")}
className="accent-[var(--primary)]" className="accent-[var(--primary)]"
/> />
{t("import.config.singleAmount")} {t("import.config.singleAmount")}
@ -90,7 +103,7 @@ export default function ColumnMappingEditor({
name="amountMode" name="amountMode"
value="debit_credit" value="debit_credit"
checked={amountMode === "debit_credit"} checked={amountMode === "debit_credit"}
onChange={() => onAmountModeChange("debit_credit")} onChange={() => selectMode("debit_credit")}
className="accent-[var(--primary)]" className="accent-[var(--primary)]"
/> />
{t("import.config.debitCredit")} {t("import.config.debitCredit")}

View file

@ -314,8 +314,8 @@ export default function SourceConfigPanel({
onMappingChange={(mapping: ColumnMapping) => onMappingChange={(mapping: ColumnMapping) =>
onConfigChange({ ...config, columnMapping: mapping }) onConfigChange({ ...config, columnMapping: mapping })
} }
onAmountModeChange={(mode: AmountMode) => onAmountModeChange={(mode: AmountMode, mapping: ColumnMapping) =>
onConfigChange({ ...config, amountMode: mode }) onConfigChange({ ...config, amountMode: mode, columnMapping: mapping })
} }
/> />
)} )}

View file

@ -69,6 +69,8 @@ const SOURCE_CHEQUING: ImportSource = {
column_mapping: "{}", column_mapping: "{}",
skip_lines: 0, skip_lines: 0,
has_header: true, has_header: true,
amount_mode: "single",
sign_convention: "negative_expense",
created_at: "2026-01-01", created_at: "2026-01-01",
updated_at: "2026-01-01", updated_at: "2026-01-01",
}; };

View file

@ -11,7 +11,6 @@ import type {
ImportReport, ImportReport,
ImportSource, ImportSource,
ImportConfigTemplate, ImportConfigTemplate,
ColumnMapping,
} from "../shared/types"; } from "../shared/types";
import { import {
getImportFolder, getImportFolder,
@ -46,6 +45,17 @@ import {
preprocessQuotedCSV, preprocessQuotedCSV,
autoDetectConfig as runAutoDetect, autoDetectConfig as runAutoDetect,
} from "../utils/csvAutoDetect"; } from "../utils/csvAutoDetect";
import {
formatFromRow,
formatToRow,
ImportFormatError,
} from "../utils/importFormat";
/** Error text for the banner: an i18n key when we have one, the raw message otherwise. */
function errorMessage(e: unknown): string {
if (e instanceof ImportFormatError) return e.i18nKey;
return e instanceof Error ? e.message : String(e);
}
interface WizardState { interface WizardState {
step: ImportWizardStep; step: ImportWizardStep;
@ -297,37 +307,48 @@ export function useImportWizard() {
dispatch({ type: "SET_SELECTED_SOURCE", payload: sortedSource }); dispatch({ type: "SET_SELECTED_SOURCE", payload: sortedSource });
dispatch({ type: "SET_SELECTED_FILES", payload: newFiles }); dispatch({ type: "SET_SELECTED_FILES", payload: newFiles });
dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: null });
try {
// Check if this source already has config in DB // Check if this source already has config in DB
const existing = await getSourceByName(source.folder_name); const existing = await getSourceByName(source.folder_name);
dispatch({ type: "SET_EXISTING_SOURCE", payload: existing }); dispatch({ type: "SET_EXISTING_SOURCE", payload: existing });
// Provenance of the stored format, not the format itself: the template
// is never re-read, it is only shown as the source's origin. Blindly
// resetting it to null used to make an already-linked source look
// unconfigured.
dispatch({
type: "SET_SELECTED_TEMPLATE_ID",
payload: existing?.template_id ?? null,
});
let activeDelimiter = defaultConfig.delimiter; let activeDelimiter = defaultConfig.delimiter;
let activeEncoding = "utf-8"; let activeEncoding = "utf-8";
let activeSkipLines = 0; let activeSkipLines = 0;
let activeHasHeader = true; let activeHasHeader = true;
// Restore the format as it was SAVED. Nothing is re-inferred and
// nothing is defaulted here — the amount mode and the sign convention
// are columns now (v17), and reading them is the whole point of #324.
let restored: SourceConfig | null = null;
if (existing) { if (existing) {
// Restore config from DB try {
const mapping = JSON.parse(existing.column_mapping) as ColumnMapping; restored = { name: existing.name, ...formatFromRow(existing) };
const config: SourceConfig = { } catch (e) {
name: existing.name, // A stored format we cannot decode is REPORTED, not quietly swapped
delimiter: existing.delimiter, // for a plausible default — that substitution is how amounts got
encoding: existing.encoding, // flipped. The wizard then opens on a fresh configuration, because
dateFormat: existing.date_format, // "reconfigure this source" has to be an action the user can take.
skipLines: existing.skip_lines, dispatch({ type: "SET_ERROR", payload: errorMessage(e) });
columnMapping: mapping, }
amountMode: }
mapping.debitAmount !== undefined ? "debit_credit" : "single",
signConvention: "negative_expense", if (restored) {
hasHeader: !!existing.has_header, dispatch({ type: "SET_SOURCE_CONFIG", payload: restored });
}; activeDelimiter = restored.delimiter;
dispatch({ type: "SET_SOURCE_CONFIG", payload: config }); activeEncoding = restored.encoding;
activeDelimiter = existing.delimiter; activeSkipLines = restored.skipLines;
activeEncoding = existing.encoding; activeHasHeader = restored.hasHeader;
activeSkipLines = existing.skip_lines;
activeHasHeader = !!existing.has_header;
} else { } else {
// Auto-detect encoding for first file // Auto-detect encoding for first file
if (source.files.length > 0) { if (source.files.length > 0) {
@ -362,6 +383,9 @@ export function useImportWizard() {
} }
dispatch({ type: "SET_STEP", payload: "source-config" }); dispatch({ type: "SET_STEP", payload: "source-config" });
} catch (e) {
dispatch({ type: "SET_ERROR", payload: errorMessage(e) });
}
}, },
[state.importedFilesBySource] // eslint-disable-line react-hooks/exhaustive-deps [state.importedFilesBySource] // eslint-disable-line react-hooks/exhaustive-deps
); );
@ -576,36 +600,12 @@ export function useImportWizard() {
} }
}, [state.selectedFiles, parseFilesInternal]); }, [state.selectedFiles, parseFilesInternal]);
// Internal helper: runs duplicate checking against parsed rows // Internal helper: runs duplicate checking against parsed rows.
//
// Deliberately writes NOTHING. The source config used to be saved here, so an
// import abandoned at the duplicate step still left a — possibly wrong —
// configuration behind. It is written by `executeImport` now (#324).
const checkDuplicatesInternal = useCallback(async (parsedRows: ParsedRow[]) => { const checkDuplicatesInternal = useCallback(async (parsedRows: ParsedRow[]) => {
// Save/update source config in DB
const config = state.sourceConfig;
const mappingJson = JSON.stringify(config.columnMapping);
let sourceId: number;
if (state.existingSource) {
sourceId = state.existingSource.id;
await updateSource(sourceId, {
name: config.name,
delimiter: config.delimiter,
encoding: config.encoding,
date_format: config.dateFormat,
column_mapping: mappingJson,
skip_lines: config.skipLines,
has_header: config.hasHeader,
});
} else {
sourceId = await createSource({
name: config.name,
delimiter: config.delimiter,
encoding: config.encoding,
date_format: config.dateFormat,
column_mapping: mappingJson,
skip_lines: config.skipLines,
has_header: config.hasHeader,
});
}
// Check file-level duplicates (check ALL selected files, not just the first) // Check file-level duplicates (check ALL selected files, not just the first)
let fileAlreadyImported = false; let fileAlreadyImported = false;
let existingFileId: number | undefined; let existingFileId: number | undefined;
@ -679,7 +679,7 @@ export function useImportWizard() {
}, },
}); });
dispatch({ type: "SET_STEP", payload: "duplicate-check" }); dispatch({ type: "SET_STEP", payload: "duplicate-check" });
}, [state.sourceConfig, state.existingSource, state.selectedFiles]); }, [state.selectedFiles]);
// Check duplicates using already-parsed preview data // Check duplicates using already-parsed preview data
const checkDuplicates = useCallback(async () => { const checkDuplicates = useCallback(async () => {
@ -727,10 +727,26 @@ export function useImportWizard() {
try { try {
const config = state.sourceConfig; const config = state.sourceConfig;
// Get or create source ID // Persist the format — the ONLY write point. It happens here rather than
const dbSource = await getSourceByName(config.name); // at the duplicate step so an abandoned import leaves no configuration
if (!dbSource) throw new Error("Source not found in database"); // behind, and it goes through `formatToRow` so no field can be dropped.
const sourceId = dbSource.id; // It has to precede the file records, which carry a `source_id` FK.
const formatRow = formatToRow(config);
let sourceId: number;
if (state.existingSource) {
sourceId = state.existingSource.id;
await updateSource(sourceId, {
name: config.name,
...formatRow,
template_id: state.selectedTemplateId,
});
} else {
sourceId = await createSource({
name: config.name,
...formatRow,
template_id: state.selectedTemplateId,
});
}
// Determine rows to import: new rows + non-excluded duplicates // Determine rows to import: new rows + non-excluded duplicates
const includedDuplicates = state.duplicateResult.duplicateRows const includedDuplicates = state.duplicateResult.duplicateRows
@ -854,6 +870,8 @@ export function useImportWizard() {
}, [ }, [
state.duplicateResult, state.duplicateResult,
state.sourceConfig, state.sourceConfig,
state.existingSource,
state.selectedTemplateId,
state.excludedDuplicateIndices, state.excludedDuplicateIndices,
state.parsedPreview, state.parsedPreview,
state.selectedFiles, state.selectedFiles,
@ -919,18 +937,7 @@ export function useImportWizard() {
}, [state.selectedFiles, state.sourceConfig, loadHeadersWithConfig]); }, [state.selectedFiles, state.sourceConfig, loadHeadersWithConfig]);
const saveConfigAsTemplate = useCallback(async (name: string) => { const saveConfigAsTemplate = useCallback(async (name: string) => {
const config = state.sourceConfig; await createTemplate({ name, ...formatToRow(state.sourceConfig) });
await createTemplate({
name,
delimiter: config.delimiter,
encoding: config.encoding,
date_format: config.dateFormat,
skip_lines: config.skipLines,
has_header: config.hasHeader ? 1 : 0,
column_mapping: JSON.stringify(config.columnMapping),
amount_mode: config.amountMode,
sign_convention: config.signConvention,
});
const templates = await getAllTemplates(); const templates = await getAllTemplates();
dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates }); dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates });
}, [state.sourceConfig]); }, [state.sourceConfig]);
@ -938,19 +945,21 @@ export function useImportWizard() {
const applyConfigTemplate = useCallback((templateId: number) => { const applyConfigTemplate = useCallback((templateId: number) => {
const template = state.configTemplates.find((t) => t.id === templateId); const template = state.configTemplates.find((t) => t.id === templateId);
if (!template) return; if (!template) return;
const mapping = JSON.parse(template.column_mapping) as ColumnMapping;
const newConfig: SourceConfig = { let newConfig: SourceConfig;
try {
newConfig = {
name: state.sourceConfig.name, name: state.sourceConfig.name,
delimiter: template.delimiter, ...formatFromRow(template),
encoding: template.encoding,
dateFormat: template.date_format,
skipLines: template.skip_lines,
columnMapping: mapping,
amountMode: template.amount_mode,
signConvention: template.sign_convention,
hasHeader: !!template.has_header,
}; };
} catch (e) {
dispatch({ type: "SET_ERROR", payload: errorMessage(e) });
return;
}
dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig }); dispatch({ type: "SET_SOURCE_CONFIG", payload: newConfig });
// Applying a template COPIES its format onto the source. The id recorded
// here is provenance only — the copy is what the next import reads.
dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: templateId }); dispatch({ type: "SET_SELECTED_TEMPLATE_ID", payload: templateId });
// Reload headers with new config // Reload headers with new config
@ -969,17 +978,11 @@ export function useImportWizard() {
if (!state.selectedTemplateId) return; if (!state.selectedTemplateId) return;
const template = state.configTemplates.find((t) => t.id === state.selectedTemplateId); const template = state.configTemplates.find((t) => t.id === state.selectedTemplateId);
if (!template) return; if (!template) return;
const config = state.sourceConfig; // Writes to `import_config_templates` only: a source configured from this
// template keeps its own eight columns and is untouched.
await updateTemplate(state.selectedTemplateId, { await updateTemplate(state.selectedTemplateId, {
name: template.name, name: template.name,
delimiter: config.delimiter, ...formatToRow(state.sourceConfig),
encoding: config.encoding,
date_format: config.dateFormat,
skip_lines: config.skipLines,
has_header: config.hasHeader ? 1 : 0,
column_mapping: JSON.stringify(config.columnMapping),
amount_mode: config.amountMode,
sign_convention: config.signConvention,
}); });
const templates = await getAllTemplates(); const templates = await getAllTemplates();
dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates }); dispatch({ type: "SET_CONFIG_TEMPLATES", payload: templates });

View file

@ -186,6 +186,11 @@
"confirm": "Confirm", "confirm": "Confirm",
"import": "Import" "import": "Import"
}, },
"errors": {
"unsupportedAmountMode": "The amount mode saved for this source is not recognized. Reconfigure the source before importing.",
"unsupportedSignConvention": "The sign convention saved for this source is not recognized. Reconfigure the source before importing.",
"invalidColumnMapping": "The column mapping saved for this source cannot be read. Reconfigure the source before importing."
},
"help": { "help": {
"title": "How to import bank statements", "title": "How to import bank statements",
"tips": [ "tips": [

View file

@ -186,6 +186,11 @@
"confirm": "Confirmer", "confirm": "Confirmer",
"import": "Importer" "import": "Importer"
}, },
"errors": {
"unsupportedAmountMode": "Le mode de montant enregistré pour cette source n'est pas reconnu. Reconfigurez la source avant d'importer.",
"unsupportedSignConvention": "La convention de signe enregistrée pour cette source n'est pas reconnue. Reconfigurez la source avant d'importer.",
"invalidColumnMapping": "Le mapping de colonnes enregistré pour cette source est illisible. Reconfigurez la source avant d'importer."
},
"help": { "help": {
"title": "Comment importer des relevés bancaires", "title": "Comment importer des relevés bancaires",
"tips": [ "tips": [

View file

@ -58,8 +58,13 @@ export default function ImportPage() {
{state.error && ( {state.error && (
<div className="mb-4 p-3 rounded-xl bg-[var(--card)] border-2 border-[var(--negative)] flex items-center gap-2"> <div className="mb-4 p-3 rounded-xl bg-[var(--card)] border-2 border-[var(--negative)] flex items-center gap-2">
<AlertCircle size={16} className="text-[var(--negative)] shrink-0" /> <AlertCircle size={16} className="text-[var(--negative)] shrink-0" />
{/*
The wizard reports a translation key when it has one (a stored
format it cannot decode) and a raw message otherwise; `defaultValue`
renders the latter unchanged.
*/}
<p className="text-sm text-[var(--foreground)]"> <p className="text-sm text-[var(--foreground)]">
{state.error} {t(state.error, { defaultValue: state.error })}
</p> </p>
</div> </div>
)} )}

View file

@ -1,5 +1,15 @@
import { getDb } from "./db"; import { getDb } from "./db";
import type { ImportConfigTemplate } from "../shared/types"; import type { ImportConfigTemplate, ImportFormatRow } from "../shared/types";
/**
* A template is a named format, nothing more. Taking `ImportFormatRow` here
* the same shape `importSourceService` takes is what keeps both writers on
* the codec: a field added to the format cannot reach one table and miss the
* other.
*/
export interface ImportTemplateInput extends ImportFormatRow {
name: string;
}
export async function getAllTemplates(): Promise<ImportConfigTemplate[]> { export async function getAllTemplates(): Promise<ImportConfigTemplate[]> {
const db = await getDb(); const db = await getDb();
@ -9,7 +19,7 @@ export async function getAllTemplates(): Promise<ImportConfigTemplate[]> {
} }
export async function createTemplate( export async function createTemplate(
template: Omit<ImportConfigTemplate, "id" | "created_at"> template: ImportTemplateInput
): Promise<number> { ): Promise<number> {
const db = await getDb(); const db = await getDb();
const result = await db.execute( const result = await db.execute(
@ -30,9 +40,14 @@ export async function createTemplate(
return result.lastInsertId as number; return result.lastInsertId as number;
} }
/**
* Editing a template touches `import_config_templates` and nothing else. A
* source that was configured from this template keeps its own eight columns
* `import_sources.template_id` is a provenance tag, never re-read as format.
*/
export async function updateTemplate( export async function updateTemplate(
id: number, id: number,
template: Omit<ImportConfigTemplate, "id" | "created_at"> template: ImportTemplateInput
): Promise<void> { ): Promise<void> {
const db = await getDb(); const db = await getDb();
await db.execute( await db.execute(

View file

@ -1,5 +1,24 @@
import { getDb } from "./db"; import { getDb } from "./db";
import type { ImportSource } from "../shared/types"; import type { ImportFormatRow, ImportSource } from "../shared/types";
/**
* What a writer hands to this service: the eight format fields in their
* persisted shape (produced by `formatToRow` never assembled by hand) plus
* the source's own columns.
*
* Note it is NOT `Omit<ImportSource, "id" | ...>`: `ImportSource.has_header`
* is declared boolean while the codec emits the 0/1 SQLite actually stores.
* Taking `ImportFormatRow` here is what makes the codec the only place that
* normalization happens.
*/
export interface ImportSourceInput extends ImportFormatRow {
name: string;
description?: string | null;
/** Drift-detection metadata, written by #330. */
header_signature?: string | null;
/** Provenance tag — recorded and displayed, never re-read as format. */
template_id?: number | null;
}
export async function getAllSources(): Promise<ImportSource[]> { export async function getAllSources(): Promise<ImportSource[]> {
const db = await getDb(); const db = await getDb();
@ -29,12 +48,12 @@ export async function getSourceById(
} }
export async function createSource( export async function createSource(
source: Omit<ImportSource, "id" | "created_at" | "updated_at"> source: ImportSourceInput
): Promise<number> { ): Promise<number> {
const db = await getDb(); const db = await getDb();
const result = await db.execute( const result = await db.execute(
`INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header) `INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention, template_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT(name) DO UPDATE SET ON CONFLICT(name) DO UPDATE SET
description = excluded.description, description = excluded.description,
date_format = excluded.date_format, date_format = excluded.date_format,
@ -43,16 +62,22 @@ export async function createSource(
column_mapping = excluded.column_mapping, column_mapping = excluded.column_mapping,
skip_lines = excluded.skip_lines, skip_lines = excluded.skip_lines,
has_header = excluded.has_header, has_header = excluded.has_header,
amount_mode = excluded.amount_mode,
sign_convention = excluded.sign_convention,
template_id = excluded.template_id,
updated_at = CURRENT_TIMESTAMP`, updated_at = CURRENT_TIMESTAMP`,
[ [
source.name, source.name,
source.description || null, source.description ?? null,
source.date_format, source.date_format,
source.delimiter, source.delimiter,
source.encoding, source.encoding,
source.column_mapping, source.column_mapping,
source.skip_lines, source.skip_lines,
source.has_header ? 1 : 0, source.has_header,
source.amount_mode,
source.sign_convention,
source.template_id ?? null,
] ]
); );
// On conflict, lastInsertId may be 0 — look up the existing row // On conflict, lastInsertId may be 0 — look up the existing row
@ -63,45 +88,30 @@ export async function createSource(
export async function updateSource( export async function updateSource(
id: number, id: number,
source: Partial<Omit<ImportSource, "id" | "created_at" | "updated_at">> source: Partial<ImportSourceInput>
): Promise<void> { ): Promise<void> {
const db = await getDb(); const db = await getDb();
const fields: string[] = []; const fields: string[] = [];
const values: unknown[] = []; const values: unknown[] = [];
let paramIndex = 1; let paramIndex = 1;
if (source.name !== undefined) { const setColumn = (column: string, value: unknown) => {
fields.push(`name = $${paramIndex++}`); fields.push(`${column} = $${paramIndex++}`);
values.push(source.name); values.push(value);
} };
if (source.description !== undefined) {
fields.push(`description = $${paramIndex++}`); if (source.name !== undefined) setColumn("name", source.name);
values.push(source.description); if (source.description !== undefined) setColumn("description", source.description);
} if (source.date_format !== undefined) setColumn("date_format", source.date_format);
if (source.date_format !== undefined) { if (source.delimiter !== undefined) setColumn("delimiter", source.delimiter);
fields.push(`date_format = $${paramIndex++}`); if (source.encoding !== undefined) setColumn("encoding", source.encoding);
values.push(source.date_format); if (source.column_mapping !== undefined) setColumn("column_mapping", source.column_mapping);
} if (source.skip_lines !== undefined) setColumn("skip_lines", source.skip_lines);
if (source.delimiter !== undefined) { if (source.has_header !== undefined) setColumn("has_header", source.has_header);
fields.push(`delimiter = $${paramIndex++}`); if (source.amount_mode !== undefined) setColumn("amount_mode", source.amount_mode);
values.push(source.delimiter); if (source.sign_convention !== undefined) setColumn("sign_convention", source.sign_convention);
} if (source.header_signature !== undefined) setColumn("header_signature", source.header_signature);
if (source.encoding !== undefined) { if (source.template_id !== undefined) setColumn("template_id", source.template_id);
fields.push(`encoding = $${paramIndex++}`);
values.push(source.encoding);
}
if (source.column_mapping !== undefined) {
fields.push(`column_mapping = $${paramIndex++}`);
values.push(source.column_mapping);
}
if (source.skip_lines !== undefined) {
fields.push(`skip_lines = $${paramIndex++}`);
values.push(source.skip_lines);
}
if (source.has_header !== undefined) {
fields.push(`has_header = $${paramIndex++}`);
values.push(source.has_header ? 1 : 0);
}
if (fields.length === 0) return; if (fields.length === 0) return;

View file

@ -10,6 +10,28 @@ export interface ImportSource {
column_mapping: string; column_mapping: string;
skip_lines: number; skip_lines: number;
has_header: boolean; has_header: boolean;
/**
* The two fields that decide how an amount is READ, added by migration v17.
* Before v17 they lived only on `import_config_templates`, so restoring a
* source re-inferred the mode and hardcoded the convention (#324).
*
* They are never read directly: `formatFromRow` is the single conversion
* point and validates them, because the column `CHECK` admits
* `absolute_indicator` a value the app cannot map today.
*/
amount_mode: AmountMode;
sign_convention: SignConvention;
/**
* Normalized header labels seen at the last successful import, for drift
* detection. NOT a format field: written by #330, not by the codec.
*/
header_signature?: string | null;
/**
* Provenance tag only records which template this source was configured
* from, and is NEVER re-read as format. The eight format fields above are
* authoritative, so editing a template alters no linked source.
*/
template_id?: number | null;
created_at: string; created_at: string;
updated_at: string; updated_at: string;
} }
@ -154,17 +176,14 @@ export interface BudgetYearRow {
previousYearTotal: number; // actual (transactions) total from the previous year previousYearTotal: number; // actual (transactions) total from the previous year
} }
export interface ImportConfigTemplate { /**
* A named, reusable format. It carries exactly the eight persisted format
* fields (via `ImportFormatRow`) plus its identity the same eight an
* `import_sources` row carries since v17.
*/
export interface ImportConfigTemplate extends ImportFormatRow {
id: number; id: number;
name: string; name: string;
delimiter: string;
encoding: string;
date_format: string;
skip_lines: number;
has_header: number;
column_mapping: string;
amount_mode: AmountMode;
sign_convention: SignConvention;
created_at: string; created_at: string;
} }
@ -213,16 +232,57 @@ export interface ColumnMapping {
export type AmountMode = "single" | "debit_credit"; export type AmountMode = "single" | "debit_credit";
export type SignConvention = "negative_expense" | "positive_expense"; export type SignConvention = "negative_expense" | "positive_expense";
export interface SourceConfig { /**
name: string; * --- The import format ------------------------------------------------------
*
* The eight fields that fully decide how a CSV file is read. They exist in two
* shapes that CANNOT be a single composed type: the persisted rows are
* snake_case with the mapping serialized to JSON and no boolean type in SQLite,
* while the wizard works in camelCase on a parsed mapping.
*
* The guarantee that no field is ever half-persisted therefore does not come
* from the type structure but from `src/utils/importFormat.ts`, the SINGLE
* conversion point between the two shapes, and from its completeness test.
*/
/** Persisted shape — snake_case, mapping as JSON, `has_header` normalized to 0/1. */
export interface ImportFormatRow {
delimiter: string;
encoding: string;
date_format: string;
skip_lines: number;
/** SQLite has no boolean: written as 0/1. Reads tolerate both, see `ImportFormatRowInput`. */
has_header: number;
column_mapping: string;
amount_mode: AmountMode;
sign_convention: SignConvention;
}
/**
* What `formatFromRow` accepts. `import_sources` rows declare `has_header` as a
* boolean and `import_config_templates` rows as a number; both are the same 0/1
* integer at runtime, and the codec normalizes it. Widening the field here is
* what lets a row from either table be decoded without a cast.
*/
export type ImportFormatRowInput = Omit<ImportFormatRow, "has_header"> & {
has_header: number | boolean;
};
/** Domain shape — camelCase, mapping parsed. What the wizard manipulates. */
export interface ImportFormat {
delimiter: string; delimiter: string;
encoding: string; encoding: string;
dateFormat: string; dateFormat: string;
skipLines: number; skipLines: number;
hasHeader: boolean;
columnMapping: ColumnMapping; columnMapping: ColumnMapping;
amountMode: AmountMode; amountMode: AmountMode;
signConvention: SignConvention; signConvention: SignConvention;
hasHeader: boolean; }
/** A format plus the source it belongs to. */
export interface SourceConfig extends ImportFormat {
name: string;
} }
export interface ParsedRow { export interface ParsedRow {

View file

@ -0,0 +1,388 @@
// importFormat — the format codec (#324).
//
// The chantier's root bug was a format field that never reached the database
// and a restore that invented a value for it. These tests hold the two halves
// of the repair: the codec carries EVERY field in both directions, and the
// wizard restores what was saved instead of re-deriving it.
import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import type {
ImportFormat,
ImportFormatRow,
ImportFormatRowInput,
} from "../shared/types";
import {
AMOUNT_MODES,
FORMAT_FIELD_PAIRS,
ImportFormatError,
SIGN_CONVENTIONS,
clearMappingForMode,
formatFromRow,
formatToRow,
} from "./importFormat";
// ---------------------------------------------------------------------------
// Samples. Every field deliberately differs from the wizard's default config,
// so a dropped field surfaces as a mismatch rather than as an accidental pass.
// ---------------------------------------------------------------------------
const SAMPLE_FORMAT: ImportFormat = {
delimiter: ",",
encoding: "windows-1252",
dateFormat: "YYYY-MM-DD",
skipLines: 3,
hasHeader: false,
columnMapping: { date: 2, description: 5, amount: 7 },
amountMode: "single",
signConvention: "positive_expense",
};
const SAMPLE_ROW: ImportFormatRow = {
delimiter: ",",
encoding: "windows-1252",
date_format: "YYYY-MM-DD",
skip_lines: 3,
has_header: 0,
column_mapping: '{"date":2,"description":5,"amount":7}',
amount_mode: "single",
sign_convention: "positive_expense",
};
// ---------------------------------------------------------------------------
// Completeness — the property the codec exists to hold
// ---------------------------------------------------------------------------
describe("codec completeness (#324)", () => {
// `FORMAT_FIELD_PAIRS` is typed `Record<keyof ImportFormat, keyof
// ImportFormatRow>`, so a field added to `ImportFormat` fails to BUILD until
// it is listed. These two assertions close the other half: a field listed but
// not wired through a codec body fails the TEST.
it("formatToRow emits exactly the persisted fields of the table", () => {
expect(Object.keys(formatToRow(SAMPLE_FORMAT)).sort()).toEqual(
Object.values(FORMAT_FIELD_PAIRS).sort()
);
});
it("formatFromRow emits exactly the domain fields of the table", () => {
expect(Object.keys(formatFromRow(SAMPLE_ROW)).sort()).toEqual(
Object.keys(FORMAT_FIELD_PAIRS).sort()
);
});
it("covers the eight format fields, no more and no fewer", () => {
expect(Object.keys(FORMAT_FIELD_PAIRS)).toHaveLength(8);
});
it("maps every domain field to a distinct persisted field", () => {
const persisted = Object.values(FORMAT_FIELD_PAIRS);
expect(new Set(persisted).size).toBe(persisted.length);
});
});
// ---------------------------------------------------------------------------
// Round trip — configure -> save -> reload
// ---------------------------------------------------------------------------
describe("round trip (#324)", () => {
it("survives domain -> row -> domain field by field", () => {
const restored = formatFromRow(formatToRow(SAMPLE_FORMAT));
for (const field of Object.keys(FORMAT_FIELD_PAIRS) as Array<
keyof ImportFormat
>) {
expect(restored[field]).toEqual(SAMPLE_FORMAT[field]);
}
expect(restored).toEqual(SAMPLE_FORMAT);
});
it("survives row -> domain -> row field by field", () => {
expect(formatToRow(formatFromRow(SAMPLE_ROW))).toEqual(SAMPLE_ROW);
});
// THE regression: the credit-card case. Before v17 the convention was not a
// column, so restoring one wrote "negative_expense" and every expense of the
// next import landed as income.
it("keeps positive_expense across a save and a reload", () => {
const positive: ImportFormat = {
...SAMPLE_FORMAT,
signConvention: "positive_expense",
};
expect(formatFromRow(formatToRow(positive)).signConvention).toBe(
"positive_expense"
);
});
it("keeps negative_expense across a save and a reload", () => {
const negative: ImportFormat = {
...SAMPLE_FORMAT,
signConvention: "negative_expense",
};
expect(formatFromRow(formatToRow(negative)).signConvention).toBe(
"negative_expense"
);
});
// The mode used to be re-derived from `mapping.debitAmount !== undefined`. A
// debit/credit source whose debit column is not mapped yet came back as
// "single" — this is that shape.
it("keeps debit_credit even when the mapping carries no debit column", () => {
const partial: ImportFormat = {
...SAMPLE_FORMAT,
amountMode: "debit_credit",
columnMapping: { date: 0, description: 1, creditAmount: 3 },
};
const restored = formatFromRow(formatToRow(partial));
expect(restored.amountMode).toBe("debit_credit");
expect(restored.columnMapping).toEqual({
date: 0,
description: 1,
creditAmount: 3,
});
});
it("keeps single even when the mapping still carries debit/credit keys", () => {
const stale: ImportFormat = {
...SAMPLE_FORMAT,
amountMode: "single",
columnMapping: { date: 0, description: 1, amount: 2, debitAmount: 3 },
};
expect(formatFromRow(formatToRow(stale)).amountMode).toBe("single");
});
});
// ---------------------------------------------------------------------------
// has_header normalization
// ---------------------------------------------------------------------------
describe("has_header normalization", () => {
it("writes 0/1, never a boolean — SQLite has no boolean type", () => {
expect(formatToRow({ ...SAMPLE_FORMAT, hasHeader: true }).has_header).toBe(1);
expect(formatToRow({ ...SAMPLE_FORMAT, hasHeader: false }).has_header).toBe(0);
});
it("reads the integer an import_config_templates row carries", () => {
expect(formatFromRow({ ...SAMPLE_ROW, has_header: 1 }).hasHeader).toBe(true);
expect(formatFromRow({ ...SAMPLE_ROW, has_header: 0 }).hasHeader).toBe(false);
});
it("reads the boolean an import_sources row is declared with", () => {
const asBoolean: ImportFormatRowInput = { ...SAMPLE_ROW, has_header: true };
expect(formatFromRow(asBoolean).hasHeader).toBe(true);
expect(formatFromRow({ ...asBoolean, has_header: false }).hasHeader).toBe(false);
});
});
// ---------------------------------------------------------------------------
// Validation — a value the app cannot map must raise, never fall back
// ---------------------------------------------------------------------------
describe("formatFromRow validation", () => {
it("accepts every mode and convention the app implements", () => {
for (const amount_mode of AMOUNT_MODES) {
expect(formatFromRow({ ...SAMPLE_ROW, amount_mode }).amountMode).toBe(
amount_mode
);
}
for (const sign_convention of SIGN_CONVENTIONS) {
expect(
formatFromRow({ ...SAMPLE_ROW, sign_convention }).signConvention
).toBe(sign_convention);
}
});
// The v17 CHECK admits 'absolute_indicator' so the third mode ships without a
// migration. Until it is implemented, reading one must raise — a fall-back to
// the `single` branch would read the wrong column for every row.
it("rejects an amount mode the app cannot map", () => {
const row = {
...SAMPLE_ROW,
amount_mode: "absolute_indicator",
} as unknown as ImportFormatRowInput;
expect(() => formatFromRow(row)).toThrow(ImportFormatError);
try {
formatFromRow(row);
} catch (e) {
expect((e as ImportFormatError).i18nKey).toBe(
"import.errors.unsupportedAmountMode"
);
}
});
it("rejects an unknown sign convention rather than defaulting", () => {
const row = {
...SAMPLE_ROW,
sign_convention: "whatever",
} as unknown as ImportFormatRowInput;
expect(() => formatFromRow(row)).toThrow(ImportFormatError);
try {
formatFromRow(row);
} catch (e) {
expect((e as ImportFormatError).i18nKey).toBe(
"import.errors.unsupportedSignConvention"
);
}
});
it("rejects a column mapping that is not valid JSON", () => {
expect(() =>
formatFromRow({ ...SAMPLE_ROW, column_mapping: "{not json" })
).toThrow(ImportFormatError);
});
it("rejects a column mapping without usable date/description columns", () => {
for (const column_mapping of ["null", "[]", '{"date":"2"}', "{}"]) {
expect(() => formatFromRow({ ...SAMPLE_ROW, column_mapping })).toThrow(
ImportFormatError
);
}
});
});
// ---------------------------------------------------------------------------
// clearMappingForMode — the mode owns the mapping
// ---------------------------------------------------------------------------
describe("clearMappingForMode", () => {
const full = {
date: 0,
description: 1,
amount: 2,
debitAmount: 3,
creditAmount: 4,
};
it("drops the debit/credit columns when switching to single", () => {
expect(clearMappingForMode(full, "single")).toEqual({
date: 0,
description: 1,
amount: 2,
});
});
it("drops the amount column when switching to debit/credit", () => {
expect(clearMappingForMode(full, "debit_credit")).toEqual({
date: 0,
description: 1,
debitAmount: 3,
creditAmount: 4,
});
});
it("keeps date and description untouched in both directions", () => {
expect(clearMappingForMode({ date: 4, description: 6 }, "single")).toEqual({
date: 4,
description: 6,
});
expect(
clearMappingForMode({ date: 4, description: 6 }, "debit_credit")
).toEqual({ date: 4, description: 6 });
});
// #325 replaces the `?? 0` fallbacks with an explicit "amount column not
// mapped" row error. Materializing the column the <select> merely DISPLAYS
// would make that error unreachable, so an absent column stays absent.
it("does not invent the column the select displays by default", () => {
const cleared = clearMappingForMode({ date: 0, description: 1 }, "single");
expect("amount" in cleared).toBe(false);
const pair = clearMappingForMode({ date: 0, description: 1 }, "debit_credit");
expect("debitAmount" in pair).toBe(false);
expect("creditAmount" in pair).toBe(false);
});
it("emits no undefined-valued keys, so the persisted JSON stays clean", () => {
const row = formatToRow({
...SAMPLE_FORMAT,
columnMapping: clearMappingForMode(full, "single"),
});
expect(row.column_mapping).toBe('{"date":0,"description":1,"amount":2}');
});
it("is idempotent", () => {
const once = clearMappingForMode(full, "debit_credit");
expect(clearMappingForMode(once, "debit_credit")).toEqual(once);
});
});
// ---------------------------------------------------------------------------
// Static guards on useImportWizard.
//
// The restore path and the config write point live inside `useCallback`s of a
// React hook and the repository has no jsdom (see FilterPanel.test.tsx), so the
// only way to assert on them is to read the source. Same technique as the guard
// link 1 left on `parseFilesInternal` in csvAutoDetect.test.ts.
// ---------------------------------------------------------------------------
const WIZARD_SRC = readFileSync(
resolve(import.meta.dirname, "..", "hooks", "useImportWizard.ts"),
"utf-8"
);
/** The body of a `const <name> = useCallback(...)` block, up to the next one. */
function callbackBody(name: string, nextName: string): string {
const start = WIZARD_SRC.indexOf(`const ${name} =`);
const end = WIZARD_SRC.indexOf(`const ${nextName} =`, start);
expect(start).toBeGreaterThan(-1);
expect(end).toBeGreaterThan(start);
return WIZARD_SRC.slice(start, end);
}
/**
* The block of `selectSource` that rebuilds a stored format. Scoped tightly on
* purpose: the fresh-source arm below it legitimately names `encoding` while
* auto-detecting one.
*/
function restoreBranch(): string {
const body = callbackBody("selectSource", "loadHeadersWithConfig");
const start = body.indexOf("let restored: SourceConfig | null = null;");
const end = body.indexOf("if (restored) {", start);
expect(start).toBeGreaterThan(-1);
expect(end).toBeGreaterThan(start);
return body.slice(start, end);
}
describe("useImportWizard — restore reads, never re-derives (#324)", () => {
it("routes the restore through the codec", () => {
expect(restoreBranch()).toContain("...formatFromRow(existing)");
});
it("no longer re-infers the amount mode from the mapping shape", () => {
expect(WIZARD_SRC).not.toContain("mapping.debitAmount !== undefined");
});
// The spread alone is not enough: a literal AFTER it silently wins, which is
// how `signConvention: "negative_expense"` overrode the stored value. The
// restore must name no format field of its own.
it("names no format field of its own in the restore", () => {
const branch = restoreBranch();
for (const field of Object.keys(FORMAT_FIELD_PAIRS)) {
expect(branch, `restore hardcodes ${field}`).not.toContain(`${field}:`);
}
});
it("restores the template provenance instead of blanking it", () => {
expect(callbackBody("selectSource", "loadHeadersWithConfig")).toContain(
"existing?.template_id ?? null"
);
});
});
describe("useImportWizard — the config write point (#324)", () => {
it("writes nothing at the duplicate step", () => {
const body = callbackBody("checkDuplicatesInternal", "checkDuplicates");
expect(body).not.toContain("createSource(");
expect(body).not.toContain("updateSource(");
});
it("writes the format at import time, through the codec", () => {
const body = callbackBody("executeImport", "goToStep");
expect(body).toContain("formatToRow(config)");
expect(body).toContain("createSource(");
expect(body).toContain("updateSource(");
});
it("keeps a single write point in the whole hook", () => {
expect(WIZARD_SRC.match(/await createSource\(/g)).toHaveLength(1);
expect(WIZARD_SRC.match(/await updateSource\(/g)).toHaveLength(1);
});
});

187
src/utils/importFormat.ts Normal file
View file

@ -0,0 +1,187 @@
/**
* The import format codec the SINGLE conversion point between the persisted
* shape of a CSV format (`ImportFormatRow`, shared by `import_sources` and
* `import_config_templates`) and its domain shape (`ImportFormat`).
*
* Why a codec and not a shared type: the two shapes cannot be unified. The
* persisted rows are snake_case, store the column mapping as JSON, and have no
* boolean type; the wizard works in camelCase on a parsed mapping. See the
* comment block on `ImportFormatRow` in `shared/types`.
*
* The property this file exists to hold: a field cannot be added to the format
* and then silently skipped on the way to or from the database. That is
* enforced on two levels `FORMAT_FIELD_PAIRS` is typed against
* `keyof ImportFormat`, so the build breaks until a new field is listed, and
* `importFormat.test.ts` compares the codec's actual output keys against that
* table, so listing a field without wiring it fails the test. Losing
* `sign_convention` on the way out is what this chantier is repairing (#324).
*/
import type {
AmountMode,
ColumnMapping,
ImportFormat,
ImportFormatRow,
ImportFormatRowInput,
SignConvention,
} from "../shared/types";
/**
* Values the application can actually map. The database `CHECK` on
* `import_sources.amount_mode` deliberately admits `absolute_indicator` so the
* third mode ships without another migration but until it is implemented,
* reading one must be a visible error, never a silent fall-through to the
* `single` branch (which would read the wrong column for every row).
*/
export const AMOUNT_MODES: readonly AmountMode[] = ["single", "debit_credit"];
export const SIGN_CONVENTIONS: readonly SignConvention[] = [
"negative_expense",
"positive_expense",
];
/**
* The field correspondence table, domain key -> persisted key.
*
* Typed as `Record<keyof ImportFormat, keyof ImportFormatRow>`: adding a field
* to `ImportFormat` makes this literal stop satisfying the mapped type and the
* build fails until the field is listed here. The test then verifies that both
* codec directions actually emit every listed field.
*/
export const FORMAT_FIELD_PAIRS: Record<keyof ImportFormat, keyof ImportFormatRow> = {
delimiter: "delimiter",
encoding: "encoding",
dateFormat: "date_format",
skipLines: "skip_lines",
hasHeader: "has_header",
columnMapping: "column_mapping",
amountMode: "amount_mode",
signConvention: "sign_convention",
};
/** i18n keys for the ways a stored format can be unreadable. */
export type ImportFormatErrorKey =
| "import.errors.unsupportedAmountMode"
| "import.errors.unsupportedSignConvention"
| "import.errors.invalidColumnMapping";
/**
* Thrown when a persisted format cannot be decoded. It carries an i18n key so
* the wizard can surface it verbatim instead of falling back to a default
* format a wrong default is precisely how the original bug wrote reversed
* amounts without an error.
*/
export class ImportFormatError extends Error {
readonly i18nKey: ImportFormatErrorKey;
constructor(i18nKey: ImportFormatErrorKey, detail: string) {
super(`${i18nKey}: ${detail}`);
this.name = "ImportFormatError";
this.i18nKey = i18nKey;
}
}
/** Domain format -> persisted row. `has_header` is normalized to 0/1 here. */
export function formatToRow(format: ImportFormat): ImportFormatRow {
return {
delimiter: format.delimiter,
encoding: format.encoding,
date_format: format.dateFormat,
skip_lines: format.skipLines,
has_header: format.hasHeader ? 1 : 0,
column_mapping: JSON.stringify(format.columnMapping),
amount_mode: format.amountMode,
sign_convention: format.signConvention,
};
}
/**
* Persisted row -> domain format.
*
* Every enumerated field is validated against the application's whitelist and
* an unknown value raises rather than falls back, because both fall-backs are
* silent data corruption: an unmapped `amount_mode` reads the wrong column, and
* anything other than `positive_expense` would mean `negative_expense` and flip
* every sign.
*/
export function formatFromRow(row: ImportFormatRowInput): ImportFormat {
if (!AMOUNT_MODES.includes(row.amount_mode)) {
throw new ImportFormatError(
"import.errors.unsupportedAmountMode",
String(row.amount_mode)
);
}
if (!SIGN_CONVENTIONS.includes(row.sign_convention)) {
throw new ImportFormatError(
"import.errors.unsupportedSignConvention",
String(row.sign_convention)
);
}
let columnMapping: ColumnMapping;
try {
columnMapping = JSON.parse(row.column_mapping) as ColumnMapping;
} catch {
throw new ImportFormatError(
"import.errors.invalidColumnMapping",
row.column_mapping
);
}
if (
columnMapping === null ||
typeof columnMapping !== "object" ||
typeof columnMapping.date !== "number" ||
typeof columnMapping.description !== "number"
) {
throw new ImportFormatError(
"import.errors.invalidColumnMapping",
row.column_mapping
);
}
return {
delimiter: row.delimiter,
encoding: row.encoding,
dateFormat: row.date_format,
skipLines: row.skip_lines,
hasHeader: !!row.has_header,
columnMapping,
amountMode: row.amount_mode,
signConvention: row.sign_convention,
};
}
/**
* Drop the columns belonging to the mode being left behind, so the amount mode
* is the source of truth for the mapping rather than the reverse.
*
* Before v17 the mode was re-inferred from `mapping.debitAmount !== undefined`,
* which made a stale key from an abandoned mode silently decide how amounts
* were read. The mode is persisted now, but a mapping still carrying both
* shapes would keep the two disagreeing so switching mode prunes.
*
* The column the `<select>` merely DISPLAYS by default (`mapping.amount ?? 0`)
* is deliberately not materialized: #325 turns an unmapped amount column into
* an explicit row error, and writing a 0 here would make that error
* unreachable.
*/
export function clearMappingForMode(
mapping: ColumnMapping,
mode: AmountMode
): ColumnMapping {
const base = { date: mapping.date, description: mapping.description };
if (mode === "debit_credit") {
return {
...base,
...(mapping.debitAmount !== undefined
? { debitAmount: mapping.debitAmount }
: {}),
...(mapping.creditAmount !== undefined
? { creditAmount: mapping.creditAmount }
: {}),
};
}
return {
...base,
...(mapping.amount !== undefined ? { amount: mapping.amount } : {}),
};
}