From f377d760af7b49bcb5c719fb3d210b7900c50114 Mon Sep 17 00:00:00 2001 From: le king fu Date: Fri, 14 Aug 2026 10:58:52 -0400 Subject: [PATCH] fix(export): preserve import sources and templates across data export/import Exporting then re-importing data destroyed every import configuration: dataExportService serialised only categories, suppliers, keywords and transactions, then ran DELETE FROM import_sources on restore and replaced them with a synthetic 'Data Import' source. After restoring a backup, every source had to be reconfigured by hand. - Serialise import_sources and import_config_templates into the envelope, with an explicit format_version; a file without one is the earlier format and its missing arrays are treated as empty. - Wrap wipe + restore in withTransaction, which the service had nowhere: a constraint violation mid-restore used to destroy financial history with no rollback. - Restore templates BEFORE sources (template_id is a foreign key), upserting by name and remapping template_id through the resolved ids, so restoring into a profile that already has templates no longer hits UNIQUE(name). - Whitelist amount_mode and sign_convention at the import boundary with a readable message rather than an SQLite constraint error. Resolves #331 Co-Authored-By: Claude Opus 5 --- .../settings/ImportConfirmModal.tsx | 18 + src/hooks/useDataExport.ts | 7 + src/hooks/useDataImport.ts | 37 +- src/i18n/locales/en.json | 9 +- src/i18n/locales/fr.json | 9 +- src/services/categoryBackupService.test.ts | 2 + src/services/categoryBackupService.ts | 25 +- src/services/dataExportService.test.ts | 730 ++++++++++++++++++ src/services/dataExportService.ts | 677 +++++++++++----- src/utils/importFormat.test.ts | 41 + src/utils/importFormat.ts | 27 + 11 files changed, 1384 insertions(+), 198 deletions(-) create mode 100644 src/services/dataExportService.test.ts diff --git a/src/components/settings/ImportConfirmModal.tsx b/src/components/settings/ImportConfirmModal.tsx index 2e4391e..e67ed7a 100644 --- a/src/components/settings/ImportConfirmModal.tsx +++ b/src/components/settings/ImportConfirmModal.tsx @@ -69,6 +69,24 @@ export default function ImportConfirmModal({ ); } + // The import configurations travel with both transaction modes (#331). They + // are listed so the user sees that a restore now brings its sources back + // rather than leaving every one of them to reconfigure. + if (importType !== "categories_only") { + if (summary.importSourcesCount > 0) + willImport.push( + t("settings.dataManagement.import.countImportSources", { + count: summary.importSourcesCount, + }) + ); + if (summary.importTemplatesCount > 0) + willImport.push( + t("settings.dataManagement.import.countImportTemplates", { + count: summary.importTemplatesCount, + }) + ); + } + const confirmWord = t("settings.dataManagement.import.confirmWord"); const canConfirm = confirmText === confirmWord && !isImporting; diff --git a/src/hooks/useDataExport.ts b/src/hooks/useDataExport.ts index 3a4b3b9..184276d 100644 --- a/src/hooks/useDataExport.ts +++ b/src/hooks/useDataExport.ts @@ -6,6 +6,8 @@ import { getExportSuppliers, getExportKeywords, getExportTransactions, + getExportImportSources, + getExportImportTemplates, serializeToJson, serializeTransactionsToCsv, type ExportMode, @@ -61,6 +63,11 @@ export function useDataExport() { } if (mode === "transactions_with_categories" || mode === "transactions_only") { data.transactions = await getExportTransactions(); + // The import configurations travel with the transaction modes — the + // two the restore wipes `import_sources` for. A categories-only + // backup neither carries them nor touches them (#331). + data.import_sources = await getExportImportSources(); + data.import_config_templates = await getExportImportTemplates(); } // Serialize diff --git a/src/hooks/useDataImport.ts b/src/hooks/useDataImport.ts index 4bfc0bc..8711db6 100644 --- a/src/hooks/useDataImport.ts +++ b/src/hooks/useDataImport.ts @@ -1,4 +1,5 @@ import { useReducer, useCallback } from "react"; +import { useTranslation } from "react-i18next"; import { invoke } from "@tauri-apps/api/core"; import { parseImportedJson, @@ -6,6 +7,7 @@ import { importCategoriesOnly, importTransactionsWithCategories, importTransactionsOnly, + SrefValidationError, type ExportEnvelope, type ImportSummary, } from "../services/dataExportService"; @@ -108,6 +110,20 @@ function parseContent( export function useDataImport() { const [state, dispatch] = useReducer(reducer, initialState); + const { t } = useTranslation(); + + /** + * A refusal at the configuration boundary carries an i18n key and the name of + * the offending source; everything else is a raw exception message. The card + * renders this string as it comes, so the translation happens here. + */ + const describeError = useCallback( + (e: unknown): string => { + if (e instanceof SrefValidationError) return t(e.i18nKey, e.params); + return e instanceof Error ? e.message : String(e); + }, + [t] + ); const pickAndRead = useCallback(async () => { dispatch({ type: "READ_START" }); @@ -136,12 +152,9 @@ export function useDataImport() { const { summary, data, importType } = parseContent(content, filePath); dispatch({ type: "CONFIRMING", filePath, summary, data, importType }); } catch (e) { - dispatch({ - type: "IMPORT_ERROR", - error: e instanceof Error ? e.message : String(e), - }); + dispatch({ type: "IMPORT_ERROR", error: describeError(e) }); } - }, []); + }, [describeError]); const readWithPassword = useCallback( async (password: string) => { @@ -156,13 +169,10 @@ export function useDataImport() { const { summary, data, importType } = parseContent(content, state.filePath); dispatch({ type: "CONFIRMING", filePath: state.filePath, summary, data, importType }); } catch (e) { - dispatch({ - type: "IMPORT_ERROR", - error: e instanceof Error ? e.message : String(e), - }); + dispatch({ type: "IMPORT_ERROR", error: describeError(e) }); } }, - [state.filePath] + [state.filePath, describeError] ); const executeImport = useCallback(async () => { @@ -183,12 +193,9 @@ export function useDataImport() { } dispatch({ type: "IMPORT_SUCCESS" }); } catch (e) { - dispatch({ - type: "IMPORT_ERROR", - error: e instanceof Error ? e.message : String(e), - }); + dispatch({ type: "IMPORT_ERROR", error: describeError(e) }); } - }, [state.parsedData, state.importType, state.filePath]); + }, [state.parsedData, state.importType, state.filePath, describeError]); const reset = useCallback(() => dispatch({ type: "RESET" }), []); diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index 728fcb0..d3ec25e 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -620,12 +620,19 @@ "countSuppliers": "{{count}} supplier(s)", "countKeywords": "{{count}} keyword(s)", "countTransactions": "{{count}} transaction(s)", + "countImportSources": "{{count}} import source(s)", + "countImportTemplates": "{{count}} import template(s)", "irreversibleWarning": "This action is irreversible. All existing data of the selected type will be permanently deleted and replaced.", "typeToConfirm": "Type \"{{word}}\" to confirm:", "confirmWord": "REPLACE", "replaceButton": "Replace Data", "success": "Import completed successfully", - "tryAgain": "Try again" + "tryAgain": "Try again", + "errors": { + "unsupportedAmountMode": "This file cannot be imported: the source \"{{name}}\" was saved with an amount mode this version does not support ({{value}}). Nothing has been modified.", + "unsupportedSignConvention": "This file cannot be imported: the source \"{{name}}\" was saved with a sign convention this version does not support ({{value}}). Nothing has been modified.", + "invalidFormatRow": "This file cannot be imported: the import configuration of \"{{name}}\" is incomplete or damaged. Nothing has been modified." + } } }, "userGuide": { diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index 13db969..2db2e8d 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -620,12 +620,19 @@ "countSuppliers": "{{count}} fournisseur(s)", "countKeywords": "{{count}} mot(s)-clé(s)", "countTransactions": "{{count}} transaction(s)", + "countImportSources": "{{count}} source(s) d'import", + "countImportTemplates": "{{count}} modèle(s) d'import", "irreversibleWarning": "Cette action est irréversible. Toutes les données existantes du type sélectionné seront définitivement supprimées et remplacées.", "typeToConfirm": "Tapez « {{word}} » pour confirmer :", "confirmWord": "REMPLACER", "replaceButton": "Remplacer les données", "success": "Import terminé avec succès", - "tryAgain": "Réessayer" + "tryAgain": "Réessayer", + "errors": { + "unsupportedAmountMode": "Ce fichier ne peut pas être importé : la source « {{name}} » a été enregistrée avec un mode de montant que cette version ne prend pas en charge ({{value}}). Rien n'a été modifié.", + "unsupportedSignConvention": "Ce fichier ne peut pas être importé : la source « {{name}} » a été enregistrée avec une convention de signe que cette version ne prend pas en charge ({{value}}). Rien n'a été modifié.", + "invalidFormatRow": "Ce fichier ne peut pas être importé : la configuration d'import de « {{name}} » est incomplète ou endommagée. Rien n'a été modifié." + } } }, "userGuide": { diff --git a/src/services/categoryBackupService.test.ts b/src/services/categoryBackupService.test.ts index d7ea1b4..e5e6fb7 100644 --- a/src/services/categoryBackupService.test.ts +++ b/src/services/categoryBackupService.test.ts @@ -22,6 +22,8 @@ vi.mock("./dataExportService", async () => { getExportSuppliers: vi.fn(async () => []), getExportKeywords: vi.fn(async () => []), getExportTransactions: vi.fn(async () => []), + getExportImportSources: vi.fn(async () => []), + getExportImportTemplates: vi.fn(async () => []), }; }); diff --git a/src/services/categoryBackupService.ts b/src/services/categoryBackupService.ts index 8ce43bf..a487b7e 100644 --- a/src/services/categoryBackupService.ts +++ b/src/services/categoryBackupService.ts @@ -6,6 +6,8 @@ import { getExportSuppliers, getExportKeywords, getExportTransactions, + getExportImportSources, + getExportImportTemplates, serializeToJson, parseImportedJson, type ExportEnvelope, @@ -254,17 +256,36 @@ export async function createPreMigrationBackup( } // 1. Gather data — same mode as "transactions_with_categories" export. + // Import sources and templates are part of that mode since #331: this + // backup is a safety net, and one that came back without a single import + // configuration would be a poor one. const appVersion = await getVersion(); - const [categories, suppliers, keywords, transactions] = await Promise.all([ + const [ + categories, + suppliers, + keywords, + transactions, + import_sources, + import_config_templates, + ] = await Promise.all([ getExportCategories(), getExportSuppliers(), getExportKeywords(), getExportTransactions(), + getExportImportSources(), + getExportImportTemplates(), ]); const content = serializeToJson( "transactions_with_categories", - { categories, suppliers, keywords, transactions }, + { + categories, + suppliers, + keywords, + transactions, + import_sources, + import_config_templates, + }, appVersion, ); diff --git a/src/services/dataExportService.test.ts b/src/services/dataExportService.test.ts new file mode 100644 index 0000000..fdef24e --- /dev/null +++ b/src/services/dataExportService.test.ts @@ -0,0 +1,730 @@ +/** + * The data export/restore cycle preserves import configurations (#331). + * + * Three properties are under test here, and each one is a way the format used + * to be lost: + * 1. What goes OUT — the file carries `import_sources` and + * `import_config_templates`, which it never did. + * 2. What comes back IN — templates before sources, `template_id` remapped + * through resolved ids, and the synthetic "Data Import" source demoted to + * what it always should have been: a host for transactions that describe + * no folder of their own. + * 3. What happens when it goes wrong — a restore that fails at row N leaves + * the profile untouched. That one is the reason this file exists at all: + * before #331 the service deleted six tables and then re-inserted them + * with no transaction around any of it. + * + * Real `tauri-plugin-sql` cannot run outside the Tauri WebView, so the service + * runs against an in-memory FakeDb that interprets the statements it issues — + * the same approach as `import-format-roundtrip.test.ts`. This one additionally + * models `UNIQUE(name)` and BEGIN/ROLLBACK, because those are the failures the + * transaction exists to survive. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; + +vi.mock("./db", () => { + const getDb = vi.fn(); + return { + getDb, + withTransaction: vi.fn(async (fn: (db: unknown) => unknown) => fn(await getDb())), + }; +}); + +import { getDb } from "./db"; +import { + getExportImportSources, + getExportImportTemplates, + importCategoriesOnly, + importTransactionsOnly, + importTransactionsWithCategories, + parseImportedJson, + serializeToJson, + validateImportedFormatRows, + LEGACY_SREF_FORMAT_VERSION, + SREF_FORMAT_VERSION, + SrefValidationError, + type ExportEnvelope, + type ExportImportSource, + type ExportImportTemplate, +} from "./dataExportService"; +import { formatFromRow, FORMAT_FIELD_PAIRS } from "../utils/importFormat"; +import type { Category } from "../shared/types"; +import fr from "../i18n/locales/fr.json"; +import en from "../i18n/locales/en.json"; + +// --------------------------------------------------------------------------- +// FakeDb — enough SQLite to make the failure modes real. +// --------------------------------------------------------------------------- + +type Row = Record; + +const UNIQUE_NAME_TABLES = ["import_sources", "import_config_templates"]; + +function makeFakeDb(shouldFail?: (sql: string, params: unknown[]) => boolean) { + const tables: Record = { + categories: [], + suppliers: [], + keywords: [], + transactions: [], + imported_files: [], + import_sources: [], + import_config_templates: [], + }; + const log: string[] = []; + let snapshot: Record | null = null; + let nextId = 1; + + const clone = (source: Record) => + Object.fromEntries( + Object.entries(source).map(([name, rows]) => [ + name, + rows.map((row) => ({ ...row })), + ]) + ); + + const insert = (sql: string, params: unknown[]) => { + const [, table, cols] = /INSERT INTO (\w+) \(([^)]+)\)/.exec(sql) ?? []; + const columns = cols.split(",").map((c) => c.trim()); + const row: Row = { id: nextId++ }; + columns.forEach((c, i) => (row[c] = params[i])); + + const clash = tables[table].find((r) => r.name === row.name); + if (/ON CONFLICT\(name\) DO UPDATE/.test(sql)) { + if (clash) { + columns.forEach((c, i) => (clash[c] = params[i])); + nextId--; + return { lastInsertId: 0, rowsAffected: 1 }; + } + } else if (clash && UNIQUE_NAME_TABLES.includes(table)) { + throw new Error(`UNIQUE constraint failed: ${table}.name`); + } + + tables[table].push(row); + return { lastInsertId: row.id as number, rowsAffected: 1 }; + }; + + return { + tables, + log, + seed(table: string, rows: Row[]) { + rows.forEach((row) => tables[table].push({ id: nextId++, ...row })); + }, + execute: vi.fn(async (sql: string, params: unknown[] = []) => { + const head = sql.trimStart().split(/\s+/).slice(0, 3).join(" "); + log.push(head); + if (shouldFail?.(sql, params)) throw new Error("boom"); + + const trimmed = sql.trimStart(); + if (trimmed === "BEGIN") { + snapshot = clone(tables); + return { rowsAffected: 0 }; + } + if (trimmed === "COMMIT") { + snapshot = null; + return { rowsAffected: 0 }; + } + if (trimmed === "ROLLBACK") { + if (snapshot) { + for (const [name, rows] of Object.entries(snapshot)) { + tables[name] = rows; + } + snapshot = null; + } + return { rowsAffected: 0 }; + } + if (trimmed.startsWith("DELETE FROM")) { + const [, table] = /DELETE FROM (\w+)/.exec(trimmed) ?? []; + tables[table] = []; + return { rowsAffected: 0 }; + } + if (trimmed.startsWith("UPDATE transactions SET")) { + return { rowsAffected: 0 }; + } + if (trimmed.startsWith("INSERT")) return insert(sql, params); + throw new Error(`FakeDb: unsupported statement ${sql.slice(0, 40)}`); + }), + select: vi.fn(async (sql: string, params: unknown[] = []) => { + const [, table] = /FROM (\w+)/.exec(sql) ?? []; + const rows = tables[table] ?? []; + if (/WHERE (?:\w+\.)?name = \$1/.test(sql)) + return rows.filter((r) => r.name === params[0]); + // `getExportImportSources` resolves the template through a LEFT JOIN. + if (table === "import_sources" && /LEFT JOIN/.test(sql)) { + return rows.map((r) => ({ + ...r, + template_name: + tables.import_config_templates.find((t) => t.id === r.template_id) + ?.name ?? null, + })); + } + return [...rows]; + }), + }; +} + +let db: ReturnType; + +function wire(instance: ReturnType) { + db = instance; + vi.mocked(getDb).mockResolvedValue(instance as never); +} + +beforeEach(() => { + vi.mocked(getDb).mockReset(); + wire(makeFakeDb()); +}); + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const TEMPLATE: ExportImportTemplate = { + name: "Desjardins EOP", + delimiter: ";", + encoding: "windows-1252", + date_format: "YYYY-MM-DD", + skip_lines: 1, + has_header: 1, + column_mapping: JSON.stringify({ date: 0, description: 2, amount: 3 }), + amount_mode: "single", + sign_convention: "positive_expense", +}; + +const SOURCE: ExportImportSource = { + name: "Visa Desjardins", + description: "Credit card", + header_signature: "date|description|amount", + template_name: TEMPLATE.name, + delimiter: ",", + encoding: "utf-8", + date_format: "DD/MM/YYYY", + skip_lines: 2, + has_header: 0, + column_mapping: JSON.stringify({ date: 1, description: 3, debitAmount: 4, creditAmount: 5 }), + amount_mode: "debit_credit", + sign_convention: "negative_expense", +}; + +function envelopeData( + overrides: Partial = {} +): ExportEnvelope["data"] { + return { + categories: [], + suppliers: [], + keywords: [], + transactions: [], + import_sources: [SOURCE], + import_config_templates: [TEMPLATE], + ...overrides, + }; +} + +function transactionFixture(id: number) { + return { + id, + date: "2026-03-10", + description: `tx ${id}`, + amount: -10, + category_id: null, + category_name: null, + original_description: null, + notes: null, + is_manually_categorized: 0, + is_split: 0, + parent_transaction_id: null, + }; +} + +// --------------------------------------------------------------------------- +// 1. What goes out +// --------------------------------------------------------------------------- + +describe("export carries the import configurations (#331)", () => { + it("projects every format field of a source, plus its own columns", async () => { + db.seed("import_config_templates", [{ name: TEMPLATE.name }]); + const templateId = db.tables.import_config_templates[0].id; + db.seed("import_sources", [ + { + name: SOURCE.name, + description: SOURCE.description, + header_signature: SOURCE.header_signature, + template_id: templateId, + delimiter: SOURCE.delimiter, + encoding: SOURCE.encoding, + date_format: SOURCE.date_format, + skip_lines: SOURCE.skip_lines, + has_header: SOURCE.has_header, + column_mapping: SOURCE.column_mapping, + amount_mode: SOURCE.amount_mode, + sign_convention: SOURCE.sign_convention, + }, + ]); + + const [exported] = await getExportImportSources(); + for (const column of Object.values(FORMAT_FIELD_PAIRS)) { + expect(exported[column], `column ${column}`).toEqual(SOURCE[column]); + } + expect(exported.name).toBe(SOURCE.name); + expect(exported.description).toBe(SOURCE.description); + // Drift metadata rides along beside the format, never through the codec. + expect(exported.header_signature).toBe(SOURCE.header_signature); + // The foreign key travels as a NAME — ids mean nothing in another profile. + expect(exported.template_name).toBe(TEMPLATE.name); + expect("id" in exported).toBe(false); + expect("template_id" in exported).toBe(false); + }); + + it("carries a source an older build left unreadable rather than aborting", async () => { + // The `'{}'` mapping the restore itself used to write: `formatFromRow` + // refuses it, so decoding on the way out would make the whole profile + // impossible to back up. + db.seed("import_sources", [ + { + name: "Data Import", + column_mapping: "{}", + delimiter: ",", + encoding: "utf-8", + date_format: "%Y-%m-%d", + skip_lines: 0, + has_header: 1, + amount_mode: "single", + sign_convention: "negative_expense", + }, + ]); + const [exported] = await getExportImportSources(); + expect(exported.column_mapping).toBe("{}"); + }); + + it("exports templates by name with their eight fields", async () => { + db.seed("import_config_templates", [{ ...TEMPLATE }]); + const [exported] = await getExportImportTemplates(); + expect(exported.name).toBe(TEMPLATE.name); + for (const column of Object.values(FORMAT_FIELD_PAIRS)) { + expect(exported[column], `column ${column}`).toEqual(TEMPLATE[column]); + } + }); + + it("stamps the envelope with an explicit format version", () => { + const json = serializeToJson("transactions_with_categories", envelopeData(), "0.15.0"); + expect(JSON.parse(json).format_version).toBe(SREF_FORMAT_VERSION); + }); + + it("round-trips the two arrays through serialize -> parse", () => { + const json = serializeToJson("transactions_with_categories", envelopeData(), "0.15.0"); + const { envelope, summary } = parseImportedJson(json); + expect(envelope.data.import_sources).toEqual([SOURCE]); + expect(envelope.data.import_config_templates).toEqual([TEMPLATE]); + expect(summary.importSourcesCount).toBe(1); + expect(summary.importTemplatesCount).toBe(1); + expect(summary.formatVersion).toBe(SREF_FORMAT_VERSION); + }); +}); + +// --------------------------------------------------------------------------- +// 2. Backward compatibility — a backup written before this change +// --------------------------------------------------------------------------- + +const LEGACY_BACKUP = JSON.stringify({ + export_type: "transactions_with_categories", + app_version: "0.14.0", + exported_at: "2026-07-01T00:00:00Z", + data: { + categories: [], + suppliers: [], + keywords: [], + transactions: [transactionFixture(1)], + }, +}); + +describe("a backup written before #331 still imports", () => { + it("reads as format version 1 with both counts at zero", () => { + const { envelope, summary } = parseImportedJson(LEGACY_BACKUP); + expect(summary.formatVersion).toBe(LEGACY_SREF_FORMAT_VERSION); + expect(summary.importSourcesCount).toBe(0); + expect(summary.importTemplatesCount).toBe(0); + expect(envelope.data.import_sources).toBeUndefined(); + }); + + it("restores exactly as it did before — wipe, then one host source", async () => { + db.seed("import_sources", [{ name: "Old source", column_mapping: "{}" }]); + const { envelope } = parseImportedJson(LEGACY_BACKUP); + await importTransactionsWithCategories(envelope.data, "backup.json"); + + expect(db.tables.import_sources).toHaveLength(1); + expect(db.tables.import_sources[0].name).toBe("Data Import"); + expect(db.tables.transactions).toHaveLength(1); + }); +}); + +// --------------------------------------------------------------------------- +// 3. The whitelist at the import boundary +// --------------------------------------------------------------------------- + +describe("the import boundary refuses a format it cannot store", () => { + it("accepts a file that carries no configuration at all", () => { + expect(() => validateImportedFormatRows(undefined, undefined)).not.toThrow(); + }); + + it("rejects an amount mode outside the whitelist, naming the source", () => { + // `absolute_indicator` passes the v17 CHECK but no code path can read it. + const bad = { ...SOURCE, amount_mode: "absolute_indicator" as never }; + try { + validateImportedFormatRows([bad], undefined); + expect.unreachable("should have thrown"); + } catch (e) { + expect(e).toBeInstanceOf(SrefValidationError); + const err = e as SrefValidationError; + expect(err.i18nKey).toBe( + "settings.dataManagement.import.errors.unsupportedAmountMode" + ); + expect(err.params).toEqual({ + name: SOURCE.name, + value: "absolute_indicator", + }); + } + }); + + it("rejects an unknown sign convention", () => { + const bad = { ...SOURCE, sign_convention: "inverted" as never }; + expect(() => validateImportedFormatRows([bad], undefined)).toThrow( + SrefValidationError + ); + }); + + it("rejects a row with no column mapping instead of letting NOT NULL fire", () => { + const bad = { ...SOURCE, column_mapping: undefined as never }; + try { + validateImportedFormatRows([bad], undefined); + expect.unreachable("should have thrown"); + } catch (e) { + expect((e as SrefValidationError).i18nKey).toBe( + "settings.dataManagement.import.errors.invalidFormatRow" + ); + } + }); + + it("checks templates too", () => { + const bad = { ...TEMPLATE, amount_mode: "absolute_indicator" as never }; + expect(() => validateImportedFormatRows(undefined, [bad])).toThrow( + SrefValidationError + ); + }); + + it("refuses at PARSE time, so the confirmation never opens", () => { + const json = serializeToJson( + "transactions_with_categories", + envelopeData({ + import_sources: [{ ...SOURCE, amount_mode: "absolute_indicator" as never }], + }), + "0.15.0" + ); + expect(() => parseImportedJson(json)).toThrow(SrefValidationError); + }); + + it("refuses again at the restore, before a single DELETE", async () => { + db.seed("categories", [{ name: "Épicerie" }]); + await expect( + importTransactionsWithCategories( + envelopeData({ + import_sources: [{ ...SOURCE, sign_convention: "inverted" as never }], + }), + "backup.json" + ) + ).rejects.toBeInstanceOf(SrefValidationError); + expect(db.tables.categories).toHaveLength(1); + expect(db.log).not.toContain("BEGIN"); + }); +}); + +// --------------------------------------------------------------------------- +// 4. The restore itself +// --------------------------------------------------------------------------- + +describe("restoring brings the configurations back (#331)", () => { + it("restores a source field by field, template resolved to the new id", async () => { + await importTransactionsWithCategories(envelopeData(), "backup.json"); + + expect(db.tables.import_config_templates).toHaveLength(1); + const template = db.tables.import_config_templates[0]; + const [source] = db.tables.import_sources; + + for (const column of Object.values(FORMAT_FIELD_PAIRS)) { + expect(source[column], `column ${column}`).toEqual(SOURCE[column]); + } + expect(source.description).toBe(SOURCE.description); + expect(source.header_signature).toBe(SOURCE.header_signature); + expect(source.template_id).toBe(template.id); + }); + + it("writes the templates BEFORE the sources — template_id is a foreign key", async () => { + await importTransactionsWithCategories(envelopeData(), "backup.json"); + const inserts = db.log.filter((l) => l.startsWith("INSERT INTO")); + expect(inserts.indexOf("INSERT INTO import_config_templates")).toBeLessThan( + inserts.indexOf("INSERT INTO import_sources") + ); + }); + + it("upserts a template that already exists by name and remaps to its id", async () => { + // Restoring into a profile that already holds templates is the NORMAL path; + // a plain INSERT would hit UNIQUE(name) and roll the whole restore back. + db.seed("import_config_templates", [ + { ...TEMPLATE, delimiter: "\t", skip_lines: 99 }, + ]); + const existingId = db.tables.import_config_templates[0].id; + + await importTransactionsWithCategories(envelopeData(), "backup.json"); + + expect(db.tables.import_config_templates).toHaveLength(1); + expect(db.tables.import_config_templates[0].id).toBe(existingId); + expect(db.tables.import_config_templates[0].delimiter).toBe(TEMPLATE.delimiter); + expect(db.tables.import_sources[0].template_id).toBe(existingId); + }); + + it("leaves template_id null when the name resolves to nothing", async () => { + await importTransactionsWithCategories( + envelopeData({ + import_sources: [{ ...SOURCE, template_name: "Gone" }], + import_config_templates: [], + }), + "backup.json" + ); + expect(db.tables.import_sources[0].template_id).toBeNull(); + }); + + it("creates NO host source when the file carries no transactions", async () => { + await importTransactionsWithCategories(envelopeData(), "backup.json"); + expect(db.tables.import_sources.map((s) => s.name)).toEqual([SOURCE.name]); + expect(db.tables.imported_files).toHaveLength(0); + }); + + it("creates the host source only to carry transactions, with a readable mapping", async () => { + await importTransactionsWithCategories( + envelopeData({ transactions: [transactionFixture(1), transactionFixture(2)] }), + "backup.json" + ); + + const host = db.tables.import_sources.find((s) => s.name === "Data Import")!; + expect(host).toBeDefined(); + // The mapping used to be `'{}'`, which the codec cannot decode: a source + // the app wrote and then refused to read. + const decoded = formatFromRow(host as never); + expect(decoded.columnMapping.date).toBe(0); + expect(decoded.columnMapping.description).toBe(1); + expect(decoded.columnMapping.amount).toBe(2); + + expect(db.tables.transactions).toHaveLength(2); + expect(db.tables.transactions.every((t) => t.source_id === host.id)).toBe(true); + }); + + it("reuses a restored source that already bears the host name", async () => { + // A profile restored once already holds a "Data Import" source, which the + // backup then carries — inserting a second one would break UNIQUE(name) + // and roll back everything. + await importTransactionsWithCategories( + envelopeData({ + import_sources: [{ ...SOURCE, name: "Data Import" }], + transactions: [transactionFixture(1)], + }), + "backup.json" + ); + expect(db.tables.import_sources).toHaveLength(1); + // The restored configuration wins; it is not overwritten by the host stub. + expect(db.tables.import_sources[0].column_mapping).toBe(SOURCE.column_mapping); + expect(db.tables.transactions[0].source_id).toBe(db.tables.import_sources[0].id); + }); + + it("restores sources in transactions_only mode too", async () => { + await importTransactionsOnly(envelopeData(), "backup.json"); + expect(db.tables.import_sources.map((s) => s.name)).toEqual([SOURCE.name]); + expect(db.tables.import_config_templates).toHaveLength(1); + }); + + it("leaves import_sources alone in categories_only mode", async () => { + db.seed("import_sources", [{ name: "Kept", column_mapping: "{}" }]); + await importCategoriesOnly({ categories: [], suppliers: [], keywords: [] }); + expect(db.tables.import_sources.map((s) => s.name)).toEqual(["Kept"]); + }); +}); + +// --------------------------------------------------------------------------- +// 5. All or nothing — the property the whole restore rests on +// --------------------------------------------------------------------------- + +const CATEGORY_FIXTURES: Category[] = [1, 2, 3].map( + (id) => + ({ + id, + name: `Cat ${id}`, + parent_id: null, + color: null, + icon: null, + type: "expense", + is_active: 1, + is_inputable: 1, + sort_order: id, + }) as unknown as Category +); + +describe("a restore that fails at row N leaves the profile untouched", () => { + it("rolls the categories, sources and transactions back", async () => { + let categoryInserts = 0; + wire( + makeFakeDb((sql) => { + if (!/INSERT INTO categories/.test(sql)) return false; + return ++categoryInserts === 3; + }) + ); + db.seed("categories", [{ id: 900, name: "Existing" }]); + db.seed("import_sources", [{ name: "Existing source", column_mapping: "{}" }]); + db.seed("transactions", [{ description: "existing tx" }]); + + await expect( + importTransactionsWithCategories( + envelopeData({ + categories: CATEGORY_FIXTURES, + transactions: [transactionFixture(1)], + }), + "backup.json" + ) + ).rejects.toThrow("boom"); + + expect(db.log).toContain("ROLLBACK"); + expect(db.log).not.toContain("COMMIT"); + expect(db.tables.categories.map((c) => c.name)).toEqual(["Existing"]); + expect(db.tables.import_sources.map((s) => s.name)).toEqual(["Existing source"]); + expect(db.tables.transactions).toHaveLength(1); + }); + + it("rolls back a UNIQUE(name) collision between two restored sources", async () => { + db.seed("categories", [{ id: 900, name: "Existing" }]); + await expect( + importTransactionsWithCategories( + envelopeData({ import_sources: [SOURCE, { ...SOURCE }] }), + "backup.json" + ) + ).rejects.toThrow(/UNIQUE constraint failed/); + expect(db.tables.categories.map((c) => c.name)).toEqual(["Existing"]); + expect(db.tables.import_sources).toHaveLength(0); + }); + + it("wraps categories_only as well", async () => { + let keywordInserts = 0; + wire( + makeFakeDb((sql) => { + if (!/INSERT INTO keywords/.test(sql)) return false; + return ++keywordInserts === 1; + }) + ); + db.seed("categories", [{ id: 900, name: "Existing" }]); + + await expect( + importCategoriesOnly({ + categories: CATEGORY_FIXTURES, + suppliers: [], + keywords: [ + { id: 1, keyword: "epicerie", category_id: 1, priority: 1, is_active: 1 } as never, + ], + }) + ).rejects.toThrow("boom"); + + expect(db.log).toContain("ROLLBACK"); + expect(db.tables.categories.map((c) => c.name)).toEqual(["Existing"]); + }); + + it("wraps transactions_only as well", async () => { + let txInserts = 0; + wire( + makeFakeDb((sql) => { + if (!/INSERT INTO transactions/.test(sql)) return false; + return ++txInserts === 2; + }) + ); + db.seed("transactions", [{ description: "existing tx" }]); + + await expect( + importTransactionsOnly( + envelopeData({ + transactions: [transactionFixture(1), transactionFixture(2)], + }), + "backup.json" + ) + ).rejects.toThrow("boom"); + + expect(db.log).toContain("ROLLBACK"); + expect(db.tables.transactions.map((t) => t.description)).toEqual(["existing tx"]); + expect(db.tables.import_sources).toHaveLength(0); + }); +}); + +// --------------------------------------------------------------------------- +// 6. Every string the refusal can show exists in both languages +// --------------------------------------------------------------------------- + +describe("the refusal messages are translated (#331)", () => { + const errorKeys = [ + "unsupportedAmountMode", + "unsupportedSignConvention", + "invalidFormatRow", + ] as const; + + it("carries every refusal key in FR and EN", () => { + for (const key of errorKeys) { + expect( + fr.settings.dataManagement.import.errors[key].length, + `fr.${key}` + ).toBeGreaterThan(0); + expect( + en.settings.dataManagement.import.errors[key].length, + `en.${key}` + ).toBeGreaterThan(0); + } + }); + + it("names the offending source in both languages", () => { + for (const key of errorKeys) { + expect(fr.settings.dataManagement.import.errors[key], `fr.${key}`).toContain( + "{{name}}" + ); + expect(en.settings.dataManagement.import.errors[key], `en.${key}`).toContain( + "{{name}}" + ); + } + }); + + it("interpolates the two configuration counts", () => { + for (const key of ["countImportSources", "countImportTemplates"] as const) { + expect(fr.settings.dataManagement.import[key], `fr.${key}`).toContain( + "{{count}}" + ); + expect(en.settings.dataManagement.import[key], `en.${key}`).toContain( + "{{count}}" + ); + } + }); + + it("keys every SrefValidationError to a string that exists", () => { + const cases: Array<[ExportImportSource, string]> = [ + [{ ...SOURCE, amount_mode: "absolute_indicator" as never }, "unsupportedAmountMode"], + [{ ...SOURCE, sign_convention: "inverted" as never }, "unsupportedSignConvention"], + [{ ...SOURCE, column_mapping: undefined as never }, "invalidFormatRow"], + ]; + for (const [row, suffix] of cases) { + try { + validateImportedFormatRows([row], undefined); + expect.unreachable(`should have thrown for ${suffix}`); + } catch (e) { + const key = (e as SrefValidationError).i18nKey; + expect(key).toBe(`settings.dataManagement.import.errors.${suffix}`); + // The key must resolve in the bundle, not merely look plausible. + const resolved = key + .split(".") + .reduce( + (node, part) => (node as Record)[part], + en as unknown + ); + expect(typeof resolved).toBe("string"); + } + } + }); +}); diff --git a/src/services/dataExportService.ts b/src/services/dataExportService.ts index ba050e6..7050564 100644 --- a/src/services/dataExportService.ts +++ b/src/services/dataExportService.ts @@ -1,6 +1,19 @@ -import { getDb } from "./db"; +import type Database from "@tauri-apps/plugin-sql"; +import { getDb, withTransaction } from "./db"; import Papa from "papaparse"; -import type { Category, Supplier, Keyword } from "../shared/types"; +import type { + Category, + ImportConfigTemplate, + ImportFormatRow, + ImportSource, + Keyword, + Supplier, +} from "../shared/types"; +import { + AMOUNT_MODES, + SIGN_CONVENTIONS, + pickFormatRow, +} from "../utils/importFormat"; // --- Export types --- @@ -11,15 +24,34 @@ export type ExportMode = export type ExportFormat = "json" | "csv"; +/** + * Version of the SREF envelope this build writes. + * + * Version 2 is the first to carry `import_sources` and + * `import_config_templates`. A file with no `format_version` at all is a + * version 1 file: it predates #331, so its missing arrays mean "this backup + * never held any configuration", not "this profile had none". The restore then + * behaves exactly as it did before — wipe the sources and hang the transactions + * off a synthetic one — which is the only reading that cannot invent data. + */ +export const SREF_FORMAT_VERSION = 2; + +/** What an envelope without an explicit `format_version` is. */ +export const LEGACY_SREF_FORMAT_VERSION = 1; + export interface ExportEnvelope { export_type: ExportMode; app_version: string; + /** Absent on files written before #331 — see `SREF_FORMAT_VERSION`. */ + format_version?: number; exported_at: string; data: { categories?: Category[]; suppliers?: Supplier[]; keywords?: Keyword[]; transactions?: ExportTransaction[]; + import_sources?: ExportImportSource[]; + import_config_templates?: ExportImportTemplate[]; }; } @@ -37,6 +69,35 @@ export interface ExportTransaction { parent_transaction_id: number | null; } +/** + * An import source as it travels in the file: the eight format fields in their + * persisted shape, plus the source's own columns. + * + * No `id`: nothing in the envelope points at a source by id (`ExportTransaction` + * carries no `source_id`), so ids would only be noise that collides on restore. + * The link to a template travels as `template_name` for the same reason — the + * name is the table's natural key (`UNIQUE`), it survives renumbering, and it is + * what the restore resolves the foreign key through. + */ +export interface ExportImportSource extends ImportFormatRow { + name: string; + description: string | null; + /** + * Drift metadata (#330), NOT one of the eight format fields — it travels + * beside them, never through the codec. Round-tripping it keeps drift + * detection armed across a restore; a source that comes back without one just + * has detection off until its next successful import records it. + */ + header_signature: string | null; + /** Name of the template this source was configured from, or null. */ + template_name: string | null; +} + +/** A template as it travels in the file: a named format, nothing more. */ +export interface ExportImportTemplate extends ImportFormatRow { + name: string; +} + // --- Import types --- export interface ImportSummary { @@ -45,6 +106,37 @@ export interface ImportSummary { suppliersCount: number; keywordsCount: number; transactionsCount: number; + importSourcesCount: number; + importTemplatesCount: number; + /** `LEGACY_SREF_FORMAT_VERSION` when the file declares none. */ + formatVersion: number; +} + +/** i18n keys for the ways an imported configuration can be refused. */ +export type SrefValidationErrorKey = + | "settings.dataManagement.import.errors.unsupportedAmountMode" + | "settings.dataManagement.import.errors.unsupportedSignConvention" + | "settings.dataManagement.import.errors.invalidFormatRow"; + +/** + * Thrown when a backup carries a configuration this build cannot store. + * + * The `CHECK` added by migration v17 already refuses these values, but a SQLite + * constraint error is not something a user can act on — and it would surface + * only once the restore had already started deleting. This is raised at the + * boundary, before anything is touched, and carries an i18n key plus the + * offending source name so the message can name what to fix. + */ +export class SrefValidationError extends Error { + readonly i18nKey: SrefValidationErrorKey; + readonly params: Record; + + constructor(i18nKey: SrefValidationErrorKey, params: Record) { + super(`${i18nKey}: ${JSON.stringify(params)}`); + this.name = "SrefValidationError"; + this.i18nKey = i18nKey; + this.params = params; + } } // --- Data gathering --- @@ -76,6 +168,45 @@ export async function getExportTransactions(): Promise { ); } +/** + * Every configured import source, with its template resolved to a name. + * + * This is what the export was missing (#331): the file carried transactions but + * not a single line of how they had been read, so restoring a backup meant + * reconfiguring every source by hand. + */ +export async function getExportImportSources(): Promise { + const db = await getDb(); + const rows = await db.select< + Array + >( + `SELECT s.*, t.name AS template_name + FROM import_sources s + LEFT JOIN import_config_templates t ON s.template_id = t.id + ORDER BY s.id` + ); + return rows.map((row) => ({ + name: row.name, + description: row.description ?? null, + header_signature: row.header_signature ?? null, + template_name: row.template_name ?? null, + ...pickFormatRow(row as unknown as Record), + })); +} + +export async function getExportImportTemplates(): Promise< + ExportImportTemplate[] +> { + const db = await getDb(); + const rows = await db.select( + "SELECT * FROM import_config_templates ORDER BY id" + ); + return rows.map((row) => ({ + name: row.name, + ...pickFormatRow(row as unknown as Record), + })); +} + // --- Serialization --- export function serializeToJson( @@ -86,6 +217,7 @@ export function serializeToJson( const envelope: ExportEnvelope = { export_type: exportType, app_version: appVersion, + format_version: SREF_FORMAT_VERSION, exported_at: new Date().toISOString(), data, }; @@ -113,6 +245,49 @@ export function serializeTransactionsToCsv( // --- Import parsing --- +/** + * Refuse a configuration this build cannot store, BEFORE anything is deleted. + * + * Only the two enumerated fields are checked. `column_mapping` deliberately is + * not decoded: a profile that already went through an old restore holds sources + * with `'{}'` there, and refusing to restore a backup because of a row the app + * itself wrote would strand the user with no way back in. It is carried through + * as the opaque string the column stores, exactly as it was found — the wizard + * validates it when it is actually about to read a file. + * + * PURE: no I/O, so it can be called from the parse step and again at the top of + * each restore, which is where the guarantee has to hold. + */ +export function validateImportedFormatRows( + sources: ExportImportSource[] | undefined, + templates: ExportImportTemplate[] | undefined +): void { + const check = (row: ImportFormatRow, name: unknown) => { + const label = typeof name === "string" && name.length > 0 ? name : "?"; + if (typeof row?.column_mapping !== "string") { + throw new SrefValidationError( + "settings.dataManagement.import.errors.invalidFormatRow", + { name: label } + ); + } + if (!AMOUNT_MODES.includes(row.amount_mode)) { + throw new SrefValidationError( + "settings.dataManagement.import.errors.unsupportedAmountMode", + { name: label, value: String(row.amount_mode) } + ); + } + if (!SIGN_CONVENTIONS.includes(row.sign_convention)) { + throw new SrefValidationError( + "settings.dataManagement.import.errors.unsupportedSignConvention", + { name: label, value: String(row.sign_convention) } + ); + } + }; + + for (const template of templates ?? []) check(template, template?.name); + for (const source of sources ?? []) check(source, source?.name); +} + export function parseImportedJson(content: string): { envelope: ExportEnvelope; summary: ImportSummary; @@ -141,6 +316,13 @@ export function parseImportedJson(content: string): { throw new Error(`Unknown export type: ${envelope.export_type}`); } + // Refuse an unreadable configuration while the file is only being LOOKED at: + // the confirmation dialog never opens, so nothing is deleted. + validateImportedFormatRows( + envelope.data.import_sources, + envelope.data.import_config_templates + ); + return { envelope, summary: { @@ -149,6 +331,9 @@ export function parseImportedJson(content: string): { suppliersCount: envelope.data.suppliers?.length ?? 0, keywordsCount: envelope.data.keywords?.length ?? 0, transactionsCount: envelope.data.transactions?.length ?? 0, + importSourcesCount: envelope.data.import_sources?.length ?? 0, + importTemplatesCount: envelope.data.import_config_templates?.length ?? 0, + formatVersion: envelope.format_version ?? LEGACY_SREF_FORMAT_VERSION, }, }; } @@ -190,213 +375,347 @@ export function parseImportedCsv(content: string): { suppliersCount: 0, keywordsCount: 0, transactionsCount: transactions.length, + // A flat CSV carries no configuration at all. + importSourcesCount: 0, + importTemplatesCount: 0, + formatVersion: LEGACY_SREF_FORMAT_VERSION, }, }; } // --- Import execution --- -export async function importCategoriesOnly(data: ExportEnvelope["data"]): Promise { - const db = await getDb(); +/** + * The source that hosts transactions restored from a file, which describe no + * import folder of their own. + * + * Its mapping used to be `'{}'`, which `formatFromRow` cannot decode — a source + * the app itself wrote and then refused to read. It describes the CSV this very + * service exports instead: date, description, amount, comma-separated, ISO + * dates, expenses negative. That is true of the file the rows came from, so the + * wizard finds a usable format if a folder ever bears this name. + */ +const HOST_SOURCE_NAME = "Data Import"; +const HOST_SOURCE_MAPPING = JSON.stringify({ + date: 0, + description: 1, + amount: 2, +}); - // Wipe keywords, suppliers, categories - await db.execute("DELETE FROM keywords"); - await db.execute("DELETE FROM suppliers"); - await db.execute("DELETE FROM categories"); +/** + * Run `body` as one all-or-nothing restore. + * + * Every one of these functions starts by DELETING the profile's financial + * history and then re-inserts it row by row, against tables carrying + * `UNIQUE(name)` and a foreign key. Without a transaction, the first constraint + * violation leaves the profile emptied at whatever row it reached, with nothing + * to go back to. `withTransaction` holds the single DB lock for the whole + * BEGIN..COMMIT so the statements cannot be split across pooled connections + * (see `db.ts`); the BEGIN/COMMIT/ROLLBACK themselves are ours to issue. + */ +async function runRestore( + body: (db: Database) => Promise +): Promise { + return withTransaction(async (db) => { + await db.execute("BEGIN"); + let open = true; + try { + await body(db); + await db.execute("COMMIT"); + open = false; + } catch (e) { + if (open) { + try { + await db.execute("ROLLBACK"); + } catch { + // Preserve the original error — it is the one that explains the failure. + } + } + throw e; + } + }); +} - // Nullify category/supplier references on transactions - await db.execute( - "UPDATE transactions SET category_id = NULL, supplier_id = NULL, is_manually_categorized = 0" +async function restoreCategories( + db: Database, + categories: Category[] | undefined +): Promise { + for (const cat of categories ?? []) { + await db.execute( + `INSERT INTO categories (id, name, parent_id, color, icon, type, is_active, is_inputable, sort_order) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + [ + cat.id, + cat.name, + cat.parent_id ?? null, + cat.color ?? null, + cat.icon ?? null, + cat.type, + cat.is_active ? 1 : 0, + cat.is_inputable ? 1 : 0, + cat.sort_order, + ] + ); + } +} + +async function restoreSuppliers( + db: Database, + suppliers: Supplier[] | undefined +): Promise { + for (const sup of suppliers ?? []) { + await db.execute( + `INSERT INTO suppliers (id, name, normalized_name, category_id, is_active) + VALUES ($1, $2, $3, $4, $5)`, + [sup.id, sup.name, sup.normalized_name, sup.category_id ?? null, sup.is_active ? 1 : 0] + ); + } +} + +async function restoreKeywords( + db: Database, + keywords: Keyword[] | undefined +): Promise { + for (const kw of keywords ?? []) { + await db.execute( + `INSERT INTO keywords (id, keyword, category_id, supplier_id, priority, is_active) + VALUES ($1, $2, $3, $4, $5, $6)`, + [kw.id, kw.keyword, kw.category_id, kw.supplier_id ?? null, kw.priority, kw.is_active ? 1 : 0] + ); + } +} + +/** + * Restore the templates and return their resolved ids, keyed by name. + * + * `import_config_templates` is in NEITHER wipe list, and restoring into a + * profile that already has templates is the normal path, not an edge case — a + * plain INSERT would hit `UNIQUE constraint failed: import_config_templates.name` + * and (now) roll the whole restore back. Wiping the table instead was the other + * option and was rejected: it would destroy templates the user never asked to + * replace and that no confirmation dialog mentions. So: upsert by name, and give + * the caller the map it needs to remap `import_sources.template_id`, since the + * ids in the file mean nothing here. + * + * Templates go in BEFORE sources — `template_id` is a foreign key to this table. + */ +async function restoreImportTemplates( + db: Database, + templates: ExportImportTemplate[] | undefined +): Promise> { + const idsByName = new Map(); + for (const template of templates ?? []) { + await db.execute( + `INSERT INTO import_config_templates (name, delimiter, encoding, date_format, skip_lines, has_header, column_mapping, amount_mode, sign_convention) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT(name) DO UPDATE SET + delimiter = excluded.delimiter, + encoding = excluded.encoding, + date_format = excluded.date_format, + skip_lines = excluded.skip_lines, + has_header = excluded.has_header, + column_mapping = excluded.column_mapping, + amount_mode = excluded.amount_mode, + sign_convention = excluded.sign_convention`, + [ + template.name, + template.delimiter, + template.encoding, + template.date_format, + template.skip_lines, + template.has_header, + template.column_mapping, + template.amount_mode, + template.sign_convention, + ] + ); + // `lastInsertId` is 0 on the conflict branch, so the id is read back by + // name — the same reason `importSourceService.createSource` does it. + const rows = await db.select>( + "SELECT id FROM import_config_templates WHERE name = $1", + [template.name] + ); + if (rows.length > 0) idsByName.set(template.name, rows[0].id); + } + return idsByName; +} + +/** + * Restore the sources, resolving each `template_name` to the id the templates + * pass just settled. A name with no match becomes NULL: `template_id` is a + * provenance tag, never re-read as format, so losing it costs no configuration. + * + * Ids are not carried over — nothing in the envelope refers to a source by id. + */ +async function restoreImportSources( + db: Database, + sources: ExportImportSource[] | undefined, + templateIds: Map +): Promise { + for (const source of sources ?? []) { + await db.execute( + `INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention, header_signature, template_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + [ + source.name, + source.description ?? null, + source.date_format, + source.delimiter, + source.encoding, + source.column_mapping, + source.skip_lines, + source.has_header, + source.amount_mode, + source.sign_convention, + source.header_signature ?? null, + source.template_name !== null + ? templateIds.get(source.template_name) ?? null + : null, + ] + ); + } +} + +/** + * Attach the restored transactions to a source and one `imported_files` row. + * + * The host source is created ONLY here, and only when there are transactions to + * hang off it — it exists to satisfy `transactions.source_id`, not to stand in + * for the configurations the file now carries on its own. A restored source may + * already bear the name (an earlier restore of this same profile wrote one), in + * which case it is reused rather than re-inserted: `import_sources.name` is + * UNIQUE, and a collision here would roll back the entire restore. + */ +async function attachTransactions( + db: Database, + transactions: ExportTransaction[] | undefined, + filename: string +): Promise { + if (!transactions || transactions.length === 0) return; + + const existing = await db.select>( + "SELECT id FROM import_sources WHERE name = $1", + [HOST_SOURCE_NAME] ); - - // Re-insert categories - if (data.categories) { - for (const cat of data.categories) { - await db.execute( - `INSERT INTO categories (id, name, parent_id, color, icon, type, is_active, is_inputable, sort_order) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - [ - cat.id, - cat.name, - cat.parent_id ?? null, - cat.color ?? null, - cat.icon ?? null, - cat.type, - cat.is_active ? 1 : 0, - cat.is_inputable ? 1 : 0, - cat.sort_order, - ] - ); - } + let sourceId: number; + if (existing.length > 0) { + sourceId = existing[0].id; + } else { + const sourceResult = await db.execute( + `INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + [ + HOST_SOURCE_NAME, + "Imported from settings", + "%Y-%m-%d", + ",", + "utf-8", + HOST_SOURCE_MAPPING, + 0, + 1, + "single", + "negative_expense", + ] + ); + sourceId = sourceResult.lastInsertId as number; } - // Re-insert suppliers - if (data.suppliers) { - for (const sup of data.suppliers) { - await db.execute( - `INSERT INTO suppliers (id, name, normalized_name, category_id, is_active) - VALUES ($1, $2, $3, $4, $5)`, - [sup.id, sup.name, sup.normalized_name, sup.category_id ?? null, sup.is_active ? 1 : 0] - ); - } - } + const fileResult = await db.execute( + `INSERT INTO imported_files (source_id, filename, file_hash, row_count, status) + VALUES ($1, $2, $3, $4, $5)`, + [sourceId, filename, `data-import-${Date.now()}`, transactions.length, "completed"] + ); + const fileId = fileResult.lastInsertId; - // Re-insert keywords - if (data.keywords) { - for (const kw of data.keywords) { - await db.execute( - `INSERT INTO keywords (id, keyword, category_id, supplier_id, priority, is_active) - VALUES ($1, $2, $3, $4, $5, $6)`, - [kw.id, kw.keyword, kw.category_id, kw.supplier_id ?? null, kw.priority, kw.is_active ? 1 : 0] - ); - } + for (const tx of transactions) { + await db.execute( + `INSERT INTO transactions (date, description, amount, category_id, original_description, notes, is_manually_categorized, is_split, parent_transaction_id, source_id, file_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + [ + tx.date, + tx.description, + tx.amount, + tx.category_id, + tx.original_description, + tx.notes, + tx.is_manually_categorized, + tx.is_split, + tx.parent_transaction_id, + sourceId, + fileId, + ] + ); } } +export async function importCategoriesOnly( + data: ExportEnvelope["data"] +): Promise { + return runRestore(async (db) => { + // Wipe keywords, suppliers, categories + await db.execute("DELETE FROM keywords"); + await db.execute("DELETE FROM suppliers"); + await db.execute("DELETE FROM categories"); + + // Nullify category/supplier references on transactions + await db.execute( + "UPDATE transactions SET category_id = NULL, supplier_id = NULL, is_manually_categorized = 0" + ); + + await restoreCategories(db, data.categories); + await restoreSuppliers(db, data.suppliers); + await restoreKeywords(db, data.keywords); + }); +} + export async function importTransactionsWithCategories( data: ExportEnvelope["data"], filename: string ): Promise { - const db = await getDb(); + validateImportedFormatRows(data.import_sources, data.import_config_templates); - // Wipe everything - await db.execute("DELETE FROM transactions"); - await db.execute("DELETE FROM imported_files"); - await db.execute("DELETE FROM import_sources"); - await db.execute("DELETE FROM keywords"); - await db.execute("DELETE FROM suppliers"); - await db.execute("DELETE FROM categories"); + return runRestore(async (db) => { + // Wipe everything + await db.execute("DELETE FROM transactions"); + await db.execute("DELETE FROM imported_files"); + await db.execute("DELETE FROM import_sources"); + await db.execute("DELETE FROM keywords"); + await db.execute("DELETE FROM suppliers"); + await db.execute("DELETE FROM categories"); - // Re-insert categories - if (data.categories) { - for (const cat of data.categories) { - await db.execute( - `INSERT INTO categories (id, name, parent_id, color, icon, type, is_active, is_inputable, sort_order) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - [ - cat.id, - cat.name, - cat.parent_id ?? null, - cat.color ?? null, - cat.icon ?? null, - cat.type, - cat.is_active ? 1 : 0, - cat.is_inputable ? 1 : 0, - cat.sort_order, - ] - ); - } - } + await restoreCategories(db, data.categories); + await restoreSuppliers(db, data.suppliers); + await restoreKeywords(db, data.keywords); - // Re-insert suppliers - if (data.suppliers) { - for (const sup of data.suppliers) { - await db.execute( - `INSERT INTO suppliers (id, name, normalized_name, category_id, is_active) - VALUES ($1, $2, $3, $4, $5)`, - [sup.id, sup.name, sup.normalized_name, sup.category_id ?? null, sup.is_active ? 1 : 0] - ); - } - } + // Templates first: `import_sources.template_id` points at them. + const templateIds = await restoreImportTemplates( + db, + data.import_config_templates + ); + await restoreImportSources(db, data.import_sources, templateIds); - // Re-insert keywords - if (data.keywords) { - for (const kw of data.keywords) { - await db.execute( - `INSERT INTO keywords (id, keyword, category_id, supplier_id, priority, is_active) - VALUES ($1, $2, $3, $4, $5, $6)`, - [kw.id, kw.keyword, kw.category_id, kw.supplier_id ?? null, kw.priority, kw.is_active ? 1 : 0] - ); - } - } - - // Create tracking records for import history - const sourceResult = await db.execute( - `INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, - ["Data Import", "Imported from settings", "%Y-%m-%d", ",", "utf-8", "{}", 0] - ); - const sourceId = sourceResult.lastInsertId; - - const txCount = data.transactions?.length ?? 0; - const fileResult = await db.execute( - `INSERT INTO imported_files (source_id, filename, file_hash, row_count, status) - VALUES ($1, $2, $3, $4, $5)`, - [sourceId, filename, `data-import-${Date.now()}`, txCount, "completed"] - ); - const fileId = fileResult.lastInsertId; - - // Re-insert transactions linked to the import - if (data.transactions) { - for (const tx of data.transactions) { - await db.execute( - `INSERT INTO transactions (date, description, amount, category_id, original_description, notes, is_manually_categorized, is_split, parent_transaction_id, source_id, file_id) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - [ - tx.date, - tx.description, - tx.amount, - tx.category_id, - tx.original_description, - tx.notes, - tx.is_manually_categorized, - tx.is_split, - tx.parent_transaction_id, - sourceId, - fileId, - ] - ); - } - } + await attachTransactions(db, data.transactions, filename); + }); } export async function importTransactionsOnly( data: ExportEnvelope["data"], filename: string ): Promise { - const db = await getDb(); + validateImportedFormatRows(data.import_sources, data.import_config_templates); - // Wipe transactions and import history - await db.execute("DELETE FROM transactions"); - await db.execute("DELETE FROM imported_files"); - await db.execute("DELETE FROM import_sources"); + return runRestore(async (db) => { + // Wipe transactions and import history + await db.execute("DELETE FROM transactions"); + await db.execute("DELETE FROM imported_files"); + await db.execute("DELETE FROM import_sources"); - // Create tracking records for import history - const sourceResult = await db.execute( - `INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, - ["Data Import", "Imported from settings", "%Y-%m-%d", ",", "utf-8", "{}", 0] - ); - const sourceId = sourceResult.lastInsertId; + const templateIds = await restoreImportTemplates( + db, + data.import_config_templates + ); + await restoreImportSources(db, data.import_sources, templateIds); - const txCount = data.transactions?.length ?? 0; - const fileResult = await db.execute( - `INSERT INTO imported_files (source_id, filename, file_hash, row_count, status) - VALUES ($1, $2, $3, $4, $5)`, - [sourceId, filename, `data-import-${Date.now()}`, txCount, "completed"] - ); - const fileId = fileResult.lastInsertId; - - // Re-insert transactions linked to the import - if (data.transactions) { - for (const tx of data.transactions) { - await db.execute( - `INSERT INTO transactions (date, description, amount, category_id, original_description, notes, is_manually_categorized, is_split, parent_transaction_id, source_id, file_id) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - [ - tx.date, - tx.description, - tx.amount, - tx.category_id, - tx.original_description, - tx.notes, - tx.is_manually_categorized, - tx.is_split, - tx.parent_transaction_id, - sourceId, - fileId, - ] - ); - } - } + await attachTransactions(db, data.transactions, filename); + }); } diff --git a/src/utils/importFormat.test.ts b/src/utils/importFormat.test.ts index eb1e189..0026c7c 100644 --- a/src/utils/importFormat.test.ts +++ b/src/utils/importFormat.test.ts @@ -27,6 +27,7 @@ import { formatToRow, isRowErrorKey, mapRow, + pickFormatRow, summarizeParsedRows, } from "./importFormat"; import fr from "../i18n/locales/fr.json"; @@ -88,6 +89,46 @@ describe("codec completeness (#324)", () => { const persisted = Object.values(FORMAT_FIELD_PAIRS); expect(new Set(persisted).size).toBe(persisted.length); }); + + it("pickFormatRow emits exactly the persisted fields of the table", () => { + expect(Object.keys(pickFormatRow(SAMPLE_ROW)).sort()).toEqual( + Object.values(FORMAT_FIELD_PAIRS).sort() + ); + }); +}); + +// --------------------------------------------------------------------------- +// Projection — the direction the SREF export needs (#331) +// --------------------------------------------------------------------------- + +describe("pickFormatRow (#331)", () => { + it("carries the eight columns of a row verbatim", () => { + expect(pickFormatRow({ ...SAMPLE_ROW, id: 7, name: "Visa" })).toEqual( + SAMPLE_ROW + ); + }); + + it("normalizes has_header to the 0/1 the column stores", () => { + // `ImportSource` declares it a boolean; SQLite has no such type. + expect(pickFormatRow({ ...SAMPLE_ROW, has_header: true }).has_header).toBe(1); + expect(pickFormatRow({ ...SAMPLE_ROW, has_header: false }).has_header).toBe(0); + }); + + it("does NOT validate — an unreadable mapping still travels", () => { + // `formatFromRow` refuses this row; a backup must still be able to carry a + // source an older build left in that state. + const row = { ...SAMPLE_ROW, column_mapping: "{}" }; + expect(() => formatFromRow(row)).toThrow(ImportFormatError); + expect(pickFormatRow(row).column_mapping).toBe("{}"); + }); + + it("never names the drift metadata — it is not a format field", () => { + const picked = pickFormatRow({ + ...SAMPLE_ROW, + header_signature: "date|desc", + }); + expect("header_signature" in picked).toBe(false); + }); }); // --------------------------------------------------------------------------- diff --git a/src/utils/importFormat.ts b/src/utils/importFormat.ts index 696797f..15cdc9c 100644 --- a/src/utils/importFormat.ts +++ b/src/utils/importFormat.ts @@ -157,6 +157,33 @@ export function formatFromRow(row: ImportFormatRowInput): ImportFormat { }; } +/** + * Project any persisted row onto EXACTLY the eight format columns. + * + * This is the direction the SREF export needs, and the reason it cannot simply + * call `formatFromRow`: that one decodes AND validates, which is right when a + * format is about to read a file but wrong when a whole profile is being + * serialized. A single source left unreadable by an older build — the + * `column_mapping: '{}'` that the data restore itself used to write — would + * abort the entire backup. Projection carries the row out verbatim; the + * whitelist runs on the way back IN (#331), where refusing costs nothing. + * + * Built from `FORMAT_FIELD_PAIRS` like both codec directions, so a field added + * to the format is exported without this function being touched. + */ +export function pickFormatRow(row: object): ImportFormatRow { + const source = row as Record; + const picked: Record = {}; + for (const column of Object.values(FORMAT_FIELD_PAIRS)) { + picked[column] = source[column]; + } + // SQLite has no boolean, but `ImportSource` declares `has_header` as one, so + // a row can reach here either way. Normalize exactly as `formatToRow` does: + // the exported file always carries the 0/1 the column actually stores. + picked[FORMAT_FIELD_PAIRS.hasHeader] = source[FORMAT_FIELD_PAIRS.hasHeader] ? 1 : 0; + return picked as unknown as ImportFormatRow; +} + /** * Drop the columns belonging to the mode being left behind, so the amount mode * is the source of truth for the mapping rather than the reverse.