import type Database from "@tauri-apps/plugin-sql"; import { getDb, withTransaction } from "./db"; import Papa from "papaparse"; import { toSqlFlag } from "../utils/sqlFlag"; import type { Category, ImportConfigTemplate, ImportFormatRow, ImportSource, Keyword, Supplier, } from "../shared/types"; import { AMOUNT_MODES, SIGN_CONVENTIONS, pickFormatRow, } from "../utils/importFormat"; // --- Export types --- export type ExportMode = | "transactions_with_categories" | "transactions_only" | "categories_only"; export type ExportFormat = "json" | "csv"; /** * Version of the SREF envelope this build writes. * * Version 2 is the first to carry `import_sources` and * `import_config_templates`. A file with no `format_version` at all is a * version 1 file: it predates #331, so its missing arrays mean "this backup * never held any configuration", not "this profile had none". The restore then * behaves exactly as it did before — wipe the sources and hang the transactions * off a synthetic one — which is the only reading that cannot invent data. */ export const SREF_FORMAT_VERSION = 2; /** What an envelope without an explicit `format_version` is. */ export const LEGACY_SREF_FORMAT_VERSION = 1; export interface ExportEnvelope { export_type: ExportMode; app_version: string; /** Absent on files written before #331 — see `SREF_FORMAT_VERSION`. */ format_version?: number; exported_at: string; data: { categories?: Category[]; suppliers?: Supplier[]; keywords?: Keyword[]; transactions?: ExportTransaction[]; import_sources?: ExportImportSource[]; import_config_templates?: ExportImportTemplate[]; }; } export interface ExportTransaction { id: number; date: string; description: string; amount: number; category_id: number | null; category_name: string | null; original_description: string | null; notes: string | null; is_manually_categorized: number; is_split: number; parent_transaction_id: number | null; } /** * An import source as it travels in the file: the eight format fields in their * persisted shape, plus the source's own columns. * * No `id`: nothing in the envelope points at a source by id (`ExportTransaction` * carries no `source_id`), so ids would only be noise that collides on restore. * The link to a template travels as `template_name` for the same reason — the * name is the table's natural key (`UNIQUE`), it survives renumbering, and it is * what the restore resolves the foreign key through. */ export interface ExportImportSource extends ImportFormatRow { name: string; description: string | null; /** * Drift metadata (#330), NOT one of the eight format fields — it travels * beside them, never through the codec. Round-tripping it keeps drift * detection armed across a restore; a source that comes back without one just * has detection off until its next successful import records it. */ header_signature: string | null; /** Name of the template this source was configured from, or null. */ template_name: string | null; } /** A template as it travels in the file: a named format, nothing more. */ export interface ExportImportTemplate extends ImportFormatRow { name: string; } // --- Import types --- export interface ImportSummary { type: ExportMode; categoriesCount: number; suppliersCount: number; keywordsCount: number; transactionsCount: number; importSourcesCount: number; importTemplatesCount: number; /** `LEGACY_SREF_FORMAT_VERSION` when the file declares none. */ formatVersion: number; } /** i18n keys for the ways an imported configuration can be refused. */ export type SrefValidationErrorKey = | "settings.dataManagement.import.errors.unsupportedAmountMode" | "settings.dataManagement.import.errors.unsupportedSignConvention" | "settings.dataManagement.import.errors.invalidFormatRow"; /** * Thrown when a backup carries a configuration this build cannot store. * * The `CHECK` added by migration v17 already refuses these values, but a SQLite * constraint error is not something a user can act on — and it would surface * only once the restore had already started deleting. This is raised at the * boundary, before anything is touched, and carries an i18n key plus the * offending source name so the message can name what to fix. */ export class SrefValidationError extends Error { readonly i18nKey: SrefValidationErrorKey; readonly params: Record; constructor(i18nKey: SrefValidationErrorKey, params: Record) { super(`${i18nKey}: ${JSON.stringify(params)}`); this.name = "SrefValidationError"; this.i18nKey = i18nKey; this.params = params; } } // --- Data gathering --- export async function getExportCategories(): Promise { const db = await getDb(); return db.select("SELECT * FROM categories ORDER BY id"); } export async function getExportSuppliers(): Promise { const db = await getDb(); return db.select("SELECT * FROM suppliers ORDER BY id"); } export async function getExportKeywords(): Promise { const db = await getDb(); return db.select("SELECT * FROM keywords ORDER BY id"); } export async function getExportTransactions(): Promise { const db = await getDb(); return db.select( `SELECT t.id, t.date, t.description, t.amount, t.category_id, c.name AS category_name, t.original_description, t.notes, t.is_manually_categorized, t.is_split, t.parent_transaction_id FROM transactions t LEFT JOIN categories c ON t.category_id = c.id ORDER BY t.date, t.id` ); } /** * Every configured import source, with its template resolved to a name. * * This is what the export was missing (#331): the file carried transactions but * not a single line of how they had been read, so restoring a backup meant * reconfiguring every source by hand. */ export async function getExportImportSources(): Promise { const db = await getDb(); const rows = await db.select< Array >( `SELECT s.*, t.name AS template_name FROM import_sources s LEFT JOIN import_config_templates t ON s.template_id = t.id ORDER BY s.id` ); return rows.map((row) => ({ name: row.name, description: row.description ?? null, header_signature: row.header_signature ?? null, template_name: row.template_name ?? null, ...pickFormatRow(row as unknown as Record), })); } export async function getExportImportTemplates(): Promise< ExportImportTemplate[] > { const db = await getDb(); const rows = await db.select( "SELECT * FROM import_config_templates ORDER BY id" ); return rows.map((row) => ({ name: row.name, ...pickFormatRow(row as unknown as Record), })); } // --- Serialization --- export function serializeToJson( exportType: ExportMode, data: ExportEnvelope["data"], appVersion: string ): string { const envelope: ExportEnvelope = { export_type: exportType, app_version: appVersion, format_version: SREF_FORMAT_VERSION, exported_at: new Date().toISOString(), data, }; return JSON.stringify(envelope, null, 2); } export function serializeTransactionsToCsv( transactions: ExportTransaction[] ): string { return Papa.unparse( transactions.map((t) => ({ date: t.date, description: t.description, amount: t.amount, category_name: t.category_name ?? "", category_id: t.category_id ?? "", original_description: t.original_description ?? "", notes: t.notes ?? "", is_manually_categorized: t.is_manually_categorized, is_split: t.is_split, parent_transaction_id: t.parent_transaction_id ?? "", })) ); } // --- Import parsing --- /** * Refuse a configuration this build cannot store, BEFORE anything is deleted. * * Only the two enumerated fields are checked. `column_mapping` deliberately is * not decoded: a profile that already went through an old restore holds sources * with `'{}'` there, and refusing to restore a backup because of a row the app * itself wrote would strand the user with no way back in. It is carried through * as the opaque string the column stores, exactly as it was found — the wizard * validates it when it is actually about to read a file. * * PURE: no I/O, so it can be called from the parse step and again at the top of * each restore, which is where the guarantee has to hold. */ export function validateImportedFormatRows( sources: ExportImportSource[] | undefined, templates: ExportImportTemplate[] | undefined ): void { const check = (row: ImportFormatRow, name: unknown) => { const label = typeof name === "string" && name.length > 0 ? name : "?"; if (typeof row?.column_mapping !== "string") { throw new SrefValidationError( "settings.dataManagement.import.errors.invalidFormatRow", { name: label } ); } if (!AMOUNT_MODES.includes(row.amount_mode)) { throw new SrefValidationError( "settings.dataManagement.import.errors.unsupportedAmountMode", { name: label, value: String(row.amount_mode) } ); } if (!SIGN_CONVENTIONS.includes(row.sign_convention)) { throw new SrefValidationError( "settings.dataManagement.import.errors.unsupportedSignConvention", { name: label, value: String(row.sign_convention) } ); } }; for (const template of templates ?? []) check(template, template?.name); for (const source of sources ?? []) check(source, source?.name); } export function parseImportedJson(content: string): { envelope: ExportEnvelope; summary: ImportSummary; } { let envelope: ExportEnvelope; try { envelope = JSON.parse(content); } catch { throw new Error("Invalid JSON file"); } if ( !envelope.export_type || !envelope.data || typeof envelope.data !== "object" ) { throw new Error("Invalid export file format — missing required fields"); } const validTypes: ExportMode[] = [ "transactions_with_categories", "transactions_only", "categories_only", ]; if (!validTypes.includes(envelope.export_type)) { throw new Error(`Unknown export type: ${envelope.export_type}`); } // Refuse an unreadable configuration while the file is only being LOOKED at: // the confirmation dialog never opens, so nothing is deleted. validateImportedFormatRows( envelope.data.import_sources, envelope.data.import_config_templates ); return { envelope, summary: { type: envelope.export_type, categoriesCount: envelope.data.categories?.length ?? 0, suppliersCount: envelope.data.suppliers?.length ?? 0, keywordsCount: envelope.data.keywords?.length ?? 0, transactionsCount: envelope.data.transactions?.length ?? 0, importSourcesCount: envelope.data.import_sources?.length ?? 0, importTemplatesCount: envelope.data.import_config_templates?.length ?? 0, formatVersion: envelope.format_version ?? LEGACY_SREF_FORMAT_VERSION, }, }; } export function parseImportedCsv(content: string): { transactions: ExportTransaction[]; summary: ImportSummary; } { const result = Papa.parse>(content, { header: true, skipEmptyLines: true, }); if (result.errors.length > 0 && result.data.length === 0) { throw new Error(`CSV parse error: ${result.errors[0].message}`); } const transactions: ExportTransaction[] = result.data.map((row, i) => ({ id: i, date: row.date ?? "", description: row.description ?? "", amount: parseFloat(row.amount) || 0, category_id: row.category_id ? parseInt(row.category_id) : null, category_name: row.category_name || null, original_description: row.original_description || null, notes: row.notes || null, // An export from the time of #390 carries "true" in this column. is_manually_categorized: toSqlFlag(row.is_manually_categorized), is_split: parseInt(row.is_split) || 0, parent_transaction_id: row.parent_transaction_id ? parseInt(row.parent_transaction_id) : null, })); return { transactions, summary: { type: "transactions_only", categoriesCount: 0, suppliersCount: 0, keywordsCount: 0, transactionsCount: transactions.length, // A flat CSV carries no configuration at all. importSourcesCount: 0, importTemplatesCount: 0, formatVersion: LEGACY_SREF_FORMAT_VERSION, }, }; } // --- Import execution --- /** * The source that hosts transactions restored from a file, which describe no * import folder of their own. * * Its mapping used to be `'{}'`, which `formatFromRow` cannot decode — a source * the app itself wrote and then refused to read. It describes the CSV this very * service exports instead: date, description, amount, comma-separated, ISO * dates, expenses negative. That is true of the file the rows came from, so the * wizard finds a usable format if a folder ever bears this name. */ const HOST_SOURCE_NAME = "Data Import"; const HOST_SOURCE_MAPPING = JSON.stringify({ date: 0, description: 1, amount: 2, }); /** * Run `body` as one all-or-nothing restore. * * Every one of these functions starts by DELETING the profile's financial * history and then re-inserts it row by row, against tables carrying * `UNIQUE(name)` and a foreign key. Without a transaction, the first constraint * violation leaves the profile emptied at whatever row it reached, with nothing * to go back to. `withTransaction` holds the single DB lock for the whole * BEGIN..COMMIT so the statements cannot be split across pooled connections * (see `db.ts`); the BEGIN/COMMIT/ROLLBACK themselves are ours to issue. */ async function runRestore( body: (db: Database) => Promise ): Promise { return withTransaction(async (db) => { await db.execute("BEGIN"); let open = true; try { await body(db); await db.execute("COMMIT"); open = false; } catch (e) { if (open) { try { await db.execute("ROLLBACK"); } catch { // Preserve the original error — it is the one that explains the failure. } } throw e; } }); } async function restoreCategories( db: Database, categories: Category[] | undefined ): Promise { for (const cat of categories ?? []) { await db.execute( `INSERT INTO categories (id, name, parent_id, color, icon, type, is_active, is_inputable, sort_order) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, [ cat.id, cat.name, cat.parent_id ?? null, cat.color ?? null, cat.icon ?? null, cat.type, cat.is_active ? 1 : 0, cat.is_inputable ? 1 : 0, cat.sort_order, ] ); } } async function restoreSuppliers( db: Database, suppliers: Supplier[] | undefined ): Promise { for (const sup of suppliers ?? []) { await db.execute( `INSERT INTO suppliers (id, name, normalized_name, category_id, is_active) VALUES ($1, $2, $3, $4, $5)`, [sup.id, sup.name, sup.normalized_name, sup.category_id ?? null, sup.is_active ? 1 : 0] ); } } async function restoreKeywords( db: Database, keywords: Keyword[] | undefined ): Promise { for (const kw of keywords ?? []) { await db.execute( `INSERT INTO keywords (id, keyword, category_id, supplier_id, priority, is_active) VALUES ($1, $2, $3, $4, $5, $6)`, [kw.id, kw.keyword, kw.category_id, kw.supplier_id ?? null, kw.priority, kw.is_active ? 1 : 0] ); } } /** * Restore the templates and return their resolved ids, keyed by name. * * `import_config_templates` is in NEITHER wipe list, and restoring into a * profile that already has templates is the normal path, not an edge case — a * plain INSERT would hit `UNIQUE constraint failed: import_config_templates.name` * and (now) roll the whole restore back. Wiping the table instead was the other * option and was rejected: it would destroy templates the user never asked to * replace and that no confirmation dialog mentions. So: upsert by name, and give * the caller the map it needs to remap `import_sources.template_id`, since the * ids in the file mean nothing here. * * Templates go in BEFORE sources — `template_id` is a foreign key to this table. */ async function restoreImportTemplates( db: Database, templates: ExportImportTemplate[] | undefined ): Promise> { const idsByName = new Map(); for (const template of templates ?? []) { await db.execute( `INSERT INTO import_config_templates (name, delimiter, encoding, date_format, skip_lines, has_header, column_mapping, amount_mode, sign_convention) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT(name) DO UPDATE SET delimiter = excluded.delimiter, encoding = excluded.encoding, date_format = excluded.date_format, skip_lines = excluded.skip_lines, has_header = excluded.has_header, column_mapping = excluded.column_mapping, amount_mode = excluded.amount_mode, sign_convention = excluded.sign_convention`, [ template.name, template.delimiter, template.encoding, template.date_format, template.skip_lines, template.has_header, template.column_mapping, template.amount_mode, template.sign_convention, ] ); // `lastInsertId` is 0 on the conflict branch, so the id is read back by // name — the same reason `importSourceService.createSource` does it. const rows = await db.select>( "SELECT id FROM import_config_templates WHERE name = $1", [template.name] ); if (rows.length > 0) idsByName.set(template.name, rows[0].id); } return idsByName; } /** * Restore the sources, resolving each `template_name` to the id the templates * pass just settled. A name with no match becomes NULL: `template_id` is a * provenance tag, never re-read as format, so losing it costs no configuration. * * Ids are not carried over — nothing in the envelope refers to a source by id. */ async function restoreImportSources( db: Database, sources: ExportImportSource[] | undefined, templateIds: Map ): Promise { for (const source of sources ?? []) { await db.execute( `INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention, header_signature, template_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, [ source.name, source.description ?? null, source.date_format, source.delimiter, source.encoding, source.column_mapping, source.skip_lines, source.has_header, source.amount_mode, source.sign_convention, source.header_signature ?? null, source.template_name !== null ? templateIds.get(source.template_name) ?? null : null, ] ); } } /** * Attach the restored transactions to a source and one `imported_files` row. * * The host source is created ONLY here, and only when there are transactions to * hang off it — it exists to satisfy `transactions.source_id`, not to stand in * for the configurations the file now carries on its own. A restored source may * already bear the name (an earlier restore of this same profile wrote one), in * which case it is reused rather than re-inserted: `import_sources.name` is * UNIQUE, and a collision here would roll back the entire restore. */ async function attachTransactions( db: Database, transactions: ExportTransaction[] | undefined, filename: string ): Promise { if (!transactions || transactions.length === 0) return; const existing = await db.select>( "SELECT id FROM import_sources WHERE name = $1", [HOST_SOURCE_NAME] ); let sourceId: number; if (existing.length > 0) { sourceId = existing[0].id; } else { const sourceResult = await db.execute( `INSERT INTO import_sources (name, description, date_format, delimiter, encoding, column_mapping, skip_lines, has_header, amount_mode, sign_convention) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, [ HOST_SOURCE_NAME, "Imported from settings", "%Y-%m-%d", ",", "utf-8", HOST_SOURCE_MAPPING, 0, 1, "single", "negative_expense", ] ); sourceId = sourceResult.lastInsertId as number; } const fileResult = await db.execute( `INSERT INTO imported_files (source_id, filename, file_hash, row_count, status) VALUES ($1, $2, $3, $4, $5)`, [sourceId, filename, `data-import-${Date.now()}`, transactions.length, "completed"] ); const fileId = fileResult.lastInsertId; for (const tx of transactions) { await db.execute( `INSERT INTO transactions (date, description, amount, category_id, original_description, notes, is_manually_categorized, is_split, parent_transaction_id, source_id, file_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, [ tx.date, tx.description, tx.amount, tx.category_id, tx.original_description, tx.notes, // A JSON backup from the time of #390 carries the text "true". toSqlFlag(tx.is_manually_categorized), tx.is_split, tx.parent_transaction_id, sourceId, fileId, ] ); } } export async function importCategoriesOnly( data: ExportEnvelope["data"] ): Promise { return runRestore(async (db) => { // Wipe keywords, suppliers, categories await db.execute("DELETE FROM keywords"); await db.execute("DELETE FROM suppliers"); await db.execute("DELETE FROM categories"); // Nullify category/supplier references on transactions await db.execute( "UPDATE transactions SET category_id = NULL, supplier_id = NULL, is_manually_categorized = 0" ); await restoreCategories(db, data.categories); await restoreSuppliers(db, data.suppliers); await restoreKeywords(db, data.keywords); }); } export async function importTransactionsWithCategories( data: ExportEnvelope["data"], filename: string ): Promise { validateImportedFormatRows(data.import_sources, data.import_config_templates); return runRestore(async (db) => { // Wipe everything await db.execute("DELETE FROM transactions"); await db.execute("DELETE FROM imported_files"); await db.execute("DELETE FROM import_sources"); await db.execute("DELETE FROM keywords"); await db.execute("DELETE FROM suppliers"); await db.execute("DELETE FROM categories"); await restoreCategories(db, data.categories); await restoreSuppliers(db, data.suppliers); await restoreKeywords(db, data.keywords); // Templates first: `import_sources.template_id` points at them. const templateIds = await restoreImportTemplates( db, data.import_config_templates ); await restoreImportSources(db, data.import_sources, templateIds); await attachTransactions(db, data.transactions, filename); }); } export async function importTransactionsOnly( data: ExportEnvelope["data"], filename: string ): Promise { validateImportedFormatRows(data.import_sources, data.import_config_templates); return runRestore(async (db) => { // Wipe transactions and import history await db.execute("DELETE FROM transactions"); await db.execute("DELETE FROM imported_files"); await db.execute("DELETE FROM import_sources"); const templateIds = await restoreImportTemplates( db, data.import_config_templates ); await restoreImportSources(db, data.import_sources, templateIds); await attachTransactions(db, data.transactions, filename); }); }