feat(onboarding): create_sample_import_folders command, sample CSV templates and detection test (#354)
Add a Tauri command that writes two synthetic bank statements (Desjardins
chequing account, Tangerine credit card) into the import folder, one
sub-folder each, so a new user has sources to import on the first run.
The command takes the import folder and the webview's local date only
(chrono is compiled without `clock`), renders the `{{M1}}` / `{{M2}}`
tokens as the two complete months before that date, names each file
`releve-YYYY-MM.csv` after the most recent one, and refuses to run when
the folder is missing or when either sample folder already exists -
checked before anything is written. It returns the two folder names.
The templates live under `src/__fixtures__/samples/` so the Rust command
(`include_str!`) and vitest read the same bytes. The integration test
proves detection recognises each file as its bank with a signed single
amount and reads 40/40 rows, that the production `mapRow` parses every
row and sums to +740.00 / -368.46, and that the 180 seed keywords
categorise exactly 33 / 36 lines while leaving the hand-classified ones
untouched. The frozen detection corpus (`src/__fixtures__/csv`) is not
modified.
Five Rust tests cover the missing folder, the existing folder, the
success path, an unparseable date and the year boundary, each in its own
sandbox so they run in parallel.
CHANGELOG and architecture docs are centralised in #357.
Resolves #354
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
8eac47f69b
commit
e4a0f1042e
6 changed files with 607 additions and 0 deletions
|
|
@ -1,3 +1,4 @@
|
|||
use chrono::{Datelike, Months, NaiveDate};
|
||||
use encoding_rs::{UTF_8, WINDOWS_1252, ISO_8859_15};
|
||||
use serde::Serialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
|
@ -172,6 +173,91 @@ pub async fn pick_folder(app: tauri::AppHandle) -> Result<Option<String>, String
|
|||
Ok(folder.map(|f| f.to_string()))
|
||||
}
|
||||
|
||||
/// Folder names the onboarding samples are written under — the same in FR and
|
||||
/// EN (decided in planning), so the scan shows one source per bank whatever the
|
||||
/// UI language.
|
||||
const SAMPLE_DESJARDINS_FOLDER: &str = "Exemple Desjardins";
|
||||
const SAMPLE_TANGERINE_FOLDER: &str = "Exemple Tangerine";
|
||||
|
||||
/// Sample statements embedded at compile time. The paths are relative to
|
||||
/// `src-tauri/src/commands/`; the same files are read by vitest, so the Rust
|
||||
/// command and the detection test cannot drift apart.
|
||||
const SAMPLE_DESJARDINS_TEMPLATE: &str =
|
||||
include_str!("../../../src/__fixtures__/samples/exemple-desjardins.csv");
|
||||
const SAMPLE_TANGERINE_TEMPLATE: &str =
|
||||
include_str!("../../../src/__fixtures__/samples/exemple-tangerine.csv");
|
||||
|
||||
/// Write the two onboarding sample statements (a Desjardins chequing account and
|
||||
/// a Tangerine credit card, both synthetic) into `import_folder`, one sub-folder
|
||||
/// each, so that the very next scan of the import folder shows two sources a
|
||||
/// new user can import right away.
|
||||
///
|
||||
/// `today` is the webview's local date as `YYYY-MM-DD`; chrono is compiled
|
||||
/// without `clock`, so the command never reads the system clock and stays
|
||||
/// deterministic. The templates carry `{{M1}}` / `{{M2}}` tokens, rendered as
|
||||
/// `MM/YYYY` of the two complete months before `today` (`{{M1}}` the most
|
||||
/// recent), and each file is named `releve-YYYY-MM.csv` after `{{M1}}`.
|
||||
///
|
||||
/// Refuses to run when `import_folder` is not a directory, and when either
|
||||
/// sample folder already exists — checked BEFORE anything is written, so an
|
||||
/// existing folder is never touched. The command only ever receives the import
|
||||
/// folder and a date: the names of what it writes are fixed here. Returns the
|
||||
/// two folder NAMES (not paths).
|
||||
#[tauri::command]
|
||||
pub fn create_sample_import_folders(
|
||||
import_folder: String,
|
||||
today: String,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let today = NaiveDate::parse_from_str(&today, "%Y-%m-%d")
|
||||
.map_err(|e| format!("Invalid date '{}': {}", today, e))?;
|
||||
let first_of_month = today
|
||||
.with_day(1)
|
||||
.ok_or_else(|| "Cannot compute the first day of the month".to_string())?;
|
||||
let m1 = first_of_month
|
||||
.checked_sub_months(Months::new(1))
|
||||
.ok_or_else(|| "Cannot compute the previous month".to_string())?;
|
||||
let m2 = first_of_month
|
||||
.checked_sub_months(Months::new(2))
|
||||
.ok_or_else(|| "Cannot compute the month before the previous one".to_string())?;
|
||||
|
||||
let root = Path::new(&import_folder);
|
||||
if !root.is_dir() {
|
||||
return Err(format!("Folder does not exist: {}", import_folder));
|
||||
}
|
||||
|
||||
let samples: [(&str, &str); 2] = [
|
||||
(SAMPLE_DESJARDINS_FOLDER, SAMPLE_DESJARDINS_TEMPLATE),
|
||||
(SAMPLE_TANGERINE_FOLDER, SAMPLE_TANGERINE_TEMPLATE),
|
||||
];
|
||||
|
||||
// Both existence checks run before the first write: a user who already has
|
||||
// one of the folders keeps it untouched and gets neither.
|
||||
for (folder_name, _) in &samples {
|
||||
if root.join(folder_name).exists() {
|
||||
return Err(format!("Folder already exists: {}", folder_name));
|
||||
}
|
||||
}
|
||||
|
||||
let m1_token = m1.format("%m/%Y").to_string();
|
||||
let m2_token = m2.format("%m/%Y").to_string();
|
||||
let filename = format!("releve-{}.csv", m1.format("%Y-%m"));
|
||||
|
||||
let mut created: Vec<String> = Vec::with_capacity(samples.len());
|
||||
for (folder_name, template) in &samples {
|
||||
let folder = root.join(folder_name);
|
||||
fs::create_dir(&folder)
|
||||
.map_err(|e| format!("Cannot create folder {}: {}", folder_name, e))?;
|
||||
let content = template
|
||||
.replace("{{M1}}", &m1_token)
|
||||
.replace("{{M2}}", &m2_token);
|
||||
fs::write(folder.join(&filename), content)
|
||||
.map_err(|e| format!("Cannot write {}/{}: {}", folder_name, filename, e))?;
|
||||
created.push((*folder_name).to_string());
|
||||
}
|
||||
|
||||
Ok(created)
|
||||
}
|
||||
|
||||
fn decode_bytes(bytes: &[u8], encoding: &str) -> Result<String, String> {
|
||||
// Strip BOM if present
|
||||
let bytes = if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
|
||||
|
|
@ -207,3 +293,137 @@ fn decode_bytes(bytes: &[u8], encoding: &str) -> Result<String, String> {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod sample_folder_tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// One scratch directory per CASE: the cases of this binary run in parallel
|
||||
/// inside a single process, so `process::id()` alone would make them share
|
||||
/// — and trip over — the same folder.
|
||||
fn sandbox(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"simpl-resultat-samples-{}-{}",
|
||||
std::process::id(),
|
||||
name
|
||||
));
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
fs::create_dir_all(&dir).expect("sandbox created");
|
||||
dir
|
||||
}
|
||||
|
||||
fn cleanup(dir: &Path) {
|
||||
let _ = fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
fn as_arg(path: &Path) -> String {
|
||||
path.to_string_lossy().to_string()
|
||||
}
|
||||
|
||||
/// Every `DD/MM/YYYY` date of a rendered file must fall in one of the two
|
||||
/// expected months — no token left, no third month.
|
||||
fn assert_dates_in(content: &str, months: &[&str]) {
|
||||
assert!(!content.contains("{{"), "token left in {}", content);
|
||||
for line in content.lines().skip(1) {
|
||||
let date = line.split([';', ',']).next().unwrap();
|
||||
assert_eq!(date.len(), 10, "unexpected date {}", date);
|
||||
let month_year = &date[3..];
|
||||
assert!(
|
||||
months.contains(&month_year),
|
||||
"date {} is outside {:?}",
|
||||
date,
|
||||
months
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_a_missing_import_folder() {
|
||||
let dir = sandbox("missing");
|
||||
let missing = dir.join("nope");
|
||||
let result = create_sample_import_folders(as_arg(&missing), "2026-09-25".to_string());
|
||||
cleanup(&dir);
|
||||
let err = result.unwrap_err();
|
||||
assert!(err.contains("does not exist"), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_when_desjardins_folder_exists_and_writes_nothing() {
|
||||
let dir = sandbox("existing");
|
||||
fs::create_dir(dir.join(SAMPLE_DESJARDINS_FOLDER)).unwrap();
|
||||
let result = create_sample_import_folders(as_arg(&dir), "2026-09-25".to_string());
|
||||
let tangerine_created = dir.join(SAMPLE_TANGERINE_FOLDER).exists();
|
||||
let desjardins_empty = fs::read_dir(dir.join(SAMPLE_DESJARDINS_FOLDER))
|
||||
.unwrap()
|
||||
.next()
|
||||
.is_none();
|
||||
cleanup(&dir);
|
||||
let err = result.unwrap_err();
|
||||
assert!(err.contains(SAMPLE_DESJARDINS_FOLDER), "{}", err);
|
||||
assert!(!tangerine_created, "Exemple Tangerine must not be created");
|
||||
assert!(desjardins_empty, "the existing folder must stay untouched");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_both_folders_for_the_two_previous_months() {
|
||||
let dir = sandbox("success");
|
||||
let result = create_sample_import_folders(as_arg(&dir), "2026-09-25".to_string());
|
||||
let read = |folder: &str| {
|
||||
fs::read_to_string(dir.join(folder).join("releve-2026-08.csv"))
|
||||
};
|
||||
let desjardins = read(SAMPLE_DESJARDINS_FOLDER);
|
||||
let tangerine = read(SAMPLE_TANGERINE_FOLDER);
|
||||
let desjardins_count = fs::read_dir(dir.join(SAMPLE_DESJARDINS_FOLDER)).unwrap().count();
|
||||
let tangerine_count = fs::read_dir(dir.join(SAMPLE_TANGERINE_FOLDER)).unwrap().count();
|
||||
cleanup(&dir);
|
||||
|
||||
assert_eq!(
|
||||
result.unwrap(),
|
||||
vec![
|
||||
SAMPLE_DESJARDINS_FOLDER.to_string(),
|
||||
SAMPLE_TANGERINE_FOLDER.to_string()
|
||||
]
|
||||
);
|
||||
let desjardins = desjardins.expect("Desjardins file written");
|
||||
let tangerine = tangerine.expect("Tangerine file written");
|
||||
assert_eq!(desjardins_count, 1);
|
||||
assert_eq!(tangerine_count, 1);
|
||||
assert_eq!(desjardins.lines().count(), 41);
|
||||
assert_eq!(tangerine.lines().count(), 41);
|
||||
assert_dates_in(&desjardins, &["08/2026", "07/2026"]);
|
||||
assert_dates_in(&tangerine, &["08/2026", "07/2026"]);
|
||||
assert!(desjardins.contains("08/2026") && desjardins.contains("07/2026"));
|
||||
assert!(tangerine.contains("08/2026") && tangerine.contains("07/2026"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_an_unparseable_date_before_writing() {
|
||||
let dir = sandbox("bad-date");
|
||||
let result = create_sample_import_folders(as_arg(&dir), "pas-une-date".to_string());
|
||||
let anything_written = fs::read_dir(&dir).unwrap().next().is_some();
|
||||
cleanup(&dir);
|
||||
let err = result.unwrap_err();
|
||||
assert!(err.contains("Invalid date"), "{}", err);
|
||||
assert!(!anything_written);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crosses_the_year_boundary() {
|
||||
let dir = sandbox("year-boundary");
|
||||
let result = create_sample_import_folders(as_arg(&dir), "2027-01-15".to_string());
|
||||
let desjardins =
|
||||
fs::read_to_string(dir.join(SAMPLE_DESJARDINS_FOLDER).join("releve-2026-12.csv"));
|
||||
let tangerine =
|
||||
fs::read_to_string(dir.join(SAMPLE_TANGERINE_FOLDER).join("releve-2026-12.csv"));
|
||||
cleanup(&dir);
|
||||
|
||||
assert!(result.is_ok());
|
||||
let desjardins = desjardins.expect("Desjardins file named after December");
|
||||
let tangerine = tangerine.expect("Tangerine file named after December");
|
||||
assert_dates_in(&desjardins, &["12/2026", "11/2026"]);
|
||||
assert_dates_in(&tangerine, &["12/2026", "11/2026"]);
|
||||
assert!(desjardins.contains("12/2026") && desjardins.contains("11/2026"));
|
||||
assert!(tangerine.contains("12/2026") && tangerine.contains("11/2026"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -439,6 +439,7 @@ pub fn run() {
|
|||
commands::detect_encoding,
|
||||
commands::get_file_preview,
|
||||
commands::pick_folder,
|
||||
commands::create_sample_import_folders,
|
||||
commands::pick_save_file,
|
||||
commands::pick_import_file,
|
||||
commands::write_export_file,
|
||||
|
|
|
|||
41
src/__fixtures__/samples/exemple-desjardins.csv
Normal file
41
src/__fixtures__/samples/exemple-desjardins.csv
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
Date;Description;Montant;Solde
|
||||
01/{{M2}};DEPOT PAIE EMPLOYEUR;1842,50;4258,30
|
||||
01/{{M2}};MTG/HYP PRET HYPOTHECAIRE;-1236,00;3022,30
|
||||
02/{{M2}};HYDRO QUEBEC PREAUTORISE;-118,42;2903,88
|
||||
03/{{M2}};METRO PLUS ST-LAMBERT;-96,18;2807,70
|
||||
05/{{M2}};VIDEOTRON;-89,95;2717,75
|
||||
06/{{M2}};STM OPUS EN LIGNE;-97,00;2620,75
|
||||
08/{{M2}};IGA EXTRA;-132,47;2488,28
|
||||
09/{{M2}};SHELL BOUL TASCHEREAU;-62,30;2425,98
|
||||
10/{{M2}};FIZZ MOBILE;-34,49;2391,49
|
||||
12/{{M2}};JEAN COUTU;-27,85;2363,64
|
||||
13/{{M2}};GARDERIE LES PETITS PAS;-410,00;1953,64
|
||||
15/{{M2}};DEPOT PAIE EMPLOYEUR;1842,50;3796,14
|
||||
16/{{M2}};BELAIR DIRECT ASSURANCE;-94,60;3701,54
|
||||
17/{{M2}};COSTCO WHOLESALE;-218,73;3482,81
|
||||
19/{{M2}};VIREMENT INTERAC RECU;120,00;3602,81
|
||||
20/{{M2}};PAIEMENT VISA TANGERINE;-650,00;2952,81
|
||||
22/{{M2}};CLINIQUE DENTAIRE ST-LAMBERT;-185,00;2767,81
|
||||
24/{{M2}};METRO PLUS ST-LAMBERT;-74,62;2693,19
|
||||
27/{{M2}};RESTAURANT LA BANQUISE;-48,90;2644,29
|
||||
28/{{M2}};FRAIS MENSUELS;-6,95;2637,34
|
||||
01/{{M1}};DEPOT PAIE EMPLOYEUR;1842,50;4479,84
|
||||
01/{{M1}};MTG/HYP PRET HYPOTHECAIRE;-1236,00;3243,84
|
||||
02/{{M1}};HYDRO QUEBEC PREAUTORISE;-104,17;3139,67
|
||||
04/{{M1}};METRO PLUS ST-LAMBERT;-112,36;3027,31
|
||||
05/{{M1}};VIDEOTRON;-89,95;2937,36
|
||||
06/{{M1}};STM OPUS EN LIGNE;-97,00;2840,36
|
||||
07/{{M1}};MAXI ET CIE;-88,41;2751,95
|
||||
09/{{M1}};ULTRAMAR;-58,75;2693,20
|
||||
10/{{M1}};FIZZ MOBILE;-34,49;2658,71
|
||||
11/{{M1}};FAMILIPRIX;-41,20;2617,51
|
||||
13/{{M1}};GARDERIE LES PETITS PAS;-410,00;2207,51
|
||||
15/{{M1}};DEPOT PAIE EMPLOYEUR;1842,50;4050,01
|
||||
16/{{M1}};BELAIR DIRECT ASSURANCE;-94,60;3955,41
|
||||
18/{{M1}};IGA EXTRA;-147,09;3808,32
|
||||
19/{{M1}};RETRAITE QUEBEC ALLOC FAM;232,00;4040,32
|
||||
20/{{M1}};PAIEMENT VISA TANGERINE;-712,00;3328,32
|
||||
21/{{M1}};CANAC;-63,28;3265,04
|
||||
23/{{M1}};METRO PLUS ST-LAMBERT;-69,84;3195,20
|
||||
26/{{M1}};SUSHI SHOP;-32,45;3162,75
|
||||
28/{{M1}};FRAIS MENSUELS;-6,95;3155,80
|
||||
|
41
src/__fixtures__/samples/exemple-tangerine.csv
Normal file
41
src/__fixtures__/samples/exemple-tangerine.csv
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
Date,Transaction,Name,Memo,Amount
|
||||
02/{{M2}},DEBIT,COSTCO WHOLESALE,,-164.32
|
||||
03/{{M2}},DEBIT,TIM HORTONS,,-6.45
|
||||
04/{{M2}},DEBIT,NETFLIX.COM,,-20.99
|
||||
05/{{M2}},DEBIT,AMAZON.CA,,-47.88
|
||||
06/{{M2}},DEBIT,DOLLARAMA,,-18.35
|
||||
08/{{M2}},DEBIT,STARBUCKS,,-7.90
|
||||
09/{{M2}},DEBIT,DOORDASH,,-38.64
|
||||
10/{{M2}},DEBIT,SPOTIFY,,-11.99
|
||||
11/{{M2}},DEBIT,UNIQLO,,-89.95
|
||||
12/{{M2}},DEBIT,SAQ,,-43.50
|
||||
14/{{M2}},DEBIT,MCDONALD'S,,-14.27
|
||||
15/{{M2}},DEBIT,PHARMAPRIX,,-31.18
|
||||
16/{{M2}},DEBIT,CANADIAN TIRE,,-76.42
|
||||
18/{{M2}},DEBIT,RESTAURANT LE PETIT ITALIEN,,-72.30
|
||||
19/{{M2}},DEBIT,ESSO,,-61.05
|
||||
20/{{M2}},CREDIT,PAIEMENT - MERCI,,650.00
|
||||
21/{{M2}},DEBIT,SEPAQ,,-38.00
|
||||
23/{{M2}},DEBIT,RENAUD-BRAY,,-29.95
|
||||
25/{{M2}},DEBIT,UBER TRIP,,-17.60
|
||||
27/{{M2}},DEBIT,CINEPLEX,,-28.50
|
||||
01/{{M1}},DEBIT,COSTCO WHOLESALE,,-201.17
|
||||
03/{{M1}},DEBIT,TIM HORTONS,,-5.85
|
||||
04/{{M1}},DEBIT,NETFLIX.COM,,-20.99
|
||||
05/{{M1}},DEBIT,AMAZON.CA,,-32.49
|
||||
07/{{M1}},DEBIT,DOLLARAMA,,-12.40
|
||||
08/{{M1}},DEBIT,STARBUCKS,,-8.35
|
||||
09/{{M1}},DEBIT,DOORDASH,,-42.11
|
||||
10/{{M1}},DEBIT,SPOTIFY,,-11.99
|
||||
11/{{M1}},DEBIT,SIMONS,,-124.60
|
||||
12/{{M1}},DEBIT,SAQ,,-38.75
|
||||
13/{{M1}},DEBIT,A&W,,-16.90
|
||||
15/{{M1}},DEBIT,PHARMAPRIX,,-24.66
|
||||
16/{{M1}},DEBIT,RONA,,-58.12
|
||||
17/{{M1}},DEBIT,BRASSERIE LES FRERES,,-64.80
|
||||
19/{{M1}},DEBIT,PETRO-CANADA,,-57.40
|
||||
20/{{M1}},CREDIT,PAIEMENT - MERCI,,712.00
|
||||
22/{{M1}},DEBIT,LA CORDEE,,-89.99
|
||||
24/{{M1}},DEBIT,INDIGO,,-34.50
|
||||
26/{{M1}},DEBIT,UBER TRIP,,-21.15
|
||||
27/{{M1}},DEBIT,MOUNTAIN EQUIPMENT CO,,-45.00
|
||||
|
39
src/__fixtures__/samples/index.ts
Normal file
39
src/__fixtures__/samples/index.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
/**
|
||||
* Sample statements the onboarding offers to a new user (issue #354).
|
||||
*
|
||||
* Both files are INVENTED, like the detection corpus in `../csv`: no real
|
||||
* account, no real transaction, no real person. They are the two templates the
|
||||
* Rust command `create_sample_import_folders` embeds with `include_str!` and
|
||||
* writes into the import folder — one Desjardins chequing account, one
|
||||
* Tangerine credit card — with the `{{M1}}` / `{{M2}}` tokens rendered as the
|
||||
* two complete months preceding the day the user clicks.
|
||||
*
|
||||
* This module is deliberately SEPARATE from `../csv/index.ts`: that corpus is
|
||||
* frozen at fifteen cases by `csvAutoDetect.test.ts`, and six loops pass every
|
||||
* `CsvFixtureName` to detection. Adding these two there would silently widen
|
||||
* the frozen contract.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
|
||||
/** The two sample statements, by bank. */
|
||||
export type SampleTemplateName = "desjardins" | "tangerine";
|
||||
|
||||
const FILES: Record<SampleTemplateName, string> = {
|
||||
desjardins: "exemple-desjardins.csv",
|
||||
tangerine: "exemple-tangerine.csv",
|
||||
};
|
||||
|
||||
/** Raw template text, tokens included — exactly what `include_str!` embeds. */
|
||||
export function readSampleTemplate(name: SampleTemplateName): string {
|
||||
return readFileSync(resolve(import.meta.dirname, FILES[name]), "utf-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a template the way the Rust command does: `{{M1}}` becomes the most
|
||||
* recent complete month, `{{M2}}` the one before, both as `MM/YYYY`.
|
||||
*/
|
||||
export function renderSampleTemplate(raw: string, m1: string, m2: string): string {
|
||||
return raw.replace(/\{\{M1\}\}/g, m1).replace(/\{\{M2\}\}/g, m2);
|
||||
}
|
||||
265
src/__integration__/onboarding-samples.test.ts
Normal file
265
src/__integration__/onboarding-samples.test.ts
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
/**
|
||||
* Onboarding samples — detection and categorisation contract (#354).
|
||||
*
|
||||
* `create_sample_import_folders` (Rust) writes two synthetic statements into the
|
||||
* import folder so a new user has something to import on the first run. The
|
||||
* command embeds the templates in `src/__fixtures__/samples/` with
|
||||
* `include_str!`; this file reads the SAME files and proves what the wizard will
|
||||
* do with them once the user clicks "Import":
|
||||
*
|
||||
* 1. detection recognises each file as the bank it imitates, with the
|
||||
* configuration the templates were written for, and reads every row;
|
||||
* 2. the full parse — the production `mapRow`, with the detected config —
|
||||
* yields no row in error and the signed totals the templates were balanced
|
||||
* to;
|
||||
* 3. the 180 seed keywords of a fresh profile (`consolidated_schema.sql`)
|
||||
* categorise the agreed share of each file, and leave the lines the
|
||||
* onboarding wants the user to classify BY HAND uncategorised — that step
|
||||
* of the checklist ("one transaction classified manually") needs something
|
||||
* left to do.
|
||||
*
|
||||
* The counts are FROZEN. If a seed keyword changes and moves one of them, the
|
||||
* expectation here is what tells the author the onboarding sample changed too.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import Papa from "papaparse";
|
||||
import { detectImportFormat, preprocessQuotedCSV } from "../utils/csvAutoDetect";
|
||||
import { detectAmountSeparators, mapRow } from "../utils/importFormat";
|
||||
import type { ParsedRow } from "../shared/types";
|
||||
import {
|
||||
readSampleTemplate,
|
||||
renderSampleTemplate,
|
||||
type SampleTemplateName,
|
||||
} from "../__fixtures__/samples";
|
||||
|
||||
vi.mock("../services/db", () => {
|
||||
const getDb = vi.fn();
|
||||
return {
|
||||
getDb,
|
||||
withTransaction: vi.fn(async (fn: (db: unknown) => unknown) => fn(await getDb())),
|
||||
};
|
||||
});
|
||||
|
||||
import { getDb } from "../services/db";
|
||||
import { categorizeBatch } from "../services/categorizationService";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The two rendered files, as the command writes them for a click on 2026-09-25.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const M1 = "08/2026";
|
||||
const M2 = "07/2026";
|
||||
|
||||
function renderedSample(name: SampleTemplateName): string {
|
||||
return renderSampleTemplate(readSampleTemplate(name), M1, M2);
|
||||
}
|
||||
|
||||
/** Detection outcome of a sample, or a failure the test can name. */
|
||||
function detectSample(name: SampleTemplateName) {
|
||||
const outcome = detectImportFormat(renderedSample(name));
|
||||
if (outcome.status !== "ok") {
|
||||
throw new Error(`${name}: detection returned ${outcome.status}`);
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full pipeline, as the wizard runs it: raw text -> Papa rows -> data rows ->
|
||||
* column-level separator arbitration -> `mapRow` under the detected format.
|
||||
*/
|
||||
function parseSample(name: SampleTemplateName): ParsedRow[] {
|
||||
const rawContent = renderedSample(name);
|
||||
const { config } = detectSample(name);
|
||||
const format = { ...config, encoding: "utf-8" };
|
||||
|
||||
const data = Papa.parse(preprocessQuotedCSV(rawContent), {
|
||||
delimiter: config.delimiter,
|
||||
skipEmptyLines: true,
|
||||
}).data as string[][];
|
||||
|
||||
const startIdx = config.skipLines + (config.hasHeader ? 1 : 0);
|
||||
const dataRows: string[][] = [];
|
||||
for (let i = startIdx; i < data.length; i++) {
|
||||
const raw = data[i];
|
||||
if (raw.length <= 1 && raw[0]?.trim() === "") continue;
|
||||
dataRows.push(raw);
|
||||
}
|
||||
const decimalSeparators = detectAmountSeparators(dataRows, format);
|
||||
return dataRows.map((raw, rowIndex) =>
|
||||
mapRow(raw, format, { rowIndex, decimalSeparators })
|
||||
);
|
||||
}
|
||||
|
||||
/** Sum of the signed amounts, rounded to the cent to absorb float noise. */
|
||||
function sumOf(rows: ParsedRow[]): number {
|
||||
const total = rows.reduce((acc, r) => acc + (r.parsed?.amount ?? 0), 0);
|
||||
return Math.round(total * 100) / 100;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1 + 2 — detection and parse
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const SAMPLES: {
|
||||
name: SampleTemplateName;
|
||||
bank: string;
|
||||
total: number;
|
||||
}[] = [
|
||||
{ name: "desjardins", bank: "desjardins", total: 740.0 },
|
||||
{ name: "tangerine", bank: "tangerine", total: -368.46 },
|
||||
];
|
||||
|
||||
describe("onboarding samples — detection (#354)", () => {
|
||||
it("renders every token: no `{{` survives in the files the user opens", () => {
|
||||
for (const { name } of SAMPLES) {
|
||||
const rendered = renderedSample(name);
|
||||
expect(rendered, name).not.toContain("{{");
|
||||
expect(rendered, name).toContain(M1);
|
||||
expect(rendered, name).toContain(M2);
|
||||
expect(rendered.trimEnd().split("\n"), name).toHaveLength(41);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(SAMPLES)(
|
||||
"$name: recognised as $bank, signed single amount, 40/40 rows read",
|
||||
({ name, bank }) => {
|
||||
const outcome = detectSample(name);
|
||||
expect(outcome.status).toBe("ok");
|
||||
expect(outcome.bank).toBe(bank);
|
||||
expect(outcome.config.signConvention).toBe("negative_expense");
|
||||
expect(outcome.config.amountMode).toBe("single");
|
||||
expect(outcome.config.dateFormat).toBe("DD/MM/YYYY");
|
||||
expect(outcome.score.totalRows).toBe(40);
|
||||
expect(outcome.score.readRows).toBe(40);
|
||||
expect(outcome.score.ratio).toBe(1);
|
||||
expect(outcome.score.confident).toBe(true);
|
||||
}
|
||||
);
|
||||
|
||||
it.each(SAMPLES)(
|
||||
"$name: the full parse has no row in error and sums to $total",
|
||||
({ name, total }) => {
|
||||
const rows = parseSample(name);
|
||||
expect(rows).toHaveLength(40);
|
||||
expect(rows.filter((r) => r.error !== undefined)).toEqual([]);
|
||||
expect(rows.every((r) => r.parsed !== null && r.parsed !== undefined)).toBe(
|
||||
true
|
||||
);
|
||||
expect(sumOf(rows)).toBe(total);
|
||||
}
|
||||
);
|
||||
|
||||
it("desjardins: every date is in 08/2026 or 07/2026 after rendering", () => {
|
||||
const rows = parseSample("desjardins");
|
||||
const months = new Set(rows.map((r) => r.parsed!.date.slice(0, 7)));
|
||||
expect([...months].sort()).toEqual(["2026-07", "2026-08"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3 — categorisation by the seed keywords of a fresh profile
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* The 180 keywords `consolidated_schema.sql` seeds into a new profile, read
|
||||
* from the SQL itself so this test follows the seed rather than a copy of it.
|
||||
* Every INSERT has the same three-value shape; a line that does not match the
|
||||
* pattern would be a seed change this regex has to learn about, so the count is
|
||||
* asserted.
|
||||
*/
|
||||
function seedKeywords() {
|
||||
const sql = readFileSync(
|
||||
resolve(
|
||||
import.meta.dirname,
|
||||
"../../src-tauri/src/database/consolidated_schema.sql"
|
||||
),
|
||||
"utf-8"
|
||||
);
|
||||
const pattern =
|
||||
/INSERT INTO keywords \(keyword, category_id, priority\) VALUES \('((?:[^']|'')*)', (\d+), (\d+)\);/g;
|
||||
return [...sql.matchAll(pattern)].map((m, i) => ({
|
||||
id: i + 1,
|
||||
keyword: m[1].replace(/''/g, "'"),
|
||||
category_id: Number(m[2]),
|
||||
supplier_id: null,
|
||||
priority: Number(m[3]),
|
||||
is_active: 1,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Lines the onboarding wants left for the user to classify by hand. */
|
||||
const OFF_SEED: Record<SampleTemplateName, string[]> = {
|
||||
desjardins: [
|
||||
"GARDERIE LES PETITS PAS",
|
||||
"PAIEMENT VISA TANGERINE",
|
||||
"VIREMENT INTERAC RECU",
|
||||
"CLINIQUE DENTAIRE ST-LAMBERT",
|
||||
"RETRAITE QUEBEC ALLOC FAM",
|
||||
],
|
||||
tangerine: ["DOLLARAMA", "PAIEMENT - MERCI"],
|
||||
};
|
||||
|
||||
const CATEGORISED: Record<SampleTemplateName, number> = {
|
||||
desjardins: 33,
|
||||
tangerine: 36,
|
||||
};
|
||||
|
||||
describe("onboarding samples — seed categorisation (#354)", () => {
|
||||
const select = vi.fn();
|
||||
|
||||
beforeEach(() => {
|
||||
select.mockReset();
|
||||
vi.mocked(getDb).mockResolvedValue({ select, execute: vi.fn() } as never);
|
||||
});
|
||||
|
||||
it("reads the 180 seed keywords from the consolidated schema", () => {
|
||||
const keywords = seedKeywords();
|
||||
expect(keywords).toHaveLength(180);
|
||||
expect(keywords[0]).toMatchObject({ keyword: "METRO", category_id: 1111 });
|
||||
});
|
||||
|
||||
it.each(Object.keys(CATEGORISED) as SampleTemplateName[])(
|
||||
"%s: the seed categorises the frozen share and leaves the hand-classified lines alone",
|
||||
async (name) => {
|
||||
const keywords = seedKeywords();
|
||||
// `categorizeBatch` asks for active keywords by descending priority; the
|
||||
// seed is all priority 0, so the SQL order is the file order.
|
||||
select.mockResolvedValue(
|
||||
[...keywords].sort((a, b) => b.priority - a.priority)
|
||||
);
|
||||
|
||||
const rows = parseSample(name);
|
||||
const descriptions = rows.map((r) => r.parsed!.description);
|
||||
const results = await categorizeBatch(descriptions);
|
||||
|
||||
expect(select).toHaveBeenCalledWith(
|
||||
"SELECT * FROM keywords WHERE is_active = 1 ORDER BY priority DESC"
|
||||
);
|
||||
expect(results).toHaveLength(40);
|
||||
|
||||
const categorised = results.filter((r) => r.category_id !== null).length;
|
||||
expect(categorised).toBe(CATEGORISED[name]);
|
||||
expect(categorised / results.length).toBeGreaterThanOrEqual(0.8);
|
||||
|
||||
// The off-seed lines are present in the file AND left uncategorised.
|
||||
for (const description of OFF_SEED[name]) {
|
||||
const indexes = descriptions
|
||||
.map((d, i) => (d === description ? i : -1))
|
||||
.filter((i) => i >= 0);
|
||||
expect(indexes.length, description).toBeGreaterThan(0);
|
||||
for (const i of indexes) {
|
||||
expect(results[i].category_id, description).toBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
// And they are the ONLY uncategorised lines: nothing else slipped.
|
||||
const uncategorised = descriptions.filter(
|
||||
(_, i) => results[i].category_id === null
|
||||
);
|
||||
expect(new Set(uncategorised)).toEqual(new Set(OFF_SEED[name]));
|
||||
}
|
||||
);
|
||||
});
|
||||
Loading…
Reference in a new issue