Simpl-Resultat/src/utils/csvAutoDetect.ts

import Papa from "papaparse";
import { parseDate } from "./dateParser";
import { parseFrenchAmount } from "./amountParser";
import {
  matchBankSignature,
  type BankSignatureId,
  type BankSignatureMatch,
} from "./bankSignatures";
import {
  CREDIT_INDICATOR_TOKENS,
  DEBIT_INDICATOR_TOKENS,
  matchHeaderColumn,
  matchTransactionHeaders,
  MIN_HEADER_ROLE_MATCHES,
  normalizeHeaderCell,
  type LexicalHeaderMap,
} from "./headerDictionary";
import { detectAmountSeparators, mapRow } from "./importFormat";
import type {
  ColumnMapping,
  AmountMode,
  ImportFormat,
  SignConvention,
} from "../shared/types";

export interface AutoDetectResult {
  delimiter: string;
  hasHeader: boolean;
  skipLines: number;
  dateFormat: string;
  columnMapping: ColumnMapping;
  amountMode: AmountMode;
  signConvention: SignConvention;
}

/**
 * i18n key of a file detection recognised but REFUSES to configure. Refusing is
 * a feature: the alternative for this shape is a config that imports every
 * debit as income (see `findDirectionIndicatorColumn`).
 */
export type AutoDetectRejectionKey = "import.errors.absoluteIndicatorFormat";

/**
 * How well the detected configuration reads the file it was detected from.
 *
 * Detection used to return a configuration without ever testing it: a plausible
 * delimiter, a plausible date column and a plausible amount column produced a
 * result whether or not a single row survived them. The score closes that hole
 * by REPLAYING the configuration — the confidence reported is measured, not
 * asserted.
 *
 * What it does NOT measure is whether the amounts carry the right SIGN. A file
 * of unsigned magnitudes (`all-positive` in the corpus) scores 100 % while every
 * expense imports as income, because every row parses. That is why the threshold
 * only colours the banner and blocks nothing: the preview step (#329) is the
 * real net, and it is traversed at every import whatever the score says.
 */
export interface DetectionScore {
  /** Rows whose date AND amount the configuration reads. */
  readRows: number;
  /** Data rows the replay ran on — header and skipped preamble excluded. */
  totalRows: number;
  /** `readRows / totalRows`, or 0 when the file carries no data row. */
  ratio: number;
  /** `ratio >= CONFIDENCE_THRESHOLD`. */
  confident: boolean;
}

/**
 * Share of rows that must be read for the result to be announced as recognised
 * rather than doubtful (decided in planning). A statement mixing a few unusable
 * lines into an otherwise clean file is common enough that a stricter bar would
 * cry wolf; anything below this is a mapping worth a second look.
 */
export const CONFIDENCE_THRESHOLD = 0.9;

/**
 * The full-fidelity outcome of detection.
 *
 * `autoDetectConfig` keeps its `AutoDetectResult | null` shape for the callers
 * that only need the configuration, but null cannot say WHY: "I could not read
 * this file" and "I read this file and it is a format this version imports
 * backwards" call for different messages. `detectImportFormat` separates them.
 */
export type AutoDetectOutcome =
  | {
      status: "ok";
      config: AutoDetectResult;
      score: DetectionScore;
      /**
       * The bank whose documented layout this file matched, or null when the
       * generic dictionary read the header (#330). Deliberately NOT part of
       * `config`: it says where the configuration came from, it is not one of
       * the fields that decide how a row is read, and it is never persisted as
       * format.
       */
      bank: BankSignatureId | null;
    }
  | { status: "rejected"; reason: AutoDetectRejectionKey }
  | { status: "failed" };

const DATE_FORMATS = [
  "DD/MM/YYYY",
  "MM/DD/YYYY",
  "YYYY-MM-DD",
  "YYYY/MM/DD",
  "DD-MM-YYYY",
  "DD.MM.YYYY",
  "YYYYMMDD",
];

const DELIMITERS = [",", ";", "\t"];

/** Every token a direction-indicator column may hold, both directions. */
const DIRECTION_INDICATOR_TOKENS = new Set([
  ...DEBIT_INDICATOR_TOKENS,
  ...CREDIT_INDICATOR_TOKENS,
]);

/**
 * Detect and unwrap Desjardins-style CSVs where each entire line is
 * wrapped in quotes with "" escaping inside.
 */
export function preprocessQuotedCSV(content: string): string {
  const lines = content.split(/\r?\n/);
  const nonEmpty = lines.filter((l) => l.trim());
  if (nonEmpty.length === 0) return content;

  const isLineQuoted = nonEmpty.every((l) => {
    const t = l.trim();
    return t.startsWith('"') && t.endsWith('"') && t.includes(',""');
  });

  if (!isLineQuoted) return content;

  return lines
    .map((l) => {
      const t = l.trim();
      if (!t) return "";
      return t.slice(1, -1).replace(/""/g, '"');
    })
    .join("\n");
}

/**
 * Analyze raw CSV content and return a suggested configuration,
 * or null if detection fails OR the file is a refused format.
 *
 * Callers that must tell those two apart — the wizard, which owes the user a
 * message — use `detectImportFormat` instead.
 */
export function autoDetectConfig(rawContent: string): AutoDetectResult | null {
  const outcome = detectImportFormat(rawContent);
  return outcome.status === "ok" ? outcome.config : null;
}

/**
 * Analyze raw CSV content and return a suggested configuration, the reason the
 * file is refused, or a plain failure.
 *
 * The pipeline is unchanged in its bones — delimiter, preamble, header, date,
 * numeric columns, amount mode — with a LEXICAL layer in front of it: when the
 * file has a header row, `matchTransactionHeaders` reads its labels and the
 * labels win. Every lexical hint is a preference, never a constraint: a label
 * naming a column the data contradicts (a "Date" column nothing parses as a
 * date, a "Solde" column that is the only amount candidate left) is dropped and
 * the shape heuristic decides, exactly as it did before #327.
 *
 * In FRONT of that lexical layer sits the bank-signature table (#330): a header
 * row matching a documented export layout gets its roles from that layout and
 * the generic dictionary is not consulted at all. The hints it produces are the
 * same kind of preference — the file is reported as recognised, not read
 * differently on trust. An unknown header falls straight through to the
 * dictionary, which is what every file did before.
 */
export function detectImportFormat(rawContent: string): AutoDetectOutcome {
  const failed: AutoDetectOutcome = { status: "failed" };
  const content = preprocessQuotedCSV(rawContent);
  const lines = content.split(/\r?\n/).filter((l) => l.trim());
  if (lines.length < 2) return failed;

  // Step 1: Detect delimiter
  const delimiter = detectDelimiter(lines.slice(0, 10));
  if (!delimiter) return failed;

  const parsed = Papa.parse(content, { delimiter, skipEmptyLines: true });
  const data = parsed.data as string[][];
  if (data.length < 2) return failed;

  // Step 1b: Detect preamble lines to skip
  // Find the expected column count (most frequent count > 1)
  const colCountFreq = new Map<number, number>();
  for (const row of data) {
    const len = row.length;
    if (len <= 1) continue;
    colCountFreq.set(len, (colCountFreq.get(len) || 0) + 1);
  }
  let expectedColCount = 0;
  let maxColFreq = 0;
  for (const [count, freq] of colCountFreq) {
    if (freq > maxColFreq) {
      maxColFreq = freq;
      expectedColCount = count;
    }
  }

  // Skip leading rows that don't match the expected column count
  let skipLines = 0;
  for (let i = 0; i < data.length; i++) {
    if (data[i].length >= expectedColCount) break;
    skipLines++;
  }

  const effectiveData = data.slice(skipLines);
  if (effectiveData.length < 2) return failed;

  // Step 2: Detect header
  const hasHeader = detectHeader(effectiveData[0]);

  // Step 2b: Try the known banks first (#330). A signature is claimed on the
  // WHOLE normalized label, delimiter and preamble included, so an unknown file
  // simply does not match and the generic dictionary reads it as before. A
  // headerless file matches nothing by construction — there is no label to read.
  const signature = hasHeader
    ? matchBankSignature({
        headerRow: effectiveData[0],
        delimiter,
        skipLines,
        // `preprocessQuotedCSV` returns its input untouched when the file is not
        // whole-line-quoted, so this comparison IS the quirk.
        wholeLineQuoted: content !== rawContent,
      })
    : null;

  // Step 2c: Read the header labels. A signature that matched has already named
  // the roles; otherwise the generic dictionary does. A headerless file yields
  // no map at all, which is the explicit fall-back: every step below then runs
  // on shape only.
  const lexical = hasHeader
    ? (signature?.roles ?? matchTransactionHeaders(effectiveData[0]))
    : null;

  const dataStartIdx = hasHeader ? 1 : 0;
  const sampleRows = effectiveData.slice(dataStartIdx, dataStartIdx + 20);
  if (sampleRows.length === 0) return failed;

  const colCount = Math.max(...effectiveData.slice(0, 10).map((r) => r.length));

  // Step 3: Detect date column + format
  const dateResult = detectDateColumn(sampleRows, colCount, lexical?.date);
  if (!dateResult) return failed;

  // Step 3b: Find ALL date-like columns (to exclude from amount candidates)
  const dateLikeCols = new Set<number>();
  for (let col = 0; col < colCount; col++) {
    for (const fmt of DATE_FORMATS) {
      let success = 0;
      let total = 0;
      for (const row of sampleRows) {
        const cell = row[col]?.trim();
        if (!cell) continue;
        total++;
        if (parseDate(cell, fmt)) success++;
      }
      if (total > 0 && success / total >= 0.8) {
        dateLikeCols.add(col);
        break;
      }
    }
  }

  // Step 4: Detect numeric columns
  const numericCols = detectNumericColumns(sampleRows, colCount);

  // Step 5: Detect balance columns and exclude them + date-like columns.
  // A column LABELLED "Solde"/"Balance" is excluded too — the arithmetic test
  // needs three consecutive rows and a matching amount column to fire, so a
  // short file or a statement whose balance does not reconcile keeps its
  // running balance in the running for the amount column. The exclusion is
  // dropped if it would leave nothing to map (see `narrowCandidates`).
  const balanceCols = detectBalanceColumns(sampleRows, numericCols);
  const shapeCandidates = numericCols.filter(
    (c) => !balanceCols.has(c) && !dateLikeCols.has(c)
  );
  const amountCandidates = narrowCandidates(
    shapeCandidates,
    (c) => c !== lexical?.balance
  );

  // Step 6: Detect description column
  const descriptionCol = detectDescriptionColumn(
    sampleRows,
    colCount,
    dateResult.column,
    new Set([...numericCols, ...dateLikeCols]),
    lexical?.description
  );

  // Step 7: Determine amount mode
  const amountResult = detectAmountMode(
    sampleRows,
    amountCandidates,
    lexical,
    signature
  );
  if (!amountResult) return failed;

  // Step 7b: Refuse the third amount format — unsigned magnitudes plus a
  // neighbouring D/C column. Nothing here reads that column, so the file would
  // be configured as `positive_expense` and every credit imported as an
  // expense. Detected, refused, and left to a future `absolute_indicator` mode.
  if (amountResult.mode === "single") {
    const indicatorCol = findDirectionIndicatorColumn(
      sampleRows,
      amountResult.amountCol,
      colCount
    );
    if (indicatorCol !== null) {
      return {
        status: "rejected",
        reason: "import.errors.absoluteIndicatorFormat",
      };
    }
  }

  const mapping: ColumnMapping = {
    date: dateResult.column,
    description: descriptionCol,
  };

  let signConvention: SignConvention = "negative_expense";

  if (amountResult.mode === "debit_credit") {
    mapping.debitAmount = amountResult.debitCol;
    mapping.creditAmount = amountResult.creditCol;
  } else {
    mapping.amount = amountResult.amountCol;
    signConvention = amountResult.signConvention;
  }

  const config: AutoDetectResult = {
    delimiter,
    hasHeader,
    skipLines,
    dateFormat: dateResult.format,
    columnMapping: mapping,
    amountMode: amountResult.mode,
    signConvention,
  };

  // Step 8: Replay what we just decided, over the WHOLE file. The sample above
  // is 20 rows because that is enough to decide a shape; the score is what the
  // user is told, so it has to describe the actual file.
  return {
    status: "ok",
    config,
    score: scoreConfig(config, data),
    bank: signature?.signature.id ?? null,
  };
}

/**
 * Replay a detected configuration over the file's data rows and count how many
 * of them it reads.
 *
 * Deliberately NOT a rule of its own: the row is mapped by `mapRow`, the same
 * pure function `parseFilesInternal` runs at import time, under the same
 * column-level decimal arbitration. Two mappers would be the exact class of
 * divergence this chantier exists to remove — a score could then read 100 %
 * while the import wrote different amounts.
 *
 * The row selection mirrors `parseFilesInternal` line for line, including the
 * lone-empty-cell skip: a trailing blank line must not count as an unread row.
 */
function scoreConfig(
  config: AutoDetectResult,
  data: string[][]
): DetectionScore {
  // `mapRow` reads no byte and therefore never looks at `encoding` — the
  // content reached us already decoded. Naming it here only satisfies the type.
  const format: ImportFormat = { ...config, encoding: "utf-8" };

  const dataRows: string[][] = [];
  const startIdx = config.skipLines + (config.hasHeader ? 1 : 0);
  for (let i = startIdx; i < data.length; i++) {
    const raw = data[i];
    if (raw.length <= 1 && raw[0]?.trim() === "") continue;
    dataRows.push(raw);
  }

  const decimalSeparators = detectAmountSeparators(dataRows, format);

  let readRows = 0;
  for (const raw of dataRows) {
    try {
      if (mapRow(raw, format, { decimalSeparators }).parsed) readRows++;
    } catch {
      // `mapRow` is documented not to throw; if it ever did, that is one
      // unreadable row, not a detection that collapses on the whole file.
    }
  }

  const totalRows = dataRows.length;
  const ratio = totalRows === 0 ? 0 : readRows / totalRows;
  return {
    readRows,
    totalRows,
    ratio,
    confident: totalRows > 0 && ratio >= CONFIDENCE_THRESHOLD,
  };
}

/**
 * Apply a lexical restriction to a candidate list, unless it would empty it.
 *
 * The lexical layer must never be able to turn "detected, possibly wrong" into
 * "detected nothing": a file whose only amount column happens to be labelled
 * `Solde` is still importable, and a user can fix a mapping far more easily
 * than a refusal to configure.
 */
function narrowCandidates(
  candidates: number[],
  keep: (col: number) => boolean
): number[] {
  const narrowed = candidates.filter(keep);
  return narrowed.length > 0 ? narrowed : candidates;
}

function detectDelimiter(lines: string[]): string | null {
  let bestDelimiter: string | null = null;
  let bestScore = 0;

  for (const delim of DELIMITERS) {
    const counts = lines.map(
      (line) =>
        Papa.parse(line, { delimiter: delim }).data[0] as string[]
    ).map((row) => row.length);

    // Find the most frequent column count > 1 (tolerates preamble lines)
    const countFreq = new Map<number, number>();
    for (const c of counts) {
      if (c <= 1) continue;
      countFreq.set(c, (countFreq.get(c) || 0) + 1);
    }
    if (countFreq.size === 0) continue;

    let modeCount = 0;
    let modeFreq = 0;
    for (const [count, freq] of countFreq) {
      if (freq > modeFreq || (freq === modeFreq && count > modeCount)) {
        modeFreq = freq;
        modeCount = count;
      }
    }

    const score = (modeFreq / counts.length) * modeCount;

    if (score > bestScore) {
      bestScore = score;
      bestDelimiter = delim;
    }
  }

  return bestDelimiter;
}

/**
 * Decide whether the first row is a header.
 *
 * The shape test — no parseable date, no parseable number — is kept and comes
 * first. It cannot classify a header whose cells carry bare numbers (a column
 * literally titled `2025`), and treating that row as data costs one lost
 * transaction plus a mapping computed from a sample that starts with a label
 * row. So a row the shape test rejects ONLY because of a number is put to the
 * lexical layer: two named roles in one row is a header.
 *
 * The date test stays absolute. A row carrying a parseable date is data, no
 * matter what its other cells spell — that is the guard that keeps a
 * transaction like `DEPOT PAIE EMPLOYEUR` (which contains the credit keyword
 * `depot`) from being swallowed as a header.
 *
 * The holdings flow calls this too. Its headers name no transaction role, so
 * the lexical clause never fires there.
 */
function detectHeader(firstRow: string[]): boolean {
  // A header row typically has no parseable dates and no parseable numbers
  let hasDate = false;
  let hasNumber = false;

  for (const cell of firstRow) {
    const trimmed = cell?.trim();
    if (!trimmed) continue;

    // Check for number
    if (!isNaN(parseFrenchAmount(trimmed))) {
      hasNumber = true;
    }

    // Check for date
    for (const fmt of DATE_FORMATS) {
      if (parseDate(trimmed, fmt)) {
        hasDate = true;
        break;
      }
    }
  }

  if (hasDate) return false;
  if (!hasNumber) return true;

  return matchTransactionHeaders(firstRow).roleCount >= MIN_HEADER_ROLE_MATCHES;
}

/**
 * Pick the date column and its format.
 *
 * `preferred` is the column the header labels name. It wins only if the data
 * agrees — same 0.8 parse rate the shape scan demands — so a file whose "Date
 * de production" column holds no date falls back to the best-parsing column
 * rather than mapping a label nothing supports. Its real job is arbitrating
 * between several equally parseable date columns (`Date de transaction` vs
 * `Date comptable`), which the rate alone cannot do.
 */
function detectDateColumn(
  rows: string[][],
  colCount: number,
  preferred?: number | null
): { column: number; format: string } | null {
  const rateOf = (col: number, fmt: string): number => {
    let success = 0;
    let total = 0;
    for (const row of rows) {
      const cell = row[col]?.trim();
      if (!cell) continue;
      total++;
      if (parseDate(cell, fmt)) success++;
    }
    return total === 0 ? 0 : success / total;
  };

  if (preferred !== undefined && preferred !== null && preferred < colCount) {
    let bestFormat = "";
    let bestRate = 0;
    for (const fmt of DATE_FORMATS) {
      const rate = rateOf(preferred, fmt);
      if (rate > bestRate) {
        bestRate = rate;
        bestFormat = fmt;
      }
    }
    if (bestRate >= 0.8) return { column: preferred, format: bestFormat };
  }

  let bestCol = -1;
  let bestFormat = "";
  let bestRate = 0;

  for (let col = 0; col < colCount; col++) {
    for (const fmt of DATE_FORMATS) {
      const rate = rateOf(col, fmt);
      if (rate > bestRate) {
        bestRate = rate;
        bestCol = col;
        bestFormat = fmt;
      }
    }
  }

  if (bestRate < 0.8 || bestCol < 0) return null;

  return { column: bestCol, format: bestFormat };
}

function detectNumericColumns(rows: string[][], colCount: number): number[] {
  const result: number[] = [];

  for (let col = 0; col < colCount; col++) {
    let numericCount = 0;
    let nonEmpty = 0;
    const distinctValues = new Set<number>();

    for (const row of rows) {
      const cell = row[col]?.trim();
      if (!cell) continue;
      nonEmpty++;
      const val = parseFrenchAmount(cell);
      if (!isNaN(val)) {
        numericCount++;
        distinctValues.add(val);
      }
    }

    if (nonEmpty > 0 && numericCount / nonEmpty >= 0.5) {
      // Exclude constant-value columns (e.g., account numbers, transit numbers)
      if (distinctValues.size <= 1 && nonEmpty > 2) continue;
      result.push(col);
    }
  }

  return result;
}

function detectBalanceColumns(
  rows: string[][],
  numericCols: number[]
): Set<number> {
  const balanceCols = new Set<number>();
  if (numericCols.length < 2 || rows.length < 3) return balanceCols;

  const TOLERANCE = 0.015; // tolerance for floating-point comparison

  // Parse all numeric values once
  const values: Map<number, (number | null)[]> = new Map();
  for (const col of numericCols) {
    values.set(
      col,
      rows.map((row) => {
        const cell = row[col]?.trim();
        if (!cell) return null;
        const v = parseFrenchAmount(cell);
        return isNaN(v) ? null : v;
      })
    );
  }

  for (const balCol of numericCols) {
    const balVals = values.get(balCol)!;

    // Test single-column balance: balance[i] ≈ balance[i-1] ± amount[i]
    for (const amtCol of numericCols) {
      if (amtCol === balCol) continue;
      const amtVals = values.get(amtCol)!;

      let matches = 0;
      let tested = 0;

      for (let i = 1; i < rows.length; i++) {
        if (balVals[i] === null || balVals[i - 1] === null || amtVals[i] === null)
          continue;
        tested++;

        const diff = balVals[i]! - balVals[i - 1]!;
        // balance[i] = balance[i-1] + amount[i] OR balance[i] = balance[i-1] - amount[i]
        if (
          Math.abs(diff - amtVals[i]!) < TOLERANCE ||
          Math.abs(diff + amtVals[i]!) < TOLERANCE
        ) {
          matches++;
        }
      }

      if (tested >= 2 && matches / tested >= 0.8) {
        balanceCols.add(balCol);
        break;
      }
    }

    if (balanceCols.has(balCol)) continue;

    // Test two-column balance: balance[i] ≈ balance[i-1] - debit[i] + credit[i]
    for (let a = 0; a < numericCols.length; a++) {
      for (let b = a + 1; b < numericCols.length; b++) {
        const colA = numericCols[a];
        const colB = numericCols[b];
        if (colA === balCol || colB === balCol) continue;

        const valsA = values.get(colA)!;
        const valsB = values.get(colB)!;

        let matches = 0;
        let tested = 0;

        for (let i = 1; i < rows.length; i++) {
          if (balVals[i] === null || balVals[i - 1] === null) continue;
          const da = valsA[i] ?? 0;
          const db = valsB[i] ?? 0;
          tested++;

          const diff = balVals[i]! - balVals[i - 1]!;
          // Try both orderings: diff ≈ -colA + colB or diff ≈ colA - colB
          if (
            Math.abs(diff - (-da + db)) < TOLERANCE ||
            Math.abs(diff - (da - db)) < TOLERANCE
          ) {
            matches++;
          }
        }

        if (tested >= 2 && matches / tested >= 0.8) {
          balanceCols.add(balCol);
          break;
        }
      }
      if (balanceCols.has(balCol)) break;
    }
  }

  return balanceCols;
}

/**
 * Pick the description column: the one the header labels name, else the
 * longest-on-average text column.
 *
 * The label is only honoured for a column the shape test would have been
 * allowed to pick at all — not the date column, not a numeric one. A file
 * labelling its amount column `Détail du montant` therefore cannot end up with
 * its amounts in the description.
 */
/**
 * Does this column behave like an enumeration rather than free text?
 *
 * A label alone is not enough to pick the description: Tangerine exports
 * `Date,Transaction,Name,Memo,Amount`, where `Transaction` holds DEBIT/CREDIT
 * and `Name` holds the merchant. Honouring the label there moves the merchant
 * out of the description and kills keyword categorisation.
 *
 * Cardinality separates the two — a description repeats almost nothing, an enum
 * repeats almost everything. Average length does NOT: `Note` and `Libellé` are
 * both short, and vetoing on length would reject legitimate columns.
 */
function looksLikeEnumColumn(rows: string[][], col: number): boolean {
  const distinct = new Set<string>();
  let filled = 0;

  for (const row of rows) {
    const cell = row[col]?.trim();
    if (!cell) continue;
    filled++;
    distinct.add(cell.toLowerCase());
  }

  // A labelled but entirely empty column is never the description either.
  if (filled === 0) return true;
  // Too few rows to read anything into the cardinality.
  if (filled < 4) return false;

  return distinct.size <= Math.max(2, Math.floor(filled / 4));
}

function detectDescriptionColumn(
  rows: string[][],
  colCount: number,
  dateCol: number,
  numericCols: Set<number>,
  preferred?: number | null
): number {
  if (
    preferred !== undefined &&
    preferred !== null &&
    preferred < colCount &&
    preferred !== dateCol &&
    !numericCols.has(preferred) &&
    !looksLikeEnumColumn(rows, preferred)
  ) {
    return preferred;
  }

  let bestCol = 0;
  let bestAvgLen = 0;

  for (let col = 0; col < colCount; col++) {
    if (col === dateCol || numericCols.has(col)) continue;

    let totalLen = 0;
    let count = 0;

    for (const row of rows) {
      const cell = row[col]?.trim();
      if (!cell) continue;
      totalLen += cell.length;
      count++;
    }

    const avgLen = count > 0 ? totalLen / count : 0;
    if (avgLen > bestAvgLen) {
      bestAvgLen = avgLen;
      bestCol = col;
    }
  }

  return bestCol;
}

interface SingleAmountResult {
  mode: "single";
  amountCol: number;
  signConvention: SignConvention;
}

interface DebitCreditResult {
  mode: "debit_credit";
  debitCol: number;
  creditCol: number;
}

type AmountModeResult = SingleAmountResult | DebitCreditResult;

/**
 * Decide the amount mode, and which column(s) carry it.
 *
 * `signature` is the one hint that outranks the sparse-complementary scan, and
 * only because that scan cannot be told apart from the truth by shape alone:
 * RBC's `CAD$` and `USD$` ARE complementary — the USD column is empty on a
 * Canadian account — so a file whose amounts are one signed column reads as a
 * debit/credit pair, with every credit imported as an expense. A bank that
 * documents its layout settles that; nothing else in the file can.
 *
 * It stays a preference all the same: the columns it names must be candidates
 * the shape scan itself proposed. A signature naming a column that parses as
 * nothing numeric is dropped here and the generic path decides, exactly like a
 * mismatched label.
 */
/**
 * The first sparse-complementary pair among the candidates, in column order, or
 * null. Extracted from `detectAmountMode` so a bank signature can be arbitrated
 * AGAINST the pair the shape scan would have found, instead of short-circuiting
 * a scan that never ran.
 */
function findSparseComplementaryPair(
  rows: string[][],
  amountCandidates: number[]
): [number, number] | null {
  for (let a = 0; a < amountCandidates.length; a++) {
    for (let b = a + 1; b < amountCandidates.length; b++) {
      const colA = amountCandidates[a];
      const colB = amountCandidates[b];
      if (isSparseComplementary(rows, colA, colB)) return [colA, colB];
    }
  }
  return null;
}

function detectAmountMode(
  rows: string[][],
  amountCandidates: number[],
  lexical: LexicalHeaderMap | null,
  signature: BankSignatureMatch | null
): AmountModeResult | null {
  if (amountCandidates.length === 0) return null;

  const pair = findSparseComplementaryPair(rows, amountCandidates);

  if (signature) {
    const { debit, credit, amount } = signature.roles;
    if (
      debit !== null &&
      credit !== null &&
      amountCandidates.includes(debit) &&
      amountCandidates.includes(credit)
    ) {
      return { mode: "debit_credit", debitCol: debit, creditCol: credit };
    }
    // A signature's SINGLE amount column may not silently displace a genuine
    // debit/credit pair it has no part in. RBC needs the override — its
    // near-empty `Cheque Number` really is sparse-complementary with `CAD$` —
    // but there the pair contains the declared amount column. On a
    // `Date;Description;Débit;Crédit;Montant;Solde` file the pair does not, and
    // taking `Montant` unsigned imported every deposit as an expense.
    if (
      amount !== null &&
      amountCandidates.includes(amount) &&
      (!pair || pair.includes(amount))
    ) {
      return detectSingleAmount(rows, amount);
    }
  }

  if (amountCandidates.length === 1) {
    return detectSingleAmount(rows, amountCandidates[0]);
  }

  if (pair) {
    return orderDebitCredit(pair[0], pair[1], lexical);
  }

  // No complementary pair found — the labelled amount column if there is one,
  // else the best single amount column by shape.
  const preferred = lexical?.amount;
  const bestCol =
    preferred !== undefined &&
    preferred !== null &&
    amountCandidates.includes(preferred)
      ? preferred
      : pickBestAmountColumn(rows, amountCandidates);
  return detectSingleAmount(rows, bestCol);
}

/**
 * Assign the two halves of a complementary pair to debit and credit.
 *
 * The loop above enumerates `a < b`, so before #327 the LEFTMOST column was the
 * debit, always: a file laid out `Date;Description;Crédit;Débit` was mapped
 * backwards and every sign of the import came out inverted — silently, since
 * the total is merely negated and no aggregate check notices.
 *
 * The labels decide when they name either half; position remains the fall-back
 * for a headerless file or labels the dictionary does not know. Knowing ONE of
 * the two is enough: the other column is the other role.
 */
function orderDebitCredit(
  colA: number,
  colB: number,
  lexical: LexicalHeaderMap | null
): DebitCreditResult {
  const debitFirst = {
    mode: "debit_credit",
    debitCol: colA,
    creditCol: colB,
  } as const;
  const creditFirst = {
    mode: "debit_credit",
    debitCol: colB,
    creditCol: colA,
  } as const;

  if (lexical?.debit === colA || lexical?.credit === colB) return debitFirst;
  if (lexical?.debit === colB || lexical?.credit === colA) return creditFirst;
  return debitFirst;
}

/**
 * Find a direction-indicator column next to an all-positive amount column —
 * the third amount format, refused rather than imported.
 *
 * Three conditions, each one narrowing a way to raise a false alarm:
 *  - the amount column carries no negative value, since a file that signs its
 *    amounts needs no indicator and reads correctly today;
 *  - the candidate is IMMEDIATELY next to it, as every export of this shape
 *    writes the flag beside the magnitude. A file-wide scan would refuse
 *    perfectly importable files over an unrelated one-letter flag column, and
 *    a refusal the user cannot work around is worse than a mapping they can;
 *  - it holds at least two DISTINCT tokens of the D/C alphabet. A column stuck
 *    on a single value carries no direction, and a statement of pure expenses
 *    imports correctly as `positive_expense`.
 */
function findDirectionIndicatorColumn(
  rows: string[][],
  amountCol: number,
  colCount: number
): number | null {
  let seen = 0;
  for (const row of rows) {
    const cell = row[amountCol]?.trim();
    if (!cell) continue;
    const val = parseFrenchAmount(cell);
    if (isNaN(val)) continue;
    seen++;
    if (val < 0) return null;
  }
  if (seen === 0) return null;

  for (const col of [amountCol - 1, amountCol + 1]) {
    if (col < 0 || col >= colCount) continue;

    const distinct = new Set<string>();
    let nonEmpty = 0;
    let inAlphabet = 0;

    for (const row of rows) {
      const cell = row[col]?.trim();
      if (!cell) continue;
      nonEmpty++;
      const token = cell.toLowerCase();
      if (DIRECTION_INDICATOR_TOKENS.has(token)) {
        inAlphabet++;
        distinct.add(token);
      }
    }

    if (nonEmpty > 0 && inAlphabet === nonEmpty && distinct.size >= 2) {
      return col;
    }
  }

  return null;
}

/** Pick the best amount column: prefer columns with decimal values (cents). */
function pickBestAmountColumn(rows: string[][], candidates: number[]): number {
  let bestCol = candidates[0];
  let bestScore = -1;

  for (const col of candidates) {
    let decimalCount = 0;
    let nonEmpty = 0;

    for (const row of rows) {
      const cell = row[col]?.trim();
      if (!cell) continue;
      const val = parseFrenchAmount(cell);
      if (isNaN(val)) continue;
      nonEmpty++;
      if (!Number.isInteger(val)) {
        decimalCount++;
      }
    }

    const score = nonEmpty > 0 ? decimalCount / nonEmpty : 0;
    if (score > bestScore) {
      bestScore = score;
      bestCol = col;
    }
  }

  return bestCol;
}

function detectSingleAmount(
  rows: string[][],
  col: number
): SingleAmountResult {
  let negCount = 0;
  let total = 0;

  for (const row of rows) {
    const cell = row[col]?.trim();
    if (!cell) continue;
    const val = parseFrenchAmount(cell);
    if (isNaN(val)) continue;
    total++;
    if (val < 0) negCount++;
  }

  // If most values are negative, they likely represent expenses as negative
  const signConvention: SignConvention =
    total > 0 && negCount / total > 0.5
      ? "negative_expense"
      : "positive_expense";

  return { mode: "single", amountCol: col, signConvention };
}

function isSparseComplementary(
  rows: string[][],
  colA: number,
  colB: number
): boolean {
  let complementary = 0;
  let total = 0;

  for (const row of rows) {
    const cellA = row[colA]?.trim();
    const cellB = row[colB]?.trim();
    const valA = cellA ? parseFrenchAmount(cellA) : NaN;
    const valB = cellB ? parseFrenchAmount(cellB) : NaN;
    const hasA = !isNaN(valA) && valA !== 0;
    const hasB = !isNaN(valB) && valB !== 0;

    if (!hasA && !hasB) continue;
    total++;

    // Complementary: exactly one has a value
    if (hasA !== hasB) {
      complementary++;
    }
  }

  return total > 0 && complementary / total >= 0.7;
}

// -----------------------------------------------------------------------------
// Holdings CSV detection (Issue #245) — a detailed balance account can import a
// CSV of positions instead of typing them one by one. This is a SEPARATE flow
// from the transaction import above: `autoDetectConfig` is coupled to the
// date/amount transaction model, whereas a holdings CSV maps to
// symbol / quantity / unit_price / book_cost. Price + book_cost are OPTIONAL
// (flexible price detection): when the CSV has no price column the mapping
// leaves it `null` and the holdings import without a price (the user fetches or
// types it afterwards). Detection is best-effort and the UI lets the user
// adjust every column, so an imperfect guess is always recoverable.
// -----------------------------------------------------------------------------

export interface HoldingColumnMapping {
  /** Column index of the security symbol/ticker (required). */
  symbol: number;
  /** Column index of the quantity held (required). */
  quantity: number;
  /** Column index of the unit price, or null when the CSV has no price column. */
  unit_price: number | null;
  /** Column index of the acquisition cost basis, or null when absent. */
  book_cost: number | null;
}

export interface HoldingCsvAnalysis {
  delimiter: string;
  hasHeader: boolean;
  /** Header labels (actual cells when `hasHeader`, else `Col 0`, `Col 1`, …). */
  headers: string[];
  /** DATA rows only (the header row, if any, is stripped). */
  rows: string[][];
  mapping: HoldingColumnMapping;
}

// Header keyword sets, matched against accent-stripped, alphanumeric-only
// header cells. Bilingual (FR default + EN). Order inside each list is priority
// order for the primary-keyword pass.
const SYMBOL_HEADER_KEYWORDS = ["symbol", "symbole", "ticker", "titre"];
const QUANTITY_HEADER_KEYWORDS = [
  "quantity",
  "quantite",
  "qty",
  "qte",
  "shares",
  "actions",
  "parts",
  "unites",
  "units",
  "nombre",
];
const PRICE_HEADER_KEYWORDS = [
  "prixunitaire",
  "unitprice",
  "marketprice",
  "price",
  "prix",
  "cours",
  "cotation",
  "cloture",
  "close",
];
const BOOKCOST_HEADER_KEYWORDS = [
  "bookcost",
  "costbasis",
  "prixderevient",
  "acquisition",
  "revient",
  "cost",
  "cout",
];
// Value/market-value columns must never be auto-picked as price or book_cost.
// NOTE `montant` is an EXCLUSION token here and the PRIMARY amount keyword of
// the transaction dictionary (`headerDictionary.ts`) \u2014 which is why the two
// tables are separate modules. `normalizeHeaderCell` and `matchHeaderColumn`
// are the generic part, shared from there and unchanged.
const VALUE_HEADER_KEYWORDS = ["value", "valeur", "montant", "marchande"];

/** Pick the shortest-average-length non-numeric, unused text column (symbols
 *  are short tokens; a name/description column is longer). */
function pickSymbolColumn(
  rows: string[][],
  colCount: number,
  numericCols: Set<number>,
  used: Set<number>
): number | null {
  let best: number | null = null;
  let bestAvg = Infinity;
  for (let col = 0; col < colCount; col++) {
    if (used.has(col) || numericCols.has(col)) continue;
    let totalLen = 0;
    let count = 0;
    for (const row of rows) {
      const cell = row[col]?.trim();
      if (!cell) continue;
      totalLen += cell.length;
      count++;
    }
    if (count === 0) continue;
    const avg = totalLen / count;
    if (avg < bestAvg) {
      bestAvg = avg;
      best = col;
    }
  }
  return best;
}

/** Pick an unused numeric column, optionally preferring integer-heavy (quantity)
 *  or decimal-heavy (price) columns. Returns null when none remain. */
function pickNumericColumn(
  rows: string[][],
  numericCols: number[],
  used: Set<number>,
  opts: { preferIntegers?: boolean; preferDecimals?: boolean }
): number | null {
  let best: number | null = null;
  let bestScore = -1;
  for (const col of numericCols) {
    if (used.has(col)) continue;
    let ints = 0;
    let decimals = 0;
    let nonEmpty = 0;
    for (const row of rows) {
      const cell = row[col]?.trim();
      if (!cell) continue;
      const v = parseFrenchAmount(cell);
      if (isNaN(v)) continue;
      nonEmpty++;
      if (Number.isInteger(v)) ints++;
      else decimals++;
    }
    if (nonEmpty === 0) continue;
    let score = 1;
    if (opts.preferIntegers) score = ints / nonEmpty;
    else if (opts.preferDecimals) score = decimals / nonEmpty;
    if (score > bestScore) {
      bestScore = score;
      best = col;
    }
  }
  return best;
}

/**
 * Detect the symbol / quantity / unit_price / book_cost columns of a holdings
 * CSV. `data` is the parsed 2-D array INCLUDING the header row when
 * `hasHeader`. Returns a best-effort mapping (symbol + quantity always set so
 * the editor can render), or null only when there is no usable data.
 * Exported for unit tests.
 */
export function autoDetectHoldingColumns(
  data: string[][],
  hasHeader: boolean
): HoldingColumnMapping | null {
  if (data.length === 0) return null;
  const colCount = Math.max(...data.map((r) => r.length));
  if (colCount === 0) return null;
  const dataRows = hasHeader ? data.slice(1) : data;
  if (dataRows.length === 0) return null;

  const used = new Set<number>();
  let symbol: number | null = null;
  let quantity: number | null = null;
  let unitPrice: number | null = null;
  let bookCost: number | null = null;

  // Step 1 — header keyword matching (most reliable when present).
  if (hasHeader) {
    const normalized = data[0].map(normalizeHeaderCell);
    symbol = matchHeaderColumn(normalized, SYMBOL_HEADER_KEYWORDS, used);
    if (symbol !== null) used.add(symbol);
    quantity = matchHeaderColumn(normalized, QUANTITY_HEADER_KEYWORDS, used);
    if (quantity !== null) used.add(quantity);
    unitPrice = matchHeaderColumn(
      normalized,
      PRICE_HEADER_KEYWORDS,
      used,
      VALUE_HEADER_KEYWORDS
    );
    if (unitPrice !== null) used.add(unitPrice);
    bookCost = matchHeaderColumn(
      normalized,
      BOOKCOST_HEADER_KEYWORDS,
      used,
      VALUE_HEADER_KEYWORDS
    );
    if (bookCost !== null) used.add(bookCost);
  }

  // A value / market-value column must never be auto-assigned to a numeric
  // target by the heuristics below (it is qty × price, not a source column).
  // Mark such columns used up-front so the fallback pickers skip them.
  if (hasHeader) {
    const normalized = data[0].map(normalizeHeaderCell);
    normalized.forEach((h, i) => {
      if (!used.has(i) && VALUE_HEADER_KEYWORDS.some((v) => h && h.includes(v))) {
        used.add(i);
      }
    });
  }

  // Step 2 — numeric/text heuristics fill any column the header didn't resolve.
  const sampleRows = dataRows.slice(0, 20);
  const numericCols = detectNumericColumns(sampleRows, colCount);
  const numericSet = new Set(numericCols);

  if (symbol === null) {
    symbol = pickSymbolColumn(sampleRows, colCount, numericSet, used);
    if (symbol !== null) used.add(symbol);
  }
  if (quantity === null) {
    quantity = pickNumericColumn(sampleRows, numericCols, used, {
      preferIntegers: true,
    });
    if (quantity !== null) used.add(quantity);
  }
  if (unitPrice === null) {
    unitPrice = pickNumericColumn(sampleRows, numericCols, used, {
      preferDecimals: true,
    });
    if (unitPrice !== null) used.add(unitPrice);
  }
  if (bookCost === null) {
    bookCost = pickNumericColumn(sampleRows, numericCols, used, {});
    if (bookCost !== null) used.add(bookCost);
  }

  // symbol + quantity are required; fall back to sane defaults so the editor
  // always has something to show (the user can correct it).
  if (symbol === null) symbol = 0;
  if (quantity === null) quantity = symbol === 0 && colCount > 1 ? 1 : 0;

  return { symbol, quantity, unit_price: unitPrice, book_cost: bookCost };
}

/**
 * Full holdings-CSV analysis: preprocess quoted lines, detect the delimiter and
 * header, then the column mapping. Returns the header labels + DATA rows (header
 * stripped) so the caller can render a mapping editor + preview and feed the
 * rows to `holdingsFromCsvRows`. Null when the content has no usable rows.
 * Exported for unit tests.
 */
export function analyzeHoldingsCsv(
  rawContent: string
): HoldingCsvAnalysis | null {
  const content = preprocessQuotedCSV(rawContent);
  const nonEmptyLines = content.split(/\r?\n/).filter((l) => l.trim());
  if (nonEmptyLines.length === 0) return null;

  const delimiter = detectDelimiter(nonEmptyLines.slice(0, 10));
  if (!delimiter) return null;

  const parsed = Papa.parse(content, { delimiter, skipEmptyLines: true });
  const data = (parsed.data as string[][]).filter((r) =>
    r.some((c) => (c ?? "").trim() !== "")
  );
  if (data.length === 0) return null;

  const hasHeader = detectHeader(data[0]);
  const mapping = autoDetectHoldingColumns(data, hasHeader);
  if (!mapping) return null;

  const colCount = Math.max(...data.map((r) => r.length));
  const headers = hasHeader
    ? data[0].map((h, i) => (h ?? "").trim() || `Col ${i}`)
    : Array.from({ length: colCount }, (_, i) => `Col ${i}`);
  const rows = hasHeader ? data.slice(1) : data;

  return { delimiter, hasHeader, headers, rows, mapping };
}