Spendlight/server/src/import/service.ts

import type { AccountKind, ImportColumnMap, ImportPreview, ImportSummary } from '@spendlight/shared';
import { and, between, count, desc, eq, inArray } from 'drizzle-orm';
import { createHash, randomUUID } from 'node:crypto';
import { mkdirSync, writeFileSync } from 'node:fs';
import { extname, join } from 'node:path';
import { logChange } from '../audit';
import { Suggester } from '../categorize/suggest';
import type { AppContext } from '../context';
import { accounts, importLayouts, imports, transactions, type AccountRow } from '../db/schema';
import { fail } from '../http';
import { detectBuiltIn, findHeaderRow, guessMap, headerSignature, type ResolvedLayout } from './layouts';
import { cleanMerchant, dedupeKey } from './merchant';
import { daysBetween, parseCents, parseDate, readTable, shiftDate } from './parse';
import { pairTransfers } from './transfers';

// The import pipeline: stage a file, preview what it would add, then commit.
// Nothing is written until commit, and every commit can be undone whole.

const MAX_BYTES = 10 * 1024 * 1024;
const STAGE_TTL_MS = 60 * 60_000;

interface Staged {
  id: string;
  fileName: string;
  text: string;
  hash: string;
  userId: number;
  createdAt: number;
}

export class StagingStore {
  private files = new Map<string, Staged>();

  add(fileName: string, text: string, userId: number): Staged {
    this.sweep();
    const staged = {
      id: randomUUID(),
      fileName,
      text,
      hash: createHash('sha256').update(text).digest('hex'),
      userId,
      createdAt: Date.now(),
    };
    this.files.set(staged.id, staged);
    return staged;
  }

  get(id: string, userId: number): Staged {
    const staged = this.files.get(id);
    if (!staged || staged.userId !== userId) fail(404, 'expired', 'That file has expired. Please add it again.');
    return staged;
  }

  delete(id: string): void {
    this.files.delete(id);
  }

  private sweep() {
    for (const [id, f] of this.files) if (Date.now() - f.createdAt > STAGE_TTL_MS) this.files.delete(id);
  }
}

interface Row {
  date: string;
  postedDate: string | null;
  description: string;
  merchant: string;
  mentionsLast4: string | null;
  amountCents: number;
  bankCategory: string | null;
  bankType: string | null;
  balanceCents: number | null;
}

interface Parsed {
  layout: ResolvedLayout | null;
  rows: Row[];
  skipped: number;
  header: string[] | null;
  sample: string[][];
  suggestedMap: ImportColumnMap | null;
  signature: string;
  last4Hint: string | null;
  openingBalance: { date: string; cents: number } | null;
}

function fileNameLast4(fileName: string): string | null {
  return /^[A-Za-z]+(\d{4})[_-]/.exec(fileName)?.[1] ?? /(?:x|ending|acct|card)[-_ ]?(\d{4})\b/i.exec(fileName)?.[1] ?? null;
}

/** The file name with its digits generalized, so "Chase1234_Activity_20261007.csv" matches next month's file. */
const fileNamePattern = (fileName: string) => fileName.replace(/\d{5,}/g, '#').replace(/\(\d+\)/g, '');

function parseFile(ctx: AppContext, staged: Staged, mapping?: ImportColumnMap): Parsed {
  const table = readTable(staged.text);
  let layout = mapping ? null : detectBuiltIn(table);
  const headerRow = layout?.headerRow ?? findHeaderRow(table);
  const signature = headerSignature(table, headerRow);

  if (!layout && headerRow >= 0) {
    const custom = mapping
      ? { name: 'Your saved layout', config: JSON.stringify(mapping) }
      : ctx.db.select().from(importLayouts).where(eq(importLayouts.signature, signature)).get();
    if (custom) {
      layout = {
        key: 'id' in custom ? `custom-${custom.id}` : 'custom',
        name: custom.name,
        institution: null,
        kind: null,
        headerRow,
        map: JSON.parse(custom.config) as ImportColumnMap,
      };
    }
  }

  const header = headerRow >= 0 ? table[headerRow]! : null;
  const sample = table.slice(Math.max(0, headerRow + 1), Math.max(0, headerRow + 1) + 5);
  if (!layout) {
    return { layout: null, rows: [], skipped: 0, header, sample, suggestedMap: header ? guessMap(header) : null, signature, last4Hint: null, openingBalance: null };
  }

  const m = layout.map;
  const rows: Row[] = [];
  let skipped = 0;
  let openingBalance: Parsed['openingBalance'] = null;
  let last4Hint = fileNameLast4(staged.fileName);

  for (const cells of table.slice(layout.headerRow + 1)) {
    if (cells.every((c) => !c)) continue;
    const date = parseDate(cells[m.date], m.dateOrder);
    const description = (cells[m.description] ?? '').trim();
    const balanceCents = m.balance !== undefined ? parseCents(cells[m.balance]) : null;
    if (!date) {
      skipped++;
      continue;
    }
    // "Beginning balance as of ..." rows carry a balance but no amount.
    let amount: number | null;
    if (m.amount !== undefined) {
      const raw = parseCents(cells[m.amount]);
      amount = raw === null ? null : m.positiveIs === 'out' ? -raw : raw;
    } else {
      const debit = m.debit !== undefined ? parseCents(cells[m.debit]) : null;
      const credit = m.credit !== undefined ? parseCents(cells[m.credit]) : null;
      amount = debit ? -Math.abs(debit) : credit ? Math.abs(credit) : debit === 0 || credit === 0 ? 0 : null;
    }
    if (amount === null) {
      if (balanceCents !== null && /beginning balance/i.test(description)) openingBalance = { date, cents: balanceCents };
      else skipped++;
      continue;
    }
    if ('status' in m && m.status !== undefined && /pending/i.test(cells[m.status] ?? '')) {
      skipped++;
      continue;
    }
    if ('cardNumber' in m && m.cardNumber !== undefined) last4Hint ??= /(\d{4})\s*$/.exec(cells[m.cardNumber] ?? '')?.[1] ?? null;
    const { merchant, mentionsLast4 } = cleanMerchant(description);
    rows.push({
      date,
      postedDate: m.postedDate !== undefined ? parseDate(cells[m.postedDate], m.dateOrder) : null,
      description,
      merchant,
      mentionsLast4,
      amountCents: amount,
      bankCategory: m.category !== undefined ? cells[m.category] || null : null,
      bankType: 'type' in m && m.type !== undefined ? cells[m.type] || null : null,
      balanceCents,
    });
  }
  return { layout, rows, skipped, header, sample, suggestedMap: null, signature, last4Hint, openingBalance };
}

/** Which account a file belongs to, judged from fingerprints the person never has to manage. */
function matchAccount(ctx: AppContext, parsed: Parsed, fileName: string): ImportPreview['account'] {
  const open = ctx.db.select().from(accounts).where(eq(accounts.status, 'open')).all();
  if (!open.length || !parsed.layout) return null;

  if (parsed.last4Hint) {
    const byLast4 = open.filter((a) => a.last4 === parsed.last4Hint);
    if (byLast4.length === 1) return { id: byLast4[0]!.id, confidence: 'certain', reason: `The file is for the account ending ${parsed.last4Hint}.` };
  }

  // Transactions already in an account are the strongest evidence.
  if (parsed.rows.length) {
    const dates = parsed.rows.map((r) => r.date).sort();
    let best: { account: AccountRow; hits: number } | null = null;
    for (const account of open) {
      const keys = new Set(
        ctx.db
          .select({ key: transactions.dedupeKey })
          .from(transactions)
          .where(and(eq(transactions.accountId, account.id), between(transactions.date, dates[0]!, dates.at(-1)!)))
          .all()
          .map((r) => r.key),
      );
      const hits = parsed.rows.filter((r) => keys.has(dedupeKey(account.id, r.date, r.amountCents, r.description))).length;
      if (hits && (!best || hits > best.hits)) best = { account, hits };
    }
    if (best && (best.hits >= 3 || best.hits >= parsed.rows.length * 0.3)) {
      return { id: best.account.id, confidence: 'certain', reason: `${best.hits} of these transactions are already in ${best.account.name}.` };
    }
  }

  // A statement that starts exactly where an account's last one ended.
  if (parsed.openingBalance) {
    const continues = open.filter(
      (a) => a.balanceSource === 'statement' && a.balanceCents === parsed.openingBalance!.cents && a.layoutKey === parsed.layout!.key,
    );
    if (continues.length === 1) return { id: continues[0]!.id, confidence: 'certain', reason: `It picks up where ${continues[0]!.name}’s last statement ended.` };
  }

  // Softer hints only count when no other account at the same bank could be meant.
  const institution = parsed.layout.institution?.toLowerCase();
  const sameBank = open.filter((a) => a.layoutKey === parsed.layout!.key || (institution && a.institution?.toLowerCase() === institution));
  const sameLayout = open.filter((a) => a.layoutKey === parsed.layout!.key);
  const samePattern = sameLayout.filter((a) => a.fileNamePattern === fileNamePattern(fileName));
  if (sameBank.length === 1 && samePattern.length === 1) return { id: samePattern[0]!.id, confidence: 'likely', reason: `Files like this have gone to ${samePattern[0]!.name} before.` };
  if (sameBank.length === 1 && sameLayout.length === 1) return { id: sameLayout[0]!.id, confidence: 'likely', reason: `${sameLayout[0]!.name} uses this bank’s layout.` };

  const fresh = open.filter(
    (a) => !a.layoutKey && a.institution && parsed.layout!.institution && a.institution.toLowerCase() === parsed.layout!.institution.toLowerCase(),
  );
  if (fresh.length === 1) return { id: fresh[0]!.id, confidence: 'likely', reason: `It’s a ${parsed.layout.institution} file.` };
  return null;
}

interface Planned {
  row: Row;
  key: string;
  status: 'new' | 'duplicate' | 'possible';
  possibleId: string | null;
  existingDate: string | null;
}

/** Count-based duplicate check: import only copies beyond those already stored. */
function planRows(ctx: AppContext, accountId: number, rows: Row[]): Planned[] {
  if (!rows.length) return [];
  const dates = rows.map((r) => r.date).sort();
  const from = dates[0]!;
  const to = dates.at(-1)!;
  const existing = ctx.db
    .select()
    .from(transactions)
    .where(and(eq(transactions.accountId, accountId), between(transactions.date, shiftDate(from, -3), shiftDate(to, 3))))
    .all();
  const existingCount = new Map<string, number>();
  for (const t of existing) existingCount.set(t.dedupeKey, (existingCount.get(t.dedupeKey) ?? 0) + 1);

  const seen = new Map<string, number>();
  const planned: Planned[] = rows.map((row) => {
    const key = dedupeKey(accountId, row.date, row.amountCents, row.description);
    const n = (seen.get(key) ?? 0) + 1;
    seen.set(key, n);
    return { row, key, status: n <= (existingCount.get(key) ?? 0) ? 'duplicate' : 'new', possibleId: null, existingDate: null };
  });

  // Near matches: same amount and merchant a day or three apart, inside this
  // file's dates, with no exact twin in the file. Usually a pending charge that
  // later posted with a new date.
  const fileKeys = new Set(planned.map((p) => p.key));
  const used = new Set<number>();
  for (const p of planned) {
    if (p.status !== 'new') continue;
    const near = existing.find(
      (t) =>
        !used.has(t.id) &&
        !fileKeys.has(t.dedupeKey) &&
        t.amountCents === p.row.amountCents &&
        t.merchant === p.row.merchant &&
        t.date !== p.row.date &&
        t.date >= from &&
        t.date <= to &&
        Math.abs(daysBetween(t.date, p.row.date)) <= 3,
    );
    if (near) {
      used.add(near.id);
      p.status = 'possible';
      p.possibleId = `${p.key}#${seen.get(p.key)}`;
      p.existingDate = near.date;
    }
  }
  return planned;
}

function accountHint(parsed: Parsed): ImportPreview['accountHint'] {
  if (!parsed.layout) return null;
  const kind: AccountKind = parsed.layout.kind ?? 'checking';
  const institution = parsed.layout.institution;
  const last4 = parsed.last4Hint;
  const what = kind === 'credit_card' ? 'card' : kind;
  const name = [institution, what].filter(Boolean).join(' ') + (last4 ? ` ${last4}` : '');
  return { kind, institution, last4, name: name.charAt(0).toUpperCase() + name.slice(1) };
}

export function previewImport(
  ctx: AppContext,
  stagingId: string,
  userId: number,
  options: { mapping?: ImportColumnMap; accountId?: number },
): ImportPreview {
  const staged = ctx.staging.get(stagingId, userId);
  const parsed = parseFile(ctx, staged, options.mapping);
  const account = options.accountId
    ? { id: options.accountId, confidence: 'certain' as const, reason: 'You chose this account.' }
    : matchAccount(ctx, parsed, staged.fileName);
  const planned = account ? planRows(ctx, account.id, parsed.rows) : parsed.rows.map((row) => ({ row, status: 'new' as const }));
  const dates = parsed.rows.map((r) => r.date).sort();
  const fresh = planned.filter((p) => p.status === 'new');

  return {
    stagingId,
    fileName: staged.fileName,
    layout: parsed.layout ? { key: parsed.layout.key, name: parsed.layout.name } : null,
    sampleRows: parsed.layout ? [] : parsed.sample,
    headerRow: parsed.layout ? null : parsed.header,
    suggestedMap: parsed.suggestedMap,
    account,
    dateFrom: dates[0] ?? null,
    dateTo: dates.at(-1) ?? null,
    counts: {
      rows: parsed.rows.length,
      new: fresh.length,
      duplicates: planned.filter((p) => p.status === 'duplicate').length,
      possibleDuplicates: planned.filter((p) => p.status === 'possible').length,
      skipped: parsed.skipped,
    },
    newRows: fresh.slice(0, 8).map((p) => ({ date: p.row.date, merchant: p.row.merchant, amountCents: p.row.amountCents })),
    possibleDuplicates: planned
      .filter((p): p is Planned => p.status === 'possible')
      .map((p) => ({ key: p.possibleId!, date: p.row.date, merchant: p.row.merchant, amountCents: p.row.amountCents, existingDate: p.existingDate! })),
    accountHint: accountHint(parsed),
  };
}

export function commitImport(
  ctx: AppContext,
  stagingId: string,
  userId: number,
  options: { accountId: number; mapping?: ImportColumnMap; includePossible?: string[] },
): ImportSummary {
  const staged = ctx.staging.get(stagingId, userId);
  const account = ctx.db.select().from(accounts).where(eq(accounts.id, options.accountId)).get();
  if (!account || account.status !== 'open') fail(400, 'no_account', 'Please choose an open account for this file.');
  const parsed = parseFile(ctx, staged, options.mapping);
  if (!parsed.layout) fail(400, 'needs_mapping', 'Spendlight needs to know which column is which before importing this file.');
  if (!parsed.rows.length) fail(400, 'empty', 'No transactions were found in this file.');

  const include = new Set(options.includePossible ?? []);
  const planned = planRows(ctx, account.id, parsed.rows).filter(
    (p) => p.status === 'new' || (p.status === 'possible' && include.has(p.possibleId!)),
  );
  const duplicateCount = parsed.rows.length - planned.length;
  const dates = parsed.rows.map((r) => r.date).sort();
  const now = new Date().toISOString();

  const result = ctx.sqlite.transaction(() => {
    const suggester = new Suggester(ctx);
    const record = ctx.db
      .insert(imports)
      .values({
        accountId: account.id,
        fileName: staged.fileName,
        fileHash: staged.hash,
        layoutKey: parsed.layout!.key,
        dateFrom: dates[0] ?? null,
        dateTo: dates.at(-1) ?? null,
        rowCount: parsed.rows.length,
        addedCount: planned.length,
        duplicateCount,
        skippedCount: parsed.skipped,
        previousBalance: JSON.stringify({
          balanceCents: account.balanceCents,
          balanceAsOf: account.balanceAsOf,
          balanceSource: account.balanceSource,
        }),
        userId,
        createdAt: now,
      })
      .returning()
      .get();

    const ids: number[] = [];
    for (const { row, key } of planned) {
      const rule = suggester.rule(row.merchant, account.id);
      const merchant = rule?.renameTo || row.merchant;
      const s = suggester.suggest({ ...row, merchant, accountId: account.id });
      const inserted = ctx.db
        .insert(transactions)
        .values({
          accountId: account.id,
          importId: record.id,
          date: row.date,
          postedDate: row.postedDate,
          amountCents: row.amountCents,
          description: row.description,
          merchant,
          categoryId: s.categoryId,
          suggestedBy: s.source,
          needsReview: true,
          bankCategory: row.bankCategory,
          bankType: row.bankType,
          dedupeKey: key,
          createdAt: now,
          updatedAt: now,
        })
        .returning({ id: transactions.id })
        .get();
      ids.push(inserted.id);
    }

    const transferCount = pairTransfers(ctx, suggester, ids);
    ctx.db.update(imports).set({ transferCount }).where(eq(imports.id, record.id)).run();

    // A layout from the mapping pop-up is saved, and remembered by its stored key.
    let layoutKey = parsed.layout!.key;
    if (options.mapping) {
      const existing = ctx.db.select().from(importLayouts).where(eq(importLayouts.signature, parsed.signature)).get();
      const layoutId = existing
        ? (ctx.db.update(importLayouts).set({ config: JSON.stringify(options.mapping) }).where(eq(importLayouts.id, existing.id)).run(), existing.id)
        : ctx.db
            .insert(importLayouts)
            .values({ name: `${account.name} layout`, signature: parsed.signature, config: JSON.stringify(options.mapping), createdAt: now })
            .returning()
            .get().id;
      layoutKey = `custom-${layoutId}`;
      ctx.db.update(imports).set({ layoutKey }).where(eq(imports.id, record.id)).run();
    }

    // Learn this account's fingerprints and, from a running balance, its balance.
    const updates: Partial<AccountRow> = {
      layoutKey,
      fileNamePattern: fileNamePattern(staged.fileName),
      updatedAt: now,
    };
    if (!account.last4 && parsed.last4Hint) updates.last4 = parsed.last4Hint;
    const withBalance = parsed.rows.filter((r) => r.balanceCents !== null);
    const last = withBalance.reduce<Row | null>((acc, r) => (!acc || r.date >= acc.date ? r : acc), null);
    if (last && (!account.balanceAsOf || last.date >= account.balanceAsOf || account.balanceSource !== 'statement')) {
      updates.balanceCents = last.balanceCents!;
      updates.balanceAsOf = last.date;
      updates.balanceSource = 'statement';
    }
    ctx.db.update(accounts).set(updates).where(eq(accounts.id, account.id)).run();
    return { record, transferCount };
  })();

  // Keep the original file so a fixed importer can read it again later.
  try {
    const dir = join(ctx.config.dataDir, 'imports');
    mkdirSync(dir, { recursive: true });
    writeFileSync(join(dir, `${result.record.id}-${staged.hash.slice(0, 12)}${extname(staged.fileName) || '.csv'}`), staged.text);
  } catch (error) {
    ctx.log.warn(`Could not keep a copy of ${staged.fileName}: ${(error as Error).message}`);
  }
  ctx.staging.delete(stagingId);
  logChange(ctx, userId, 'import', 'import', result.record.id, { file: staged.fileName, added: planned.length });
  return importSummary(ctx, result.record.id);
}

export function importSummary(ctx: AppContext, id: number): ImportSummary {
  const row = ctx.db
    .select({ i: imports, accountName: accounts.name })
    .from(imports)
    .innerJoin(accounts, eq(accounts.id, imports.accountId))
    .where(eq(imports.id, id))
    .get();
  if (!row) fail(404, 'not_found', 'That import wasn’t found.');
  const toReview =
    ctx.db
      .select({ n: count() })
      .from(transactions)
      .where(and(eq(transactions.importId, id), eq(transactions.needsReview, true)))
      .get()?.n ?? 0;
  const { i } = row;
  return {
    id: i.id,
    accountId: i.accountId,
    accountName: row.accountName,
    fileName: i.fileName,
    layoutKey: i.layoutKey,
    dateFrom: i.dateFrom,
    dateTo: i.dateTo,
    addedCount: i.addedCount,
    duplicateCount: i.duplicateCount,
    skippedCount: i.skippedCount,
    transferCount: i.transferCount,
    toReview,
    status: i.status,
    createdAt: i.createdAt,
  };
}

export function listImports(ctx: AppContext): ImportSummary[] {
  return ctx.db
    .select({ id: imports.id })
    .from(imports)
    .orderBy(desc(imports.createdAt))
    .limit(100)
    .all()
    .map((r) => importSummary(ctx, r.id));
}

/** Remove everything one import added. Paired transfers in other accounts go back to Review. */
export function undoImport(ctx: AppContext, id: number, userId: number): ImportSummary {
  const record = ctx.db.select().from(imports).where(eq(imports.id, id)).get();
  if (!record) fail(404, 'not_found', 'That import wasn’t found.');
  if (record.status === 'undone') fail(400, 'already_undone', 'That import was already undone.');
  const now = new Date().toISOString();

  ctx.sqlite.transaction(() => {
    const added = ctx.db.select().from(transactions).where(eq(transactions.importId, id)).all();
    const addedIds = added.map((t) => t.id);
    const peers = added.map((t) => t.transferPeerId).filter((p): p is number => p !== null && !addedIds.includes(p));
    if (peers.length) {
      ctx.db.update(transactions).set({ transferPeerId: null, needsReview: true, updatedAt: now }).where(inArray(transactions.id, peers)).run();
    }
    if (addedIds.length) ctx.db.delete(transactions).where(inArray(transactions.id, addedIds)).run();

    // Put the balance back if this import was the one that set it.
    const account = ctx.db.select().from(accounts).where(eq(accounts.id, record.accountId)).get();
    if (record.previousBalance && account?.balanceSource === 'statement' && account.balanceAsOf) {
      const fromThisFile = account.balanceAsOf >= (record.dateFrom ?? '') && account.balanceAsOf <= (record.dateTo ?? '');
      if (fromThisFile) {
        const prev = JSON.parse(record.previousBalance) as Pick<AccountRow, 'balanceCents' | 'balanceAsOf' | 'balanceSource'>;
        ctx.db.update(accounts).set({ ...prev, updatedAt: now }).where(eq(accounts.id, account.id)).run();
      }
    }
    ctx.db.update(imports).set({ status: 'undone', undoneAt: now }).where(eq(imports.id, id)).run();
  })();
  logChange(ctx, userId, 'undo_import', 'import', id);
  return importSummary(ctx, id);
}

export { MAX_BYTES };