Milestones 0 and 1: foundation and money in (20261007.10)
Foundation: single arm64 container (Hono + SQLite via Drizzle, React + Vite), local sign-in with long-lived sessions and passkeys, roles (read-only, full access, admin), nightly and pre-upgrade backups, light and dark themes, date-based version numbers, and the ideas lightbulb behind IDEAS_ENABLED. Money in: accounts with balances, CSV import with bank detection, a column mapping pop-up for unknown layouts, duplicate checks, transfer pairing and undo; one-tap review with learned rules; activity with search, splits, tags, notes and cash entries. Statement files are blocked from commits by a pre-commit hook; tests use synthetic fixtures only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
199
server/src/import/layouts.ts
Normal file
199
server/src/import/layouts.ts
Normal file
@@ -0,0 +1,199 @@
|
||||
import type { AccountKind, ImportColumnMap } from '@spendlight/shared';
|
||||
import { normalizeHeader } from './parse';
|
||||
|
||||
// Statement layouts Spendlight recognizes on sight. Each was confirmed against
|
||||
// the masked profile of a real export; anything else goes to the mapping pop-up,
|
||||
// and the answer is saved as a custom layout.
|
||||
|
||||
export interface Layout {
|
||||
key: string;
|
||||
name: string;
|
||||
institution: string | null;
|
||||
kind: AccountKind | null;
|
||||
/** Normalized header names that must all be present. */
|
||||
headers: string[];
|
||||
/** Header name → role. Missing roles are simply not used. */
|
||||
columns: {
|
||||
date: string;
|
||||
description: string;
|
||||
amount?: string;
|
||||
debit?: string;
|
||||
credit?: string;
|
||||
postedDate?: string;
|
||||
category?: string;
|
||||
type?: string;
|
||||
balance?: string;
|
||||
status?: string;
|
||||
cardNumber?: string;
|
||||
};
|
||||
positiveIs: 'out' | 'in';
|
||||
dateOrder: 'MDY' | 'DMY' | 'YMD';
|
||||
}
|
||||
|
||||
export const BUILT_IN_LAYOUTS: Layout[] = [
|
||||
{
|
||||
key: 'chase-card',
|
||||
name: 'Chase credit card',
|
||||
institution: 'Chase',
|
||||
kind: 'credit_card',
|
||||
headers: ['transaction date', 'post date', 'description', 'category', 'type', 'amount', 'memo'],
|
||||
columns: { date: 'transaction date', postedDate: 'post date', description: 'description', category: 'category', type: 'type', amount: 'amount' },
|
||||
positiveIs: 'in',
|
||||
dateOrder: 'MDY',
|
||||
},
|
||||
{
|
||||
key: 'discover-card',
|
||||
name: 'Discover card',
|
||||
institution: 'Discover',
|
||||
kind: 'credit_card',
|
||||
headers: ['trans. date', 'post date', 'description', 'amount', 'category'],
|
||||
columns: { date: 'trans. date', postedDate: 'post date', description: 'description', amount: 'amount', category: 'category' },
|
||||
positiveIs: 'out',
|
||||
dateOrder: 'MDY',
|
||||
},
|
||||
{
|
||||
key: 'bofa-deposit',
|
||||
name: 'Bank of America checking or savings',
|
||||
institution: 'Bank of America',
|
||||
kind: 'checking',
|
||||
headers: ['date', 'description', 'amount', 'running bal.'],
|
||||
columns: { date: 'date', description: 'description', amount: 'amount', balance: 'running bal.' },
|
||||
positiveIs: 'in',
|
||||
dateOrder: 'MDY',
|
||||
},
|
||||
{
|
||||
key: 'bofa-card',
|
||||
name: 'Bank of America credit card',
|
||||
institution: 'Bank of America',
|
||||
kind: 'credit_card',
|
||||
headers: ['posted date', 'reference number', 'payee', 'address', 'amount'],
|
||||
columns: { date: 'posted date', description: 'payee', amount: 'amount' },
|
||||
positiveIs: 'in',
|
||||
dateOrder: 'MDY',
|
||||
},
|
||||
{
|
||||
key: 'capital-one-card',
|
||||
name: 'Capital One card',
|
||||
institution: 'Capital One',
|
||||
kind: 'credit_card',
|
||||
headers: ['transaction date', 'posted date', 'card no.', 'description', 'category', 'debit', 'credit'],
|
||||
columns: { date: 'transaction date', postedDate: 'posted date', description: 'description', category: 'category', debit: 'debit', credit: 'credit', cardNumber: 'card no.' },
|
||||
positiveIs: 'out',
|
||||
dateOrder: 'YMD',
|
||||
},
|
||||
{
|
||||
key: 'citi-card',
|
||||
name: 'Citi card',
|
||||
institution: 'Citi',
|
||||
kind: 'credit_card',
|
||||
headers: ['status', 'date', 'description', 'debit', 'credit'],
|
||||
columns: { date: 'date', description: 'description', debit: 'debit', credit: 'credit', status: 'status' },
|
||||
positiveIs: 'out',
|
||||
dateOrder: 'MDY',
|
||||
},
|
||||
{
|
||||
key: 'apple-card',
|
||||
name: 'Apple Card',
|
||||
institution: 'Apple',
|
||||
kind: 'credit_card',
|
||||
headers: ['transaction date', 'clearing date', 'description', 'merchant', 'category', 'type', 'amount (usd)', 'purchased by'],
|
||||
columns: { date: 'transaction date', postedDate: 'clearing date', description: 'description', category: 'category', type: 'type', amount: 'amount (usd)' },
|
||||
positiveIs: 'out',
|
||||
dateOrder: 'MDY',
|
||||
},
|
||||
];
|
||||
|
||||
export interface ResolvedLayout {
|
||||
key: string;
|
||||
name: string;
|
||||
institution: string | null;
|
||||
kind: AccountKind | null;
|
||||
headerRow: number;
|
||||
map: ImportColumnMap & { type?: number; status?: number; cardNumber?: number };
|
||||
}
|
||||
|
||||
const rowSignature = (row: string[]) => row.map(normalizeHeader).join('|');
|
||||
|
||||
/** Find the header row and a built-in layout that fits it, scanning the top of the file. */
|
||||
export function detectBuiltIn(rows: string[][]): ResolvedLayout | null {
|
||||
const limit = Math.min(rows.length, 40);
|
||||
let best: { layout: Layout; row: number } | null = null;
|
||||
for (let r = 0; r < limit; r++) {
|
||||
const names = new Set(rows[r]!.map(normalizeHeader));
|
||||
for (const layout of BUILT_IN_LAYOUTS) {
|
||||
if (!layout.headers.every((h) => names.has(h))) continue;
|
||||
if (!best || layout.headers.length > best.layout.headers.length) best = { layout, row: r };
|
||||
}
|
||||
}
|
||||
if (!best) return null;
|
||||
const header = rows[best.row]!.map(normalizeHeader);
|
||||
const at = (name?: string) => (name ? header.indexOf(name) : -1);
|
||||
const index = (name?: string) => (at(name) >= 0 ? at(name) : undefined);
|
||||
const c = best.layout.columns;
|
||||
return {
|
||||
key: best.layout.key,
|
||||
name: best.layout.name,
|
||||
institution: best.layout.institution,
|
||||
kind: best.layout.kind,
|
||||
headerRow: best.row,
|
||||
map: {
|
||||
date: at(c.date),
|
||||
description: at(c.description),
|
||||
amount: index(c.amount),
|
||||
debit: index(c.debit),
|
||||
credit: index(c.credit),
|
||||
postedDate: index(c.postedDate),
|
||||
category: index(c.category),
|
||||
balance: index(c.balance),
|
||||
type: index(c.type),
|
||||
status: index(c.status),
|
||||
cardNumber: index(c.cardNumber),
|
||||
positiveIs: best.layout.positiveIs,
|
||||
dateOrder: best.layout.dateOrder,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Where the column headings are, for a file nobody has seen before. */
|
||||
export function findHeaderRow(rows: string[][]): number {
|
||||
for (let r = 0; r < Math.min(rows.length, 40); r++) {
|
||||
const row = rows[r]!;
|
||||
const labels = row.filter((c) => c && !/\d/.test(c)).length;
|
||||
const next = rows.slice(r + 1, r + 4);
|
||||
if (labels >= 2 && next.length > 0 && next.every((n) => n.length === row.length)) return r;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
export function headerSignature(rows: string[][], headerRow: number): string {
|
||||
return headerRow >= 0 ? rowSignature(rows[headerRow]!) : '';
|
||||
}
|
||||
|
||||
/** Best guesses for the mapping pop-up, from heading names. */
|
||||
export function guessMap(header: string[]): ImportColumnMap | null {
|
||||
const names = header.map(normalizeHeader);
|
||||
const find = (...patterns: RegExp[]) => {
|
||||
for (const p of patterns) {
|
||||
const i = names.findIndex((n) => p.test(n));
|
||||
if (i >= 0) return i;
|
||||
}
|
||||
return -1;
|
||||
};
|
||||
const date = find(/^(transaction |trans\.? )?date$/, /date/);
|
||||
const description = find(/description/, /payee|merchant|name/, /memo|details/);
|
||||
const amount = find(/^amount/, /amount/);
|
||||
const debit = find(/debit|withdrawal|charge/);
|
||||
const credit = find(/credit|deposit/);
|
||||
const balance = find(/balance/);
|
||||
const category = find(/category/);
|
||||
if (date < 0 || description < 0 || (amount < 0 && debit < 0)) return null;
|
||||
return {
|
||||
date,
|
||||
description,
|
||||
...(amount >= 0 ? { amount } : { debit, ...(credit >= 0 ? { credit } : {}) }),
|
||||
...(balance >= 0 ? { balance } : {}),
|
||||
...(category >= 0 ? { category } : {}),
|
||||
positiveIs: 'in',
|
||||
dateOrder: 'MDY',
|
||||
};
|
||||
}
|
||||
109
server/src/import/merchant.ts
Normal file
109
server/src/import/merchant.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
import { knownMerchant } from '../categorize/known';
|
||||
|
||||
// Turning "SQ *JOES COFFEE 8823 CHARLOTTE NC" into "Joes Coffee". Known
|
||||
// merchants get their proper names; everything else is tidied by rules.
|
||||
|
||||
const US_STATES = new Set(
|
||||
'AL AK AZ AR CA CO CT DE FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI WY DC PR'.split(
|
||||
' ',
|
||||
),
|
||||
);
|
||||
|
||||
// Payment processors that put their own name in front of the merchant's.
|
||||
const PROCESSOR = /^(SQ|TST|SP|PY|DD|IC|PP|BT|CKO|FSP|PAR|SNACK|LS|SPO|PAYPAL|PP\*|GOOGLE|STRIPE)\s*\*\s*/i;
|
||||
const NOISE_PREFIX =
|
||||
/^(POS|DEBIT CARD PURCHASE|CHECKCARD|CHECK CARD|PURCHASE AUTHORIZED ON \d{1,2}\/\d{1,2}|PURCHASE|RECURRING|PREAUTHORIZED|ACH|WEB|ONLINE)\s+/i;
|
||||
const KEEP_UPPER = new Set(['CVS', 'BP', 'AMC', 'UPS', 'USPS', 'IRS', 'YMCA', 'IKEA', 'AT&T', 'TJ', 'BJ', 'H&M', 'ATM', 'DMV', 'QT', 'KFC', 'USA', 'LLC', 'TV']);
|
||||
const SMALL_WORDS = new Set(['and', 'of', 'the', 'for', 'at', 'in', 'on', 'to', 'a']);
|
||||
|
||||
function titleCase(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.split(' ')
|
||||
.map((word, i) => {
|
||||
if (KEEP_UPPER.has(word.toUpperCase())) return word.toUpperCase();
|
||||
if (i > 0 && SMALL_WORDS.has(word)) return word;
|
||||
return word.replace(/(^|[-/.'&])([a-z])/g, (_, sep: string, ch: string) => sep + ch.toUpperCase());
|
||||
})
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
export interface CleanedMerchant {
|
||||
merchant: string;
|
||||
/** Last four digits of another account mentioned in the text, as in "transfer to SAV 1234". */
|
||||
mentionsLast4: string | null;
|
||||
}
|
||||
|
||||
export function cleanMerchant(description: string): CleanedMerchant {
|
||||
const raw = description.replace(/\s+/g, ' ').trim();
|
||||
const last4 = /\b(?:SAV|CHK|CRD|CHECKING|SAVINGS|ACCT|ACCOUNT|CARD|X{2,}|\*{2,}|ENDING(?: IN)?)\s*#?(\d{4})\b/i.exec(raw)?.[1] ?? null;
|
||||
|
||||
// Bank of America's own transfers and card payments read best spelled out.
|
||||
const boa = /^Online Banking transfer (to|from) (SAV|CHK)\b/i.exec(raw);
|
||||
if (boa) {
|
||||
const kind = boa[2]!.toUpperCase() === 'SAV' ? 'savings' : 'checking';
|
||||
return { merchant: `Transfer ${boa[1]!.toLowerCase()} ${kind}`, mentionsLast4: last4 };
|
||||
}
|
||||
if (/^Online Banking payment to CRD\b/i.test(raw)) {
|
||||
return { merchant: last4 ? `Payment to card ${last4}` : 'Card payment', mentionsLast4: last4 };
|
||||
}
|
||||
// Person-to-person payments keep the person's name.
|
||||
const p2p = /^(Zelle|Venmo|Cash ?App)\s+(?:payment|transfer|instant pmt)?\s*(to|from)\s+(.+?)(?:\s+(?:Conf(?:irmation)?#?|ID:|on\s+\d).*)?$/i.exec(raw);
|
||||
if (p2p) {
|
||||
const service = p2p[1]!.replace(/\s+/g, ' ');
|
||||
return { merchant: `${titleCase(service)} ${p2p[2]!.toLowerCase()} ${titleCase(p2p[3]!.trim())}`, mentionsLast4: null };
|
||||
}
|
||||
|
||||
const known = knownMerchant(raw);
|
||||
if (known?.name) return { merchant: known.name, mentionsLast4: last4 };
|
||||
|
||||
let s = raw;
|
||||
// ACH entries: "COMPANY NAME DES:E-PAYMENT ID:123 INDN:... CO ID:... WEB"
|
||||
const ach = /^(.*?)\s+DES:/i.exec(s);
|
||||
if (ach) s = ach[1]!;
|
||||
s = s.replace(NOISE_PREFIX, '');
|
||||
const processor = PROCESSOR.exec(s);
|
||||
if (processor) s = s.slice(processor[0].length);
|
||||
// "MERCHANT*ORDER123" keeps the merchant; anything after a * is a reference.
|
||||
if (s.includes('*')) s = s.split('*')[0]!.trim() || s.replace(/\*/g, ' ');
|
||||
// Card descriptors are written in capitals; that is judged on the merchant
|
||||
// part alone, since bank text around it (like "INDN:Jane Doe") can be mixed.
|
||||
const shouting = s === s.toUpperCase();
|
||||
|
||||
// Bill-pay entries name the payee, then say how it was paid.
|
||||
s = s.replace(/\s+(Bill Payment|Bill Pay|Online Pmt|Online Payment|Web Pmt|ACH Pmt|Autopay)$/i, '');
|
||||
// "NAME - CODE123 ..." : the part after the dash is a reference.
|
||||
const dash = / - (?=.*\d)/.exec(s);
|
||||
if (dash && dash.index > 2) s = s.slice(0, dash.index);
|
||||
// Everything after a store number is location detail.
|
||||
const store = /\s#\s?\d+/.exec(s);
|
||||
if (store) s = s.slice(0, store.index);
|
||||
const hadLocation = !!store;
|
||||
s = s
|
||||
.replace(/\b\d{3}[-.]\d{3}[-.]\d{4}\b/g, ' ') // phone numbers
|
||||
.replace(/[A-Z.]*(?=[A-Z0-9]*\d)(?=[A-Z0-9]*[A-Z])\b[A-Z0-9]{6,}\b/gi, ' ') // reference codes mixing letters and digits
|
||||
.replace(/\b\d{3,}\b/g, ' ') // long numbers
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
|
||||
// Card descriptors (written in capitals) end with "CITY ST": drop the state,
|
||||
// then the city word. Bill-pay payees in mixed case keep every word.
|
||||
const words = s.split(' ');
|
||||
if ((shouting || hadLocation) && words.length >= 3 && US_STATES.has(words.at(-1)!.toUpperCase()) && words.at(-1) === words.at(-1)!.toUpperCase()) {
|
||||
words.pop();
|
||||
if (words.length >= 2) words.pop();
|
||||
s = words.join(' ');
|
||||
}
|
||||
s = s.replace(/[\s,.;:/-]+$/, '').trim();
|
||||
|
||||
// Capitals-only descriptors get title case; text already in normal case keeps its own.
|
||||
const tidy = (text: string) => (shouting ? titleCase(text) : text.charAt(0).toUpperCase() + text.slice(1));
|
||||
const merchant = tidy(s || raw.slice(0, 40));
|
||||
return { merchant, mentionsLast4: last4 };
|
||||
}
|
||||
|
||||
/** How duplicates are recognized: same account, day, amount and cleaned description. */
|
||||
export function dedupeKey(accountId: number, date: string, amountCents: number, description: string): string {
|
||||
const desc = description.toUpperCase().replace(/[^A-Z0-9]/g, '');
|
||||
return `${accountId}|${date}|${amountCents}|${desc}`;
|
||||
}
|
||||
72
server/src/import/parse.ts
Normal file
72
server/src/import/parse.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
import Papa from 'papaparse';
|
||||
|
||||
// Reading statement files into plain rows, and turning their text into dates
|
||||
// and cents. Banks disagree on almost everything, so these are forgiving.
|
||||
|
||||
/** Every non-blank row of a delimited file, as trimmed strings. */
|
||||
export function readTable(text: string): string[][] {
|
||||
const clean = text.replace(/^/, '');
|
||||
const result = Papa.parse<string[]>(clean, { skipEmptyLines: 'greedy', delimitersToGuess: [',', '\t', ';', '|'] });
|
||||
return result.data.map((row) => row.map((cell) => (cell ?? '').trim()));
|
||||
}
|
||||
|
||||
export const normalizeHeader = (cell: string) => cell.trim().toLowerCase().replace(/\s+/g, ' ');
|
||||
|
||||
/**
|
||||
* "1,234.56" → 123456, "-$5.00" → -500, "(5.00)" → -500, "5.00 CR" → 500 with
|
||||
* the sign flipped by the caller's rules. Returns null for anything else.
|
||||
*/
|
||||
export function parseCents(raw: string | undefined): number | null {
|
||||
if (raw === undefined) return null;
|
||||
let s = raw.trim().replace(/[−–]/g, '-');
|
||||
if (!s) return null;
|
||||
let negative = false;
|
||||
if (/^\(.*\)$/.test(s)) {
|
||||
negative = true;
|
||||
s = s.slice(1, -1);
|
||||
}
|
||||
const suffix = /\s*(CR|DR)$/i.exec(s);
|
||||
if (suffix) s = s.slice(0, suffix.index);
|
||||
s = s.replace(/[$,\s]/g, '');
|
||||
if (s.startsWith('-')) {
|
||||
negative = !negative;
|
||||
s = s.slice(1);
|
||||
} else if (s.startsWith('+')) s = s.slice(1);
|
||||
if (!/^\d+(\.\d{1,4})?$|^\.\d{1,4}$/.test(s)) return null;
|
||||
const cents = Math.round(Number(s) * 100);
|
||||
return negative ? -cents : cents;
|
||||
}
|
||||
|
||||
const pad = (n: number) => String(n).padStart(2, '0');
|
||||
|
||||
function validDate(y: number, m: number, d: number): string | null {
|
||||
if (y < 1990 || y > 2100 || m < 1 || m > 12 || d < 1 || d > 31) return null;
|
||||
const date = new Date(Date.UTC(y, m - 1, d));
|
||||
if (date.getUTCMonth() !== m - 1) return null;
|
||||
return `${y}-${pad(m)}-${pad(d)}`;
|
||||
}
|
||||
|
||||
/** A statement date as YYYY-MM-DD, or null. `order` settles 03/04/2026. */
|
||||
export function parseDate(raw: string | undefined, order: 'MDY' | 'DMY' | 'YMD' = 'MDY'): string | null {
|
||||
if (!raw) return null;
|
||||
const s = raw.trim();
|
||||
let m = /^(\d{4})-(\d{1,2})-(\d{1,2})/.exec(s);
|
||||
if (m) return validDate(+m[1]!, +m[2]!, +m[3]!);
|
||||
m = /^(\d{4})(\d{2})(\d{2})(\d{6})?/.exec(s);
|
||||
if (m && s.length >= 8 && /^\d{8}/.test(s)) return validDate(+m[1]!, +m[2]!, +m[3]!);
|
||||
m = /^(\d{1,4})[/.-](\d{1,2})[/.-](\d{2,4})$/.exec(s);
|
||||
if (!m) return null;
|
||||
let [a, b, c] = [+m[1]!, +m[2]!, +m[3]!];
|
||||
if (m[1]!.length === 4) return validDate(a, b, c);
|
||||
if (c < 100) c += 2000;
|
||||
return order === 'DMY' ? validDate(c, b, a) : validDate(c, a, b);
|
||||
}
|
||||
|
||||
/** Days between two YYYY-MM-DD dates. */
|
||||
export function daysBetween(a: string, b: string): number {
|
||||
return Math.round((Date.parse(`${b}T00:00:00Z`) - Date.parse(`${a}T00:00:00Z`)) / 86_400_000);
|
||||
}
|
||||
|
||||
export function shiftDate(date: string, days: number): string {
|
||||
return new Date(Date.parse(`${date}T00:00:00Z`) + days * 86_400_000).toISOString().slice(0, 10);
|
||||
}
|
||||
534
server/src/import/service.ts
Normal file
534
server/src/import/service.ts
Normal file
@@ -0,0 +1,534 @@
|
||||
import type { AccountKind, ImportColumnMap, ImportPreview, ImportSummary } from '@spendlight/shared';
|
||||
import { and, between, count, desc, eq, inArray } from 'drizzle-orm';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||
import { extname, join } from 'node:path';
|
||||
import { logChange } from '../audit';
|
||||
import { Suggester } from '../categorize/suggest';
|
||||
import type { AppContext } from '../context';
|
||||
import { accounts, importLayouts, imports, transactions, type AccountRow } from '../db/schema';
|
||||
import { fail } from '../http';
|
||||
import { detectBuiltIn, findHeaderRow, guessMap, headerSignature, type ResolvedLayout } from './layouts';
|
||||
import { cleanMerchant, dedupeKey } from './merchant';
|
||||
import { daysBetween, parseCents, parseDate, readTable, shiftDate } from './parse';
|
||||
import { pairTransfers } from './transfers';
|
||||
|
||||
// The import pipeline: stage a file, preview what it would add, then commit.
|
||||
// Nothing is written until commit, and every commit can be undone whole.
|
||||
|
||||
const MAX_BYTES = 10 * 1024 * 1024;
|
||||
const STAGE_TTL_MS = 60 * 60_000;
|
||||
|
||||
interface Staged {
|
||||
id: string;
|
||||
fileName: string;
|
||||
text: string;
|
||||
hash: string;
|
||||
userId: number;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
export class StagingStore {
|
||||
private files = new Map<string, Staged>();
|
||||
|
||||
add(fileName: string, text: string, userId: number): Staged {
|
||||
this.sweep();
|
||||
const staged = {
|
||||
id: randomUUID(),
|
||||
fileName,
|
||||
text,
|
||||
hash: createHash('sha256').update(text).digest('hex'),
|
||||
userId,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
this.files.set(staged.id, staged);
|
||||
return staged;
|
||||
}
|
||||
|
||||
get(id: string, userId: number): Staged {
|
||||
const staged = this.files.get(id);
|
||||
if (!staged || staged.userId !== userId) fail(404, 'expired', 'That file has expired. Please add it again.');
|
||||
return staged;
|
||||
}
|
||||
|
||||
delete(id: string): void {
|
||||
this.files.delete(id);
|
||||
}
|
||||
|
||||
private sweep() {
|
||||
for (const [id, f] of this.files) if (Date.now() - f.createdAt > STAGE_TTL_MS) this.files.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
interface Row {
|
||||
date: string;
|
||||
postedDate: string | null;
|
||||
description: string;
|
||||
merchant: string;
|
||||
mentionsLast4: string | null;
|
||||
amountCents: number;
|
||||
bankCategory: string | null;
|
||||
bankType: string | null;
|
||||
balanceCents: number | null;
|
||||
}
|
||||
|
||||
interface Parsed {
|
||||
layout: ResolvedLayout | null;
|
||||
rows: Row[];
|
||||
skipped: number;
|
||||
header: string[] | null;
|
||||
sample: string[][];
|
||||
suggestedMap: ImportColumnMap | null;
|
||||
signature: string;
|
||||
last4Hint: string | null;
|
||||
openingBalance: { date: string; cents: number } | null;
|
||||
}
|
||||
|
||||
function fileNameLast4(fileName: string): string | null {
|
||||
return /^[A-Za-z]+(\d{4})[_-]/.exec(fileName)?.[1] ?? /(?:x|ending|acct|card)[-_ ]?(\d{4})\b/i.exec(fileName)?.[1] ?? null;
|
||||
}
|
||||
|
||||
/** The file name with its digits generalized, so "Chase1234_Activity_20261007.csv" matches next month's file. */
|
||||
const fileNamePattern = (fileName: string) => fileName.replace(/\d{5,}/g, '#').replace(/\(\d+\)/g, '');
|
||||
|
||||
function parseFile(ctx: AppContext, staged: Staged, mapping?: ImportColumnMap): Parsed {
|
||||
const table = readTable(staged.text);
|
||||
let layout = mapping ? null : detectBuiltIn(table);
|
||||
const headerRow = layout?.headerRow ?? findHeaderRow(table);
|
||||
const signature = headerSignature(table, headerRow);
|
||||
|
||||
if (!layout && headerRow >= 0) {
|
||||
const custom = mapping
|
||||
? { name: 'Your saved layout', config: JSON.stringify(mapping) }
|
||||
: ctx.db.select().from(importLayouts).where(eq(importLayouts.signature, signature)).get();
|
||||
if (custom) {
|
||||
layout = {
|
||||
key: 'id' in custom ? `custom-${custom.id}` : 'custom',
|
||||
name: custom.name,
|
||||
institution: null,
|
||||
kind: null,
|
||||
headerRow,
|
||||
map: JSON.parse(custom.config) as ImportColumnMap,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const header = headerRow >= 0 ? table[headerRow]! : null;
|
||||
const sample = table.slice(Math.max(0, headerRow + 1), Math.max(0, headerRow + 1) + 5);
|
||||
if (!layout) {
|
||||
return { layout: null, rows: [], skipped: 0, header, sample, suggestedMap: header ? guessMap(header) : null, signature, last4Hint: null, openingBalance: null };
|
||||
}
|
||||
|
||||
const m = layout.map;
|
||||
const rows: Row[] = [];
|
||||
let skipped = 0;
|
||||
let openingBalance: Parsed['openingBalance'] = null;
|
||||
let last4Hint = fileNameLast4(staged.fileName);
|
||||
|
||||
for (const cells of table.slice(layout.headerRow + 1)) {
|
||||
if (cells.every((c) => !c)) continue;
|
||||
const date = parseDate(cells[m.date], m.dateOrder);
|
||||
const description = (cells[m.description] ?? '').trim();
|
||||
const balanceCents = m.balance !== undefined ? parseCents(cells[m.balance]) : null;
|
||||
if (!date) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
// "Beginning balance as of ..." rows carry a balance but no amount.
|
||||
let amount: number | null;
|
||||
if (m.amount !== undefined) {
|
||||
const raw = parseCents(cells[m.amount]);
|
||||
amount = raw === null ? null : m.positiveIs === 'out' ? -raw : raw;
|
||||
} else {
|
||||
const debit = m.debit !== undefined ? parseCents(cells[m.debit]) : null;
|
||||
const credit = m.credit !== undefined ? parseCents(cells[m.credit]) : null;
|
||||
amount = debit ? -Math.abs(debit) : credit ? Math.abs(credit) : debit === 0 || credit === 0 ? 0 : null;
|
||||
}
|
||||
if (amount === null) {
|
||||
if (balanceCents !== null && /beginning balance/i.test(description)) openingBalance = { date, cents: balanceCents };
|
||||
else skipped++;
|
||||
continue;
|
||||
}
|
||||
if ('status' in m && m.status !== undefined && /pending/i.test(cells[m.status] ?? '')) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
if ('cardNumber' in m && m.cardNumber !== undefined) last4Hint ??= /(\d{4})\s*$/.exec(cells[m.cardNumber] ?? '')?.[1] ?? null;
|
||||
const { merchant, mentionsLast4 } = cleanMerchant(description);
|
||||
rows.push({
|
||||
date,
|
||||
postedDate: m.postedDate !== undefined ? parseDate(cells[m.postedDate], m.dateOrder) : null,
|
||||
description,
|
||||
merchant,
|
||||
mentionsLast4,
|
||||
amountCents: amount,
|
||||
bankCategory: m.category !== undefined ? cells[m.category] || null : null,
|
||||
bankType: 'type' in m && m.type !== undefined ? cells[m.type] || null : null,
|
||||
balanceCents,
|
||||
});
|
||||
}
|
||||
return { layout, rows, skipped, header, sample, suggestedMap: null, signature, last4Hint, openingBalance };
|
||||
}
|
||||
|
||||
/** Which account a file belongs to, judged from fingerprints the person never has to manage. */
|
||||
function matchAccount(ctx: AppContext, parsed: Parsed, fileName: string): ImportPreview['account'] {
|
||||
const open = ctx.db.select().from(accounts).where(eq(accounts.status, 'open')).all();
|
||||
if (!open.length || !parsed.layout) return null;
|
||||
|
||||
if (parsed.last4Hint) {
|
||||
const byLast4 = open.filter((a) => a.last4 === parsed.last4Hint);
|
||||
if (byLast4.length === 1) return { id: byLast4[0]!.id, confidence: 'certain', reason: `The file is for the account ending ${parsed.last4Hint}.` };
|
||||
}
|
||||
|
||||
// Transactions already in an account are the strongest evidence.
|
||||
if (parsed.rows.length) {
|
||||
const dates = parsed.rows.map((r) => r.date).sort();
|
||||
let best: { account: AccountRow; hits: number } | null = null;
|
||||
for (const account of open) {
|
||||
const keys = new Set(
|
||||
ctx.db
|
||||
.select({ key: transactions.dedupeKey })
|
||||
.from(transactions)
|
||||
.where(and(eq(transactions.accountId, account.id), between(transactions.date, dates[0]!, dates.at(-1)!)))
|
||||
.all()
|
||||
.map((r) => r.key),
|
||||
);
|
||||
const hits = parsed.rows.filter((r) => keys.has(dedupeKey(account.id, r.date, r.amountCents, r.description))).length;
|
||||
if (hits && (!best || hits > best.hits)) best = { account, hits };
|
||||
}
|
||||
if (best && (best.hits >= 3 || best.hits >= parsed.rows.length * 0.3)) {
|
||||
return { id: best.account.id, confidence: 'certain', reason: `${best.hits} of these transactions are already in ${best.account.name}.` };
|
||||
}
|
||||
}
|
||||
|
||||
// A statement that starts exactly where an account's last one ended.
|
||||
if (parsed.openingBalance) {
|
||||
const continues = open.filter(
|
||||
(a) => a.balanceSource === 'statement' && a.balanceCents === parsed.openingBalance!.cents && a.layoutKey === parsed.layout!.key,
|
||||
);
|
||||
if (continues.length === 1) return { id: continues[0]!.id, confidence: 'certain', reason: `It picks up where ${continues[0]!.name}’s last statement ended.` };
|
||||
}
|
||||
|
||||
// Softer hints only count when no other account at the same bank could be meant.
|
||||
const institution = parsed.layout.institution?.toLowerCase();
|
||||
const sameBank = open.filter((a) => a.layoutKey === parsed.layout!.key || (institution && a.institution?.toLowerCase() === institution));
|
||||
const sameLayout = open.filter((a) => a.layoutKey === parsed.layout!.key);
|
||||
const samePattern = sameLayout.filter((a) => a.fileNamePattern === fileNamePattern(fileName));
|
||||
if (sameBank.length === 1 && samePattern.length === 1) return { id: samePattern[0]!.id, confidence: 'likely', reason: `Files like this have gone to ${samePattern[0]!.name} before.` };
|
||||
if (sameBank.length === 1 && sameLayout.length === 1) return { id: sameLayout[0]!.id, confidence: 'likely', reason: `${sameLayout[0]!.name} uses this bank’s layout.` };
|
||||
|
||||
const fresh = open.filter(
|
||||
(a) => !a.layoutKey && a.institution && parsed.layout!.institution && a.institution.toLowerCase() === parsed.layout!.institution.toLowerCase(),
|
||||
);
|
||||
if (fresh.length === 1) return { id: fresh[0]!.id, confidence: 'likely', reason: `It’s a ${parsed.layout.institution} file.` };
|
||||
return null;
|
||||
}
|
||||
|
||||
interface Planned {
|
||||
row: Row;
|
||||
key: string;
|
||||
status: 'new' | 'duplicate' | 'possible';
|
||||
possibleId: string | null;
|
||||
existingDate: string | null;
|
||||
}
|
||||
|
||||
/** Count-based duplicate check: import only copies beyond those already stored. */
|
||||
function planRows(ctx: AppContext, accountId: number, rows: Row[]): Planned[] {
|
||||
if (!rows.length) return [];
|
||||
const dates = rows.map((r) => r.date).sort();
|
||||
const from = dates[0]!;
|
||||
const to = dates.at(-1)!;
|
||||
const existing = ctx.db
|
||||
.select()
|
||||
.from(transactions)
|
||||
.where(and(eq(transactions.accountId, accountId), between(transactions.date, shiftDate(from, -3), shiftDate(to, 3))))
|
||||
.all();
|
||||
const existingCount = new Map<string, number>();
|
||||
for (const t of existing) existingCount.set(t.dedupeKey, (existingCount.get(t.dedupeKey) ?? 0) + 1);
|
||||
|
||||
const seen = new Map<string, number>();
|
||||
const planned: Planned[] = rows.map((row) => {
|
||||
const key = dedupeKey(accountId, row.date, row.amountCents, row.description);
|
||||
const n = (seen.get(key) ?? 0) + 1;
|
||||
seen.set(key, n);
|
||||
return { row, key, status: n <= (existingCount.get(key) ?? 0) ? 'duplicate' : 'new', possibleId: null, existingDate: null };
|
||||
});
|
||||
|
||||
// Near matches: same amount and merchant a day or three apart, inside this
|
||||
// file's dates, with no exact twin in the file. Usually a pending charge that
|
||||
// later posted with a new date.
|
||||
const fileKeys = new Set(planned.map((p) => p.key));
|
||||
const used = new Set<number>();
|
||||
for (const p of planned) {
|
||||
if (p.status !== 'new') continue;
|
||||
const near = existing.find(
|
||||
(t) =>
|
||||
!used.has(t.id) &&
|
||||
!fileKeys.has(t.dedupeKey) &&
|
||||
t.amountCents === p.row.amountCents &&
|
||||
t.merchant === p.row.merchant &&
|
||||
t.date !== p.row.date &&
|
||||
t.date >= from &&
|
||||
t.date <= to &&
|
||||
Math.abs(daysBetween(t.date, p.row.date)) <= 3,
|
||||
);
|
||||
if (near) {
|
||||
used.add(near.id);
|
||||
p.status = 'possible';
|
||||
p.possibleId = `${p.key}#${seen.get(p.key)}`;
|
||||
p.existingDate = near.date;
|
||||
}
|
||||
}
|
||||
return planned;
|
||||
}
|
||||
|
||||
function accountHint(parsed: Parsed): ImportPreview['accountHint'] {
|
||||
if (!parsed.layout) return null;
|
||||
const kind: AccountKind = parsed.layout.kind ?? 'checking';
|
||||
const institution = parsed.layout.institution;
|
||||
const last4 = parsed.last4Hint;
|
||||
const what = kind === 'credit_card' ? 'card' : kind;
|
||||
const name = [institution, what].filter(Boolean).join(' ') + (last4 ? ` ${last4}` : '');
|
||||
return { kind, institution, last4, name: name.charAt(0).toUpperCase() + name.slice(1) };
|
||||
}
|
||||
|
||||
export function previewImport(
|
||||
ctx: AppContext,
|
||||
stagingId: string,
|
||||
userId: number,
|
||||
options: { mapping?: ImportColumnMap; accountId?: number },
|
||||
): ImportPreview {
|
||||
const staged = ctx.staging.get(stagingId, userId);
|
||||
const parsed = parseFile(ctx, staged, options.mapping);
|
||||
const account = options.accountId
|
||||
? { id: options.accountId, confidence: 'certain' as const, reason: 'You chose this account.' }
|
||||
: matchAccount(ctx, parsed, staged.fileName);
|
||||
const planned = account ? planRows(ctx, account.id, parsed.rows) : parsed.rows.map((row) => ({ row, status: 'new' as const }));
|
||||
const dates = parsed.rows.map((r) => r.date).sort();
|
||||
const fresh = planned.filter((p) => p.status === 'new');
|
||||
|
||||
return {
|
||||
stagingId,
|
||||
fileName: staged.fileName,
|
||||
layout: parsed.layout ? { key: parsed.layout.key, name: parsed.layout.name } : null,
|
||||
sampleRows: parsed.layout ? [] : parsed.sample,
|
||||
headerRow: parsed.layout ? null : parsed.header,
|
||||
suggestedMap: parsed.suggestedMap,
|
||||
account,
|
||||
dateFrom: dates[0] ?? null,
|
||||
dateTo: dates.at(-1) ?? null,
|
||||
counts: {
|
||||
rows: parsed.rows.length,
|
||||
new: fresh.length,
|
||||
duplicates: planned.filter((p) => p.status === 'duplicate').length,
|
||||
possibleDuplicates: planned.filter((p) => p.status === 'possible').length,
|
||||
skipped: parsed.skipped,
|
||||
},
|
||||
newRows: fresh.slice(0, 8).map((p) => ({ date: p.row.date, merchant: p.row.merchant, amountCents: p.row.amountCents })),
|
||||
possibleDuplicates: planned
|
||||
.filter((p): p is Planned => p.status === 'possible')
|
||||
.map((p) => ({ key: p.possibleId!, date: p.row.date, merchant: p.row.merchant, amountCents: p.row.amountCents, existingDate: p.existingDate! })),
|
||||
accountHint: accountHint(parsed),
|
||||
};
|
||||
}
|
||||
|
||||
export function commitImport(
|
||||
ctx: AppContext,
|
||||
stagingId: string,
|
||||
userId: number,
|
||||
options: { accountId: number; mapping?: ImportColumnMap; includePossible?: string[] },
|
||||
): ImportSummary {
|
||||
const staged = ctx.staging.get(stagingId, userId);
|
||||
const account = ctx.db.select().from(accounts).where(eq(accounts.id, options.accountId)).get();
|
||||
if (!account || account.status !== 'open') fail(400, 'no_account', 'Please choose an open account for this file.');
|
||||
const parsed = parseFile(ctx, staged, options.mapping);
|
||||
if (!parsed.layout) fail(400, 'needs_mapping', 'Spendlight needs to know which column is which before importing this file.');
|
||||
if (!parsed.rows.length) fail(400, 'empty', 'No transactions were found in this file.');
|
||||
|
||||
const include = new Set(options.includePossible ?? []);
|
||||
const planned = planRows(ctx, account.id, parsed.rows).filter(
|
||||
(p) => p.status === 'new' || (p.status === 'possible' && include.has(p.possibleId!)),
|
||||
);
|
||||
const duplicateCount = parsed.rows.length - planned.length;
|
||||
const dates = parsed.rows.map((r) => r.date).sort();
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const result = ctx.sqlite.transaction(() => {
|
||||
const suggester = new Suggester(ctx);
|
||||
const record = ctx.db
|
||||
.insert(imports)
|
||||
.values({
|
||||
accountId: account.id,
|
||||
fileName: staged.fileName,
|
||||
fileHash: staged.hash,
|
||||
layoutKey: parsed.layout!.key,
|
||||
dateFrom: dates[0] ?? null,
|
||||
dateTo: dates.at(-1) ?? null,
|
||||
rowCount: parsed.rows.length,
|
||||
addedCount: planned.length,
|
||||
duplicateCount,
|
||||
skippedCount: parsed.skipped,
|
||||
previousBalance: JSON.stringify({
|
||||
balanceCents: account.balanceCents,
|
||||
balanceAsOf: account.balanceAsOf,
|
||||
balanceSource: account.balanceSource,
|
||||
}),
|
||||
userId,
|
||||
createdAt: now,
|
||||
})
|
||||
.returning()
|
||||
.get();
|
||||
|
||||
const ids: number[] = [];
|
||||
for (const { row, key } of planned) {
|
||||
const rule = suggester.rule(row.merchant, account.id);
|
||||
const merchant = rule?.renameTo || row.merchant;
|
||||
const s = suggester.suggest({ ...row, merchant, accountId: account.id });
|
||||
const inserted = ctx.db
|
||||
.insert(transactions)
|
||||
.values({
|
||||
accountId: account.id,
|
||||
importId: record.id,
|
||||
date: row.date,
|
||||
postedDate: row.postedDate,
|
||||
amountCents: row.amountCents,
|
||||
description: row.description,
|
||||
merchant,
|
||||
categoryId: s.categoryId,
|
||||
suggestedBy: s.source,
|
||||
needsReview: true,
|
||||
bankCategory: row.bankCategory,
|
||||
bankType: row.bankType,
|
||||
dedupeKey: key,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.returning({ id: transactions.id })
|
||||
.get();
|
||||
ids.push(inserted.id);
|
||||
}
|
||||
|
||||
const transferCount = pairTransfers(ctx, suggester, ids);
|
||||
ctx.db.update(imports).set({ transferCount }).where(eq(imports.id, record.id)).run();
|
||||
|
||||
// A layout from the mapping pop-up is saved, and remembered by its stored key.
|
||||
let layoutKey = parsed.layout!.key;
|
||||
if (options.mapping) {
|
||||
const existing = ctx.db.select().from(importLayouts).where(eq(importLayouts.signature, parsed.signature)).get();
|
||||
const layoutId = existing
|
||||
? (ctx.db.update(importLayouts).set({ config: JSON.stringify(options.mapping) }).where(eq(importLayouts.id, existing.id)).run(), existing.id)
|
||||
: ctx.db
|
||||
.insert(importLayouts)
|
||||
.values({ name: `${account.name} layout`, signature: parsed.signature, config: JSON.stringify(options.mapping), createdAt: now })
|
||||
.returning()
|
||||
.get().id;
|
||||
layoutKey = `custom-${layoutId}`;
|
||||
ctx.db.update(imports).set({ layoutKey }).where(eq(imports.id, record.id)).run();
|
||||
}
|
||||
|
||||
// Learn this account's fingerprints and, from a running balance, its balance.
|
||||
const updates: Partial<AccountRow> = {
|
||||
layoutKey,
|
||||
fileNamePattern: fileNamePattern(staged.fileName),
|
||||
updatedAt: now,
|
||||
};
|
||||
if (!account.last4 && parsed.last4Hint) updates.last4 = parsed.last4Hint;
|
||||
const withBalance = parsed.rows.filter((r) => r.balanceCents !== null);
|
||||
const last = withBalance.reduce<Row | null>((acc, r) => (!acc || r.date >= acc.date ? r : acc), null);
|
||||
if (last && (!account.balanceAsOf || last.date >= account.balanceAsOf || account.balanceSource !== 'statement')) {
|
||||
updates.balanceCents = last.balanceCents!;
|
||||
updates.balanceAsOf = last.date;
|
||||
updates.balanceSource = 'statement';
|
||||
}
|
||||
ctx.db.update(accounts).set(updates).where(eq(accounts.id, account.id)).run();
|
||||
return { record, transferCount };
|
||||
})();
|
||||
|
||||
// Keep the original file so a fixed importer can read it again later.
|
||||
try {
|
||||
const dir = join(ctx.config.dataDir, 'imports');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, `${result.record.id}-${staged.hash.slice(0, 12)}${extname(staged.fileName) || '.csv'}`), staged.text);
|
||||
} catch (error) {
|
||||
ctx.log.warn(`Could not keep a copy of ${staged.fileName}: ${(error as Error).message}`);
|
||||
}
|
||||
ctx.staging.delete(stagingId);
|
||||
logChange(ctx, userId, 'import', 'import', result.record.id, { file: staged.fileName, added: planned.length });
|
||||
return importSummary(ctx, result.record.id);
|
||||
}
|
||||
|
||||
export function importSummary(ctx: AppContext, id: number): ImportSummary {
|
||||
const row = ctx.db
|
||||
.select({ i: imports, accountName: accounts.name })
|
||||
.from(imports)
|
||||
.innerJoin(accounts, eq(accounts.id, imports.accountId))
|
||||
.where(eq(imports.id, id))
|
||||
.get();
|
||||
if (!row) fail(404, 'not_found', 'That import wasn’t found.');
|
||||
const toReview =
|
||||
ctx.db
|
||||
.select({ n: count() })
|
||||
.from(transactions)
|
||||
.where(and(eq(transactions.importId, id), eq(transactions.needsReview, true)))
|
||||
.get()?.n ?? 0;
|
||||
const { i } = row;
|
||||
return {
|
||||
id: i.id,
|
||||
accountId: i.accountId,
|
||||
accountName: row.accountName,
|
||||
fileName: i.fileName,
|
||||
layoutKey: i.layoutKey,
|
||||
dateFrom: i.dateFrom,
|
||||
dateTo: i.dateTo,
|
||||
addedCount: i.addedCount,
|
||||
duplicateCount: i.duplicateCount,
|
||||
skippedCount: i.skippedCount,
|
||||
transferCount: i.transferCount,
|
||||
toReview,
|
||||
status: i.status,
|
||||
createdAt: i.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
export function listImports(ctx: AppContext): ImportSummary[] {
|
||||
return ctx.db
|
||||
.select({ id: imports.id })
|
||||
.from(imports)
|
||||
.orderBy(desc(imports.createdAt))
|
||||
.limit(100)
|
||||
.all()
|
||||
.map((r) => importSummary(ctx, r.id));
|
||||
}
|
||||
|
||||
/** Remove everything one import added. Paired transfers in other accounts go back to Review. */
|
||||
export function undoImport(ctx: AppContext, id: number, userId: number): ImportSummary {
|
||||
const record = ctx.db.select().from(imports).where(eq(imports.id, id)).get();
|
||||
if (!record) fail(404, 'not_found', 'That import wasn’t found.');
|
||||
if (record.status === 'undone') fail(400, 'already_undone', 'That import was already undone.');
|
||||
const now = new Date().toISOString();
|
||||
|
||||
ctx.sqlite.transaction(() => {
|
||||
const added = ctx.db.select().from(transactions).where(eq(transactions.importId, id)).all();
|
||||
const addedIds = added.map((t) => t.id);
|
||||
const peers = added.map((t) => t.transferPeerId).filter((p): p is number => p !== null && !addedIds.includes(p));
|
||||
if (peers.length) {
|
||||
ctx.db.update(transactions).set({ transferPeerId: null, needsReview: true, updatedAt: now }).where(inArray(transactions.id, peers)).run();
|
||||
}
|
||||
if (addedIds.length) ctx.db.delete(transactions).where(inArray(transactions.id, addedIds)).run();
|
||||
|
||||
// Put the balance back if this import was the one that set it.
|
||||
const account = ctx.db.select().from(accounts).where(eq(accounts.id, record.accountId)).get();
|
||||
if (record.previousBalance && account?.balanceSource === 'statement' && account.balanceAsOf) {
|
||||
const fromThisFile = account.balanceAsOf >= (record.dateFrom ?? '') && account.balanceAsOf <= (record.dateTo ?? '');
|
||||
if (fromThisFile) {
|
||||
const prev = JSON.parse(record.previousBalance) as Pick<AccountRow, 'balanceCents' | 'balanceAsOf' | 'balanceSource'>;
|
||||
ctx.db.update(accounts).set({ ...prev, updatedAt: now }).where(eq(accounts.id, account.id)).run();
|
||||
}
|
||||
}
|
||||
ctx.db.update(imports).set({ status: 'undone', undoneAt: now }).where(eq(imports.id, id)).run();
|
||||
})();
|
||||
logChange(ctx, userId, 'undo_import', 'import', id);
|
||||
return importSummary(ctx, id);
|
||||
}
|
||||
|
||||
export { MAX_BYTES };
|
||||
89
server/src/import/transfers.ts
Normal file
89
server/src/import/transfers.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import { and, between, eq, inArray, isNull, ne } from 'drizzle-orm';
|
||||
import type { Suggester } from '../categorize/suggest';
|
||||
import type { AppContext } from '../context';
|
||||
import { accounts, transactions, type TransactionRow } from '../db/schema';
|
||||
import { daysBetween, shiftDate } from './parse';
|
||||
|
||||
// Pairs the two halves of a card payment or a move between your own accounts:
|
||||
// money leaving one account and the same amount arriving in another within a
|
||||
// few days. Paired halves are filed as transfers and never count as spending
|
||||
// or income.
|
||||
|
||||
const WINDOW_DAYS = 5;
|
||||
|
||||
export function pairTransfers(ctx: AppContext, suggester: Suggester, ids: number[]): number {
|
||||
if (!ids.length) return 0;
|
||||
const last4ByAccount = new Map(
|
||||
ctx.db
|
||||
.select({ id: accounts.id, last4: accounts.last4 })
|
||||
.from(accounts)
|
||||
.all()
|
||||
.map((a) => [a.id, a.last4]),
|
||||
);
|
||||
const fresh = ctx.db.select().from(transactions).where(inArray(transactions.id, ids)).all();
|
||||
let paired = 0;
|
||||
|
||||
for (const t of fresh) {
|
||||
if (t.transferPeerId || t.amountCents === 0) continue;
|
||||
const current = ctx.db.select().from(transactions).where(eq(transactions.id, t.id)).get();
|
||||
if (!current || current.transferPeerId) continue;
|
||||
|
||||
const candidates = ctx.db
|
||||
.select()
|
||||
.from(transactions)
|
||||
.where(
|
||||
and(
|
||||
ne(transactions.accountId, t.accountId),
|
||||
eq(transactions.amountCents, -t.amountCents),
|
||||
isNull(transactions.transferPeerId),
|
||||
eq(transactions.isSplit, false),
|
||||
between(transactions.date, shiftDate(t.date, -WINDOW_DAYS), shiftDate(t.date, WINDOW_DAYS)),
|
||||
),
|
||||
)
|
||||
.all();
|
||||
|
||||
const tLike = suggester.isTransferLike(t);
|
||||
const score = (c: TransactionRow) => {
|
||||
const cLike = suggester.isTransferLike(c);
|
||||
if (!tLike && !cLike) return -1;
|
||||
let s = (tLike ? 2 : 0) + (cLike ? 2 : 0);
|
||||
const cLast4 = last4ByAccount.get(c.accountId);
|
||||
const tLast4 = last4ByAccount.get(t.accountId);
|
||||
if (cLast4 && t.description.includes(cLast4)) s += 3;
|
||||
if (tLast4 && c.description.includes(tLast4)) s += 3;
|
||||
return s * 10 - Math.abs(daysBetween(t.date, c.date));
|
||||
};
|
||||
const best = candidates
|
||||
.map((c) => ({ c, s: score(c) }))
|
||||
.filter((x) => x.s >= 0)
|
||||
.sort((a, b) => b.s - a.s)[0];
|
||||
if (!best) continue;
|
||||
|
||||
const categoryId = suggester.transferCategoryFor(t.accountId, t.description, best.c.accountId);
|
||||
const now = new Date().toISOString();
|
||||
for (const [self, peer] of [
|
||||
[t.id, best.c.id],
|
||||
[best.c.id, t.id],
|
||||
] as const) {
|
||||
ctx.db
|
||||
.update(transactions)
|
||||
.set({ transferPeerId: peer, categoryId, needsReview: false, suggestedBy: 'transfer', isSplit: false, updatedAt: now })
|
||||
.where(eq(transactions.id, self))
|
||||
.run();
|
||||
}
|
||||
paired++;
|
||||
}
|
||||
return paired;
|
||||
}
|
||||
|
||||
/** Undo a pairing, putting both halves back in Review. */
|
||||
export function unpair(ctx: AppContext, transactionId: number): void {
|
||||
const t = ctx.db.select().from(transactions).where(eq(transactions.id, transactionId)).get();
|
||||
if (!t?.transferPeerId) return;
|
||||
const now = new Date().toISOString();
|
||||
ctx.db
|
||||
.update(transactions)
|
||||
.set({ transferPeerId: null, needsReview: true, updatedAt: now })
|
||||
.where(inArray(transactions.id, [t.id, t.transferPeerId]))
|
||||
.run();
|
||||
}
|
||||
Reference in New Issue
Block a user