Milestone 3: launch (20261008.05)

Backup and restore: restore from Settings (any backup in the folder, or an
uploaded file) or from the command line with scripts/restore.sh. Each restore
checks the file first, saves a copy of the current data so it can be undone,
and keeps the person restoring signed in. Backups are now single
self-contained files. Drilled end to end: last night's backup brings
everything back exactly, including onto an empty new machine.

Speed: a phone's first load is 147 KB compressed (target 150). The ideas box,
account menu and sign-in screens load on demand, a small built-in toast
replaces sonner, the Inter font is trimmed to the characters used, and files
and API answers are served compressed. Every screen's data comes back in
under 100 ms with 50,000 transactions (server/bench).

Accessibility: no WCAG 2.2 AA issues on any screen in either theme. Darker
status text colors, 44 px touch targets on phones, reduced motion honoured,
and focus returns after dialogs close.

Also: an in-app guide ("How Spendlight works"), OFX/QFX/QBO import, and tags
on individual parts of a split purchase.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-08 12:23:45 -04:00
parent 4ecd64132b
commit a68c343647
82 changed files with 3822 additions and 265 deletions

View File

@@ -0,0 +1,79 @@
// Speed check at scale: years of heavy use (about 50,000 made-up
// transactions) and the time each screen's data takes. Run with
// scripts/dev.sh npx vitest run --root server --config server/bench/vitest.config.ts
import { it } from 'vitest';
import { makeApp, setUpAdmin } from '../test/helpers';
it('answers every screen quickly with years of data', async () => {
const t = await makeApp();
const admin = await setUpAdmin(t);
const { sqlite } = t.ctx;
const cats = (sqlite.prepare("SELECT id FROM category WHERE kind = 'expense'").all() as { id: number }[]).map((c) => c.id);
const income = (sqlite.prepare("SELECT id FROM category WHERE kind = 'income'").get() as { id: number }).id;
const now = new Date().toISOString();
const accounts = [1, 2, 3, 4, 5].map(
(i) => (sqlite.prepare("INSERT INTO account (name, kind, color, created_at, updated_at, balance_cents, balance_as_of) VALUES (?, ?, 'indigo', ?, ?, 100000, '2022-01-01') RETURNING id").get(`Account ${i}`, i === 1 ? 'checking' : 'credit_card', now, now) as { id: number }).id,
);
const merchants = Array.from({ length: 400 }, (_, i) => `Merchant ${i}`);
let seed = 7;
const rnd = () => ((seed = (seed * 16807) % 2147483647) / 2147483647);
const insert = sqlite.prepare(
'INSERT INTO txn (account_id, date, amount_cents, description, merchant, category_id, needs_review, dedupe_key, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?)',
);
const start = Date.UTC(2022, 0, 1);
const days = Math.floor((Date.now() - start) / 86_400_000);
sqlite.transaction(() => {
for (let i = 0; i < 50_000; i++) {
const date = new Date(start + Math.floor(rnd() * days) * 86_400_000).toISOString().slice(0, 10);
const m = merchants[Math.floor(rnd() * merchants.length)]!;
const isIncome = rnd() < 0.03;
insert.run(accounts[Math.floor(rnd() * 5)], date, isIncome ? 250000 : -Math.round(500 + rnd() * 15000), m, m, isIncome ? income : cats[Math.floor(rnd() * cats.length)], `k${i}`, now, now);
}
// Monthly subscriptions, so the scan and alerts have real work.
for (let s = 0; s < 25; s++) {
for (let d = new Date(2024, 0, 3 + s); d < new Date(); d.setMonth(d.getMonth() + 1)) {
insert.run(accounts[2], d.toISOString().slice(0, 10), -(999 + s * 100), `SERVICE ${s}`, `Service ${s}`, cats[0], `s${s}-${d.getTime()}`, now, now);
}
}
})();
const year = new Date().getFullYear();
await admin.post('/api/v1/budgets/fill', { month: `${year}-${String(new Date().getMonth() + 1).padStart(2, '0')}`, source: 'average3' });
const time = async (label: string, run: () => Promise<{ status: number }>) => {
const times: number[] = [];
for (let i = 0; i < 5; i++) {
const t0 = performance.now();
const res = await run();
times.push(performance.now() - t0);
if (res.status !== 200) throw new Error(`${label}: ${res.status}`);
}
times.sort((a, b) => a - b);
console.log(`${label.padEnd(34)} first ${times[0]!.toFixed(0).padStart(5)} ms median ${times[2]!.toFixed(0).padStart(5)} ms worst ${times[4]!.toFixed(0).padStart(5)} ms`);
};
const t0 = performance.now();
await admin.post('/api/v1/subscriptions/scan');
console.log(`subscription scan (first, 50k rows) ${(performance.now() - t0).toFixed(0)} ms`);
await time('Home: overview', () => admin.get('/api/v1/reports/overview'));
await time('Home: alerts (data changed)', async () => {
sqlite.prepare("UPDATE txn SET updated_at = ? WHERE id = 1").run(new Date().toISOString() + Math.random());
return admin.get('/api/v1/alerts?limit=3');
});
await time('Home: alerts (unchanged)', () => admin.get('/api/v1/alerts?limit=3'));
await time('Budget: year of months', () => admin.get(`/api/v1/reports/months?from=${year}-01&to=${year}-12`));
await time('Budget: budgets', () => admin.get(`/api/v1/budgets?year=${year}`));
await time('Insights: merchants', () => admin.get(`/api/v1/reports/merchants?from=${year}-01-01&to=${year}-12-31&limit=100`));
await time('Insights: balances', () => admin.get('/api/v1/reports/balances?months=12'));
await time('Insights: tags', () => admin.get('/api/v1/reports/tags'));
await time('Activity: first 100', () => admin.get('/api/v1/transactions?limit=100'));
await time('Activity: search', () => admin.get('/api/v1/transactions?q=Merchant%2012&limit=100'));
await time('Review', () => admin.get('/api/v1/review'));
await time('Subscriptions', () => admin.get('/api/v1/subscriptions'));
await time('Accounts', () => admin.get('/api/v1/accounts'));
const e0 = performance.now();
const res = await t.app.request('http://localhost/api/v1/export/everything.xlsx', { headers: { Cookie: `sl_session=${admin.cookie('sl_session')}` } });
const size = (await res.arrayBuffer()).byteLength;
console.log(`export everything (Excel) ${(performance.now() - e0).toFixed(0)} ms, ${(size / 1024 / 1024).toFixed(1)} MB`);
console.log(`memory: rss ${(process.memoryUsage().rss / 1048576).toFixed(0)} MB, heap ${(process.memoryUsage().heapUsed / 1048576).toFixed(0)} MB`);
t.cleanup();
}, 600_000);

View File

@@ -0,0 +1,3 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({ test: { include: ['bench/**/*.bench.ts'], testTimeout: 600_000 } });

View File

@@ -0,0 +1,9 @@
CREATE TABLE `split_tag` (
`split_id` integer NOT NULL,
`tag_id` integer NOT NULL,
FOREIGN KEY (`split_id`) REFERENCES `split`(`id`) ON UPDATE no action ON DELETE cascade,
FOREIGN KEY (`tag_id`) REFERENCES `tag`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE UNIQUE INDEX `split_tag_pk` ON `split_tag` (`split_id`,`tag_id`);--> statement-breakpoint
CREATE INDEX `split_tag_tag_idx` ON `split_tag` (`tag_id`);

File diff suppressed because it is too large Load Diff

View File

@@ -22,6 +22,13 @@
"when": 1791417594406,
"tag": "0002_full_picture",
"breakpoints": true
},
{
"idx": 3,
"version": "6",
"when": 1791473201347,
"tag": "0003_split_tags",
"breakpoints": true
}
]
}

View File

@@ -1,6 +1,7 @@
import { OpenAPIHono } from '@hono/zod-openapi';
import type { MiddlewareHandler } from 'hono';
import { HTTPException } from 'hono/http-exception';
import { compress } from 'hono/compress';
import { secureHeaders } from 'hono/secure-headers';
import { sessionMiddleware } from './auth/guards';
import type { AppContext } from './context';
@@ -82,6 +83,8 @@ export function createApp(ctx: AppContext) {
await next();
c.header('Cache-Control', 'no-store');
});
// JSON answers can be large (a year of reports, 500 transactions); send them gzipped.
app.use('/api/*', compress());
app.use('/api/*', sessionMiddleware(ctx));
app.use('/api/*', sameOriginWrites(ctx));

View File

@@ -1,4 +1,5 @@
import type { BackupInfo, BackupKind } from '@spendlight/shared';
import Database from 'better-sqlite3';
import { existsSync, mkdirSync, readdirSync, renameSync, statSync, unlinkSync } from 'node:fs';
import { join } from 'node:path';
import type { Sqlite } from '../db/client';
@@ -48,6 +49,11 @@ export async function createBackup(
}
const partial = join(dir, `.${name}.partial`);
await sqlite.backup(partial);
// A backup is one self-contained file: no write-ahead log, so reading it
// later never leaves -wal or -shm files beside it in the backup folder.
const copy = new Database(partial);
copy.pragma('journal_mode = DELETE');
copy.close();
renameSync(partial, join(dir, name));
return { name, kind, size: statSync(join(dir, name)).size, createdAt: when.toISOString() };
}
@@ -122,7 +128,8 @@ export function backupPath(dir: string, name: string): string | null {
/** Run a nightly backup at the configured local time, plus a catch-up if the last one is overdue. */
export function scheduleNightlyBackups(options: {
sqlite: Sqlite;
/** The open database; a function, because a restore swaps it for another. */
sqlite: () => Sqlite;
dir: string;
at: { hour: number; minute: number };
policy: RetentionPolicy;
@@ -133,7 +140,7 @@ export function scheduleNightlyBackups(options: {
const run = async () => {
try {
const backup = await createBackup(sqlite, dir, 'nightly');
const backup = await createBackup(sqlite(), dir, 'nightly');
const removed = pruneBackups(dir, policy);
log.info(`Nightly backup ${backup.name} (${removed.length} old copies removed)`);
} catch (error) {

View File

@@ -0,0 +1,131 @@
import Database from 'better-sqlite3';
import { randomUUID } from 'node:crypto';
import { closeSync, copyFileSync, existsSync, openSync, readFileSync, readSync, renameSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BackupInfo } from '@spendlight/shared';
import { APP_MARKER, migrateDatabase, migrationStatus, openDatabase, type DB, type Sqlite } from '../db/client';
import type { SessionRow } from '../db/schema';
import { createBackup, pruneBackups, type RetentionPolicy } from './backup';
// Restoring a backup replaces the whole database. Before anything changes,
// the file is checked (a SQLite database, intact, made by Spendlight, and not
// by a newer version), and a "before restore" copy of the current database is
// saved, so a restore can itself be undone.
export interface BackupCheck {
/** Database upgrades the copy has had; newer ones run after restoring. */
applied: number;
counts: { transactions: number; accounts: number; people: number; budgets: number; subscriptions: number };
newestTransaction: string | null;
}
export class RestoreError extends Error {
constructor(
public code: string,
message: string,
) {
super(message);
}
}
const SQLITE_HEADER = 'SQLite format 3\u0000';
/** Check a backup by reading a throwaway copy, so the original is never touched. */
export function checkBackup(file: string, migrationsDir: string, scratchDir = tmpdir()): BackupCheck {
if (!existsSync(file)) throw new RestoreError('not_found', 'That backup wasn’t found.');
const copy = join(scratchDir, `.check-${randomUUID()}.db`);
copyFileSync(file, copy);
try {
return inspect(copy, migrationsDir);
} finally {
for (const extra of ['', '-wal', '-shm', '-journal']) rmSync(copy + extra, { force: true });
}
}
function inspect(file: string, migrationsDir: string): BackupCheck {
const head = Buffer.alloc(16);
const fd = openSync(file, 'r');
try {
readSync(fd, head, 0, 16, 0);
} finally {
closeSync(fd);
}
if (head.toString('latin1') !== SQLITE_HEADER) throw new RestoreError('not_sqlite', 'That file isn’t a Spendlight backup.');
let sqlite: Sqlite | null = null;
try {
sqlite = new Database(file, { fileMustExist: true });
const ok = sqlite.pragma('quick_check', { simple: true });
if (ok !== 'ok') throw new RestoreError('damaged', 'That backup is damaged and can’t be restored.');
const table = (name: string) => !!sqlite!.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
const marker = table('setting') ? (sqlite.prepare('SELECT value FROM setting WHERE key = ?').get(APP_MARKER.key) as { value: string } | undefined) : undefined;
if (marker?.value !== APP_MARKER.value) throw new RestoreError('not_spendlight', 'That file isn’t a Spendlight backup.');
const known = (JSON.parse(readFileSync(join(migrationsDir, 'meta', '_journal.json'), 'utf8')) as { entries: unknown[] }).entries.length;
const { applied } = migrationStatus(sqlite, migrationsDir);
if (applied > known) throw new RestoreError('too_new', 'That backup was made by a newer version of Spendlight. Update Spendlight first, then restore it.');
const count = (name: string) => (table(name) ? (sqlite!.prepare(`SELECT count(*) AS n FROM ${name}`).get() as { n: number }).n : 0);
const people = table('user') ? (sqlite.prepare('SELECT count(*) AS n FROM user WHERE deleted_at IS NULL').get() as { n: number }).n : 0;
if (!people) throw new RestoreError('empty', 'That backup has nobody who can sign in, so it can’t be restored.');
return {
applied,
counts: { transactions: count('txn'), accounts: count('account'), people, budgets: count('budget'), subscriptions: count('subscription') },
newestTransaction: table('txn') ? ((sqlite.prepare('SELECT max(date) AS d FROM txn').get() as { d: string | null }).d ?? null) : null,
};
} catch (error) {
if (error instanceof RestoreError) throw error;
throw new RestoreError('unreadable', 'That backup couldn’t be read.');
} finally {
sqlite?.close();
}
}
export interface RestoreOptions {
dataDir: string;
backupDir: string;
migrationsDir: string;
policy: RetentionPolicy;
/** The open database, which gets closed and replaced. Null when the app isn't running. */
current: Sqlite | null;
/** The session of the person restoring, carried into the restored copy. */
keepSession?: { userId: number; username: string; session: SessionRow };
}
/** Replace the database with a backup. Returns the newly opened database. */
export async function restoreDatabase(source: string, options: RestoreOptions): Promise<{ sqlite: Sqlite; db: DB; safety: BackupInfo | null; check: BackupCheck }> {
const check = checkBackup(source, options.migrationsDir, options.dataDir);
const file = join(options.dataDir, 'spendlight.db');
// 1. A safety copy of how things are now.
let safety: BackupInfo | null = null;
const current = options.current ?? (existsSync(file) ? openDatabase(file).sqlite : null);
if (current) {
safety = await createBackup(current, options.backupDir, 'pre-restore');
pruneBackups(options.backupDir, options.policy);
}
// 2. Stage the copy next to the database, so the swap is a single rename.
const staged = join(options.dataDir, '.restore.db');
rmSync(staged, { force: true });
copyFileSync(source, staged);
// 3. Swap: close, replace, and drop the old write-ahead log with it.
current?.close();
for (const extra of ['-wal', '-shm']) rmSync(file + extra, { force: true });
renameSync(staged, file);
// 4. Open it and bring it up to date with this version.
const { sqlite, db } = openDatabase(file);
await migrateDatabase(db, sqlite, options.migrationsDir);
// Whoever restored stays signed in, if they're in the restored copy.
const keep = options.keepSession;
if (keep) {
const same = sqlite.prepare('SELECT 1 FROM user WHERE id = ? AND username = ? AND deleted_at IS NULL').get(keep.userId, keep.username);
if (same) {
sqlite
.prepare('INSERT OR IGNORE INTO session (id, user_id, kind, device_label, user_agent, ip, created_at, last_seen_at, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)')
.run(keep.session.id, keep.session.userId, keep.session.kind, keep.session.deviceLabel, keep.session.userAgent, keep.session.ip, keep.session.createdAt, keep.session.lastSeenAt, keep.session.expiresAt);
}
}
return { sqlite, db, safety, check };
}

View File

@@ -5,6 +5,7 @@ import { join } from 'node:path';
import { hashPassword, temporaryPassword } from './auth/password';
import { revokeUserSessions } from './auth/sessions';
import { createBackup, pruneBackups } from './backups/backup';
import { checkBackup, RestoreError, restoreDatabase } from './backups/restore';
import { loadConfig } from './config';
import type { AppContext } from './context';
import { openDatabase } from './db/client';
@@ -12,12 +13,14 @@ import { users } from './db/schema';
import { ideasToMarkdown } from './ideas/markdown';
import { listIdeas } from './ideas/queries';
import { silentLogger } from './log';
import { MIGRATIONS_DIR } from './paths';
import { findActiveUser } from './users';
// Admin commands, run inside the container:
// spendlight ideas [--status=new|planned|done|declined]
// spendlight reset-password <username>
// spendlight backup
// spendlight restore <file>
// spendlight version
const HELP = `Spendlight admin commands
@@ -25,6 +28,7 @@ const HELP = `Spendlight admin commands
spendlight ideas [--status=new] Print ideas as Markdown (all statuses by default)
spendlight reset-password <user> Give someone a temporary password
spendlight backup Back up the database now
spendlight restore <file> Replace the database with a backup (stop the app first)
spendlight version Show the running version
`;
@@ -81,6 +85,38 @@ switch (command) {
console.log(`Backed up to ${join(config.backupDir, backup.name)}`);
break;
}
case 'restore': {
const source = args[0];
if (!source) {
console.error('Usage: spendlight restore <backup file> (for example /backups/spendlight-nightly-20261007-023000.db)');
process.exit(1);
}
// Restoring underneath a running app would lose whatever it writes next.
const running = await fetch(`http://127.0.0.1:${config.port}/api/v1/health`)
.then((r) => r.ok)
.catch(() => false);
if (running) {
console.error('Spendlight is running. Restore from Settings → Backups instead, or stop the app first.');
process.exit(1);
}
try {
const check = checkBackup(source, MIGRATIONS_DIR);
console.log(`Backup holds ${check.counts.transactions} transactions, ${check.counts.accounts} accounts and ${check.counts.people} people.`);
const { sqlite, safety } = await restoreDatabase(source, {
dataDir: config.dataDir,
backupDir: config.backupDir,
migrationsDir: MIGRATIONS_DIR,
policy: { keepDaily: config.keepDaily, keepMonthly: config.keepMonthly },
current: null,
});
sqlite.close();
console.log(`Restored. The previous database is saved as ${safety ? join(config.backupDir, safety.name) : '(there was none)'}.`);
} catch (error) {
console.error(error instanceof RestoreError ? error.message : `Restore failed: ${(error as Error).message}`);
process.exit(1);
}
break;
}
case 'version':
console.log(config.version);
break;

View File

@@ -271,6 +271,20 @@ export const transactionTags = sqliteTable(
(t) => [uniqueIndex('txn_tag_pk').on(t.transactionId, t.tagId), index('txn_tag_tag_idx').on(t.tagId)],
);
/** Tags on one part of a split purchase, such as "Reimbursable" on the work lunch only. */
export const splitTags = sqliteTable(
'split_tag',
{
splitId: integer('split_id')
.notNull()
.references(() => splits.id, { onDelete: 'cascade' }),
tagId: integer('tag_id')
.notNull()
.references(() => tags.id, { onDelete: 'cascade' }),
},
(t) => [uniqueIndex('split_tag_pk').on(t.splitId, t.tagId), index('split_tag_tag_idx').on(t.tagId)],
);
/** "Always file this merchant this way." Created when someone says so, or by hand. */
export const rules = sqliteTable(
'rule',

99
server/src/import/ofx.ts Normal file
View File

@@ -0,0 +1,99 @@
import type { AccountKind } from '@spendlight/shared';
// OFX, QFX and QBO files ("Quicken", "Money" or "Web Connect" downloads). Both
// flavors are read: the older SGML one, where tags aren't closed, and XML.
// Amounts are already signed from the account holder's side: money out is
// negative, for bank accounts and cards alike.
export interface OfxTransaction {
date: string;
postedDate: string | null;
amountCents: number;
name: string;
memo: string;
type: string;
id: string | null;
}
export interface OfxStatement {
kind: AccountKind;
institution: string | null;
accountLast4: string | null;
transactions: OfxTransaction[];
ledger: { cents: number; date: string } | null;
}
export const looksLikeOfx = (text: string) => /OFXHEADER\s*:|<OFX>/i.test(text.slice(0, 4000));
const decode = (s: string) =>
s
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&nbsp;/g, ' ')
.replace(/&amp;/g, '&')
.trim();
/** The value of a leaf tag: <TRNAMT>-12.34 (SGML) or <TRNAMT>-12.34</TRNAMT> (XML). */
function leaf(block: string, tag: string): string | null {
const m = new RegExp(`<${tag}>([^<\\r\\n]*)`, 'i').exec(block);
return m ? decode(m[1]!) : null;
}
/** 20261007, 20261007120000, 20261007120000.000[-5:EST] → 2026-10-07 */
function ofxDate(value: string | null): string | null {
const m = value ? /^(\d{4})(\d{2})(\d{2})/.exec(value) : null;
if (!m) return null;
const [, y, mo, d] = m;
const date = new Date(Date.UTC(Number(y), Number(mo) - 1, Number(d)));
return date.getUTCMonth() === Number(mo) - 1 ? `${y}-${mo}-${d}` : null;
}
function cents(value: string | null): number | null {
if (!value) return null;
// Some banks write decimal commas: "-12,34".
const clean = value.replace(/\s/g, '').replace(/,(\d{1,2})$/, '.$1').replace(/,/g, '');
if (!/^[+-]?\d*(\.\d+)?$/.test(clean) || clean === '' || clean === '-' || clean === '+') return null;
return Math.round(Number(clean) * 100);
}
export function parseOfx(text: string): OfxStatement {
const card = /<CCSTMTRS>|<CREDITCARDMSGSRSV1>/i.test(text);
const acctType = leaf(text, 'ACCTTYPE')?.toUpperCase();
const kind: AccountKind = card ? 'credit_card' : acctType === 'SAVINGS' || acctType === 'MONEYMRKT' ? 'savings' : 'checking';
const acctId = leaf(text, 'ACCTID');
const accountLast4 = acctId ? (/(\d{4})\D*$/.exec(acctId)?.[1] ?? null) : null;
const institution = leaf(text, 'ORG');
const transactions: OfxTransaction[] = [];
const blocks = text.split(/<STMTTRN>/i).slice(1);
for (const raw of blocks) {
const block = raw.split(/<\/STMTTRN>|<\/BANKTRANLIST>/i)[0]!;
const posted = ofxDate(leaf(block, 'DTPOSTED'));
const user = ofxDate(leaf(block, 'DTUSER'));
const amount = cents(leaf(block, 'TRNAMT'));
const date = user ?? posted;
if (!date || amount === null) continue;
transactions.push({
date,
postedDate: user && posted && posted !== user ? posted : null,
amountCents: amount,
name: leaf(block, 'NAME') ?? leaf(block, 'PAYEE') ?? '',
memo: leaf(block, 'MEMO') ?? '',
type: (leaf(block, 'TRNTYPE') ?? '').toUpperCase(),
id: leaf(block, 'FITID'),
});
}
const ledgerBlock = /<LEDGERBAL>([\s\S]*?)(<\/LEDGERBAL>|<AVAILBAL>|<\/STMTRS>|<\/CCSTMTRS>|$)/i.exec(text)?.[1] ?? '';
const ledgerCents = cents(leaf(ledgerBlock, 'BALAMT'));
const ledgerDate = ofxDate(leaf(ledgerBlock, 'DTASOF'));
return {
kind,
institution: institution && !/^\d+$/.test(institution) ? institution : null,
accountLast4,
transactions,
ledger: ledgerCents !== null && ledgerDate ? { cents: ledgerCents, date: ledgerDate } : null,
};
}

View File

@@ -13,6 +13,7 @@ import { scanSubscriptions } from '../subscriptions/engine';
import { detectBuiltIn, findHeaderRow, guessMap, headerSignature, type ResolvedLayout } from './layouts';
import { cleanMerchant, dedupeKey } from './merchant';
import { daysBetween, parseCents, parseDate, readTable, shiftDate } from './parse';
import { looksLikeOfx, parseOfx } from './ofx';
import { pairTransfers } from './transfers';
// The import pipeline: stage a file, preview what it would add, then commit.
@@ -53,6 +54,11 @@ export class StagingStore {
return staged;
}
/** Forget every staged file, as after a restore replaces the accounts they were meant for. */
clear(): void {
this.files.clear();
}
delete(id: string): void {
this.files.delete(id);
}
@@ -93,7 +99,52 @@ function fileNameLast4(fileName: string): string | null {
/** The file name with its digits generalized, so "Chase1234_Activity_20261007.csv" matches next month's file. */
const fileNamePattern = (fileName: string) => fileName.replace(/\d{5,}/g, '#').replace(/\(\d+\)/g, '');
/** OFX, QFX and QBO downloads carry their own structure, so they need no layout or mapping. */
function parseOfxFile(staged: Staged): Parsed {
const statement = parseOfx(staged.text);
const rows: Row[] = statement.transactions.map((t) => {
const description = [t.name, t.memo && !t.name.includes(t.memo) ? t.memo : ''].filter(Boolean).join(' ').replace(/\s+/g, ' ').trim() || 'Transaction';
const { merchant, mentionsLast4 } = cleanMerchant(description);
return {
date: t.date,
postedDate: t.postedDate,
description,
merchant,
mentionsLast4,
amountCents: t.amountCents,
bankCategory: null,
bankType: t.type === 'XFER' ? 'Payment' : null,
balanceCents: null,
};
});
// The statement's ending balance, for checking and savings; card files differ on its sign, so cards are left to the person.
if (statement.ledger && statement.kind !== 'credit_card' && rows.length) {
const last = rows.reduce((a, r) => (r.date > a.date ? r : a));
if (last.date <= statement.ledger.date) last.balanceCents = statement.ledger.cents;
}
const institution = statement.institution;
return {
layout: {
key: 'ofx',
name: institution ? `${institution} (OFX/QFX)` : 'OFX/QFX file',
institution,
kind: statement.kind,
headerRow: 0,
map: { date: 0, description: 0, amount: 0, positiveIs: 'in', dateOrder: 'YMD' },
},
rows,
skipped: 0,
header: null,
sample: [],
suggestedMap: null,
signature: 'ofx',
last4Hint: statement.accountLast4 ?? fileNameLast4(staged.fileName),
openingBalance: null,
};
}
function parseFile(ctx: AppContext, staged: Staged, mapping?: ImportColumnMap): Parsed {
if (looksLikeOfx(staged.text)) return parseOfxFile(staged);
const table = readTable(staged.text);
let layout = mapping ? null : detectBuiltIn(table);
const headerRow = layout?.headerRow ?? findHeaderRow(table);

View File

@@ -9,7 +9,7 @@ const ctx = await createContext(config);
const app = createApp(ctx);
const backups = scheduleNightlyBackups({
sqlite: ctx.sqlite,
sqlite: () => ctx.sqlite,
dir: config.backupDir,
at: config.backupTime,
policy: { keepDaily: config.keepDaily, keepMonthly: config.keepMonthly },

View File

@@ -1,13 +1,32 @@
import { createRoute, OpenAPIHono, z } from '@hono/zod-openapi';
import { createReadStream, existsSync, statSync } from 'node:fs';
import { randomUUID } from 'node:crypto';
import { createReadStream, existsSync, rmSync, statSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { Readable } from 'node:stream';
import { bodyLimit } from 'hono/body-limit';
import { logChange } from '../audit';
import { requireAdmin } from '../auth/guards';
import { backupPath, createBackup, listBackups, pruneBackups } from '../backups/backup';
import { checkBackup, RestoreError, restoreDatabase } from '../backups/restore';
import type { AppContext } from '../context';
import type { SessionRow, UserRow } from '../db/schema';
import { fail, type Env } from '../http';
import { refreshAlerts } from '../insights/engine';
import { MIGRATIONS_DIR } from '../paths';
import { BackupSchema, jsonResponse, validationHook } from '../schemas';
const MAX_RESTORE_BYTES = 200 * 1024 * 1024;
const CheckSchema = z
.object({
applied: z.number().int(),
counts: z.object({ transactions: z.number().int(), accounts: z.number().int(), people: z.number().int(), budgets: z.number().int(), subscriptions: z.number().int() }),
newestTransaction: z.string().nullable(),
})
.openapi('BackupCheck');
const RestoredSchema = z.object({ check: CheckSchema, safetyCopy: z.string().nullable() }).openapi('Restored');
const NameParam = z.object({ name: z.string().openapi({ param: { name: 'name', in: 'path' } }) });
const pad = (n: number) => String(n).padStart(2, '0');
export function backupRoutes(ctx: AppContext) {
@@ -58,6 +77,100 @@ export function backupRoutes(ctx: AppContext) {
},
);
const guard = <T,>(run: () => T): T => {
try {
return run();
} catch (error) {
if (error instanceof RestoreError) fail(400, error.code, error.message);
throw error;
}
};
/** Swap the app over to a backup; everyone keeps working against the restored data. */
const restore = async (source: string, admin: UserRow, session: SessionRow | null, label: string) => {
const adminId = admin.id;
let result;
try {
result = await restoreDatabase(source, {
dataDir: ctx.config.dataDir,
backupDir,
migrationsDir: MIGRATIONS_DIR,
policy: { keepDaily, keepMonthly },
current: ctx.sqlite,
keepSession: session ? { userId: admin.id, username: admin.username, session } : undefined,
});
} catch (error) {
if (error instanceof RestoreError) fail(400, error.code, error.message);
throw error;
}
ctx.sqlite = result.sqlite;
ctx.db = result.db;
ctx.staging.clear();
try {
refreshAlerts(ctx);
} catch (error) {
ctx.log.error('Could not refresh alerts after restoring', error);
}
ctx.log.info(`Restored ${label}; the previous database is saved as ${result.safety?.name ?? '(none)'}`);
// The person who restored may not exist in the copy; the log entry goes in only if they do.
try {
logChange(ctx, adminId, 'restore', 'backup', label, { safetyCopy: result.safety?.name ?? null });
} catch {
logChange(ctx, null, 'restore', 'backup', label, { safetyCopy: result.safety?.name ?? null });
}
return { check: result.check, safetyCopy: result.safety?.name ?? null };
};
r.openapi(
createRoute({
method: 'get',
path: '/backups/{name}/check',
tags: ['Backups'],
summary: 'What a backup holds, checked before restoring it',
request: { params: NameParam },
responses: jsonResponse(CheckSchema, 'Check'),
}),
(c) => {
requireAdmin(c);
const path = backupPath(backupDir, c.req.valid('param').name);
if (!path || !existsSync(path)) fail(404, 'not_found', 'That backup wasn’t found.');
return c.json(guard(() => checkBackup(path, MIGRATIONS_DIR, ctx.config.dataDir)), 200);
},
);
r.openapi(
createRoute({
method: 'post',
path: '/backups/{name}/restore',
tags: ['Backups'],
summary: 'Replace everything with a backup from the backup folder',
description: 'A "before restore" copy of the current data is saved first, so a restore can be undone.',
request: { params: NameParam },
responses: jsonResponse(RestoredSchema, 'Restored'),
}),
async (c) => {
const admin = requireAdmin(c);
const { name } = c.req.valid('param');
const path = backupPath(backupDir, name);
if (!path || !existsSync(path)) fail(404, 'not_found', 'That backup wasn’t found.');
return c.json(await restore(path, admin, c.get('session'), name), 200);
},
);
r.post('/backups/restore-upload', bodyLimit({ maxSize: MAX_RESTORE_BYTES, onError: () => fail(413, 'too_big', 'That file is larger than 200 MB, which is far bigger than any Spendlight backup.') }), async (c) => {
const admin = requireAdmin(c);
const form = await c.req.parseBody();
const file = form.file;
if (!(file instanceof File)) fail(400, 'no_file', 'Please choose a backup file.');
const upload = join(ctx.config.dataDir, `.upload-${randomUUID()}.db`);
writeFileSync(upload, Buffer.from(await file.arrayBuffer()));
try {
return c.json(await restore(upload, admin, c.get('session'), file.name || 'uploaded file'), 200);
} finally {
rmSync(upload, { force: true });
}
});
// A file download, so it sits outside the JSON API description.
r.get('/backups/:name/download', (c) => {
requireAdmin(c);

View File

@@ -4,7 +4,7 @@ import { logChange } from '../audit';
import { requireUser, requireWrite } from '../auth/guards';
import { rememberRule } from '../categorize/suggest';
import type { AppContext } from '../context';
import { budgets, categories, categoryGroups, rules, splits, subscriptions, tags, transactions, transactionTags, type CategoryRow } from '../db/schema';
import { budgets, categories, categoryGroups, rules, splits, splitTags, subscriptions, tags, transactions, transactionTags, type CategoryRow } from '../db/schema';
import { fail, nowIso, type Env } from '../http';
import { CategoryGroupSchema, CategorySchema, IdParam, jsonBody, jsonResponse, PaletteSchema, validationHook } from '../schemas';
import { listCategoryGroups } from '../transactions/dto';
@@ -249,13 +249,14 @@ export function categoryRoutes(ctx: AppContext) {
}),
(c) => {
requireUser(c);
const rows = ctx.db
.select({ id: tags.id, name: tags.name, count: count(transactionTags.transactionId) })
.from(tags)
.leftJoin(transactionTags, eq(transactionTags.tagId, tags.id))
.groupBy(tags.id)
.orderBy(desc(count(transactionTags.transactionId)), tags.name)
.all();
// Uses: transactions tagged, plus purchases with a tagged part.
const rows = ctx.sqlite
.prepare(
`SELECT t.id, t.name, (SELECT count(*) FROM txn_tag tt WHERE tt.tag_id = t.id)
+ (SELECT count(DISTINCT s.transaction_id) FROM split_tag st JOIN split s ON s.id = st.split_id WHERE st.tag_id = t.id) AS count
FROM tag t ORDER BY count DESC, t.name`,
)
.all() as { id: number; name: string; count: number }[];
return c.json(rows, 200);
},
);
@@ -281,6 +282,9 @@ export function categoryRoutes(ctx: AppContext) {
for (const link of ctx.db.select().from(transactionTags).where(eq(transactionTags.tagId, id)).all()) {
ctx.db.insert(transactionTags).values({ transactionId: link.transactionId, tagId: other.id }).onConflictDoNothing().run();
}
for (const link of ctx.db.select().from(splitTags).where(eq(splitTags.tagId, id)).all()) {
ctx.db.insert(splitTags).values({ splitId: link.splitId, tagId: other.id }).onConflictDoNothing().run();
}
ctx.db.delete(tags).where(eq(tags.id, id)).run();
result = { id: other.id, name: other.name };
} else ctx.db.update(tags).set({ name }).where(eq(tags.id, id)).run();

View File

@@ -5,7 +5,7 @@ import { currentBalance } from '../accounts/balance';
import { requireUser } from '../auth/guards';
import { budgetYear, effective } from '../budgets/budgets';
import type { AppContext } from '../context';
import { accounts, budgets, rules, splits, subscriptions, tags, transactions, transactionTags, type TransactionRow } from '../db/schema';
import { accounts, budgets, rules, splits, splitTags, subscriptions, tags, transactions, transactionTags, type TransactionRow } from '../db/schema';
import { today } from '../dates';
import { buildCsv, buildXlsx, type Cell, type Sheet } from '../export/xlsx';
import type { Env } from '../http';
@@ -27,9 +27,19 @@ function transactionSheet(ctx: AppContext, filter: TransactionFilter): Sheet {
const chunks = <T,>(list: T[], size = 900) => Array.from({ length: Math.ceil(list.length / size) }, (_, i) => list.slice(i * size, i * size + size));
const parts = new Map<number, (typeof splits.$inferSelect)[]>();
const tagNames = new Map<number, string[]>();
const partTags = new Map<number, string[]>();
for (const chunk of chunks(ids)) {
for (const s of ctx.db.select().from(splits).where(inArray(splits.transactionId, chunk)).orderBy(asc(splits.id)).all()) {
parts.set(s.transactionId, [...(parts.get(s.transactionId) ?? []), s]);
const partRows = ctx.db.select().from(splits).where(inArray(splits.transactionId, chunk)).orderBy(asc(splits.id)).all();
for (const s of partRows) parts.set(s.transactionId, [...(parts.get(s.transactionId) ?? []), s]);
for (const partChunk of chunks(partRows.map((s) => s.id))) {
for (const pt of ctx.db
.select({ id: splitTags.splitId, name: tags.name })
.from(splitTags)
.innerJoin(tags, eq(tags.id, splitTags.tagId))
.where(inArray(splitTags.splitId, partChunk))
.all()) {
partTags.set(pt.id, [...(partTags.get(pt.id) ?? []), pt.name]);
}
}
for (const t of ctx.db
.select({ id: transactionTags.transactionId, name: tags.name })
@@ -48,7 +58,7 @@ function transactionSheet(ctx: AppContext, filter: TransactionFilter): Sheet {
const out: Cell[][] = [];
for (const t of rows) {
const base = (categoryId: number | null, cents: number, note: string | null, splitOf: number | null): Cell[] => {
const base = (categoryId: number | null, cents: number, note: string | null, splitOf: number | null, extraTags: string[] = []): Cell[] => {
const cat = categoryId ? cats.get(categoryId) : undefined;
return [
{ date: t.date },
@@ -59,7 +69,7 @@ function transactionSheet(ctx: AppContext, filter: TransactionFilter): Sheet {
{ money: cents },
splitOf === null ? '' : { money: splitOf },
t.transferPeerId ? (accountNames.get(peers.get(t.transferPeerId) ?? -1) ?? '') : '',
(tagNames.get(t.id) ?? []).join(', '),
[...new Set([...(tagNames.get(t.id) ?? []), ...extraTags])].join(', '),
note ?? '',
t.subscriptionId ? (subNames.get(t.subscriptionId) ?? '') : '',
t.needsReview ? 'To review' : 'Filed',
@@ -67,7 +77,7 @@ function transactionSheet(ctx: AppContext, filter: TransactionFilter): Sheet {
];
};
const p = parts.get(t.id);
if (t.isSplit && p?.length) for (const part of p) out.push(base(part.categoryId, part.amountCents, part.note ?? t.note, t.amountCents));
if (t.isSplit && p?.length) for (const part of p) out.push(base(part.categoryId, part.amountCents, part.note ?? t.note, t.amountCents, partTags.get(part.id)));
else out.push(base(t.categoryId, t.amountCents, t.note, null));
}
return {

View File

@@ -211,16 +211,20 @@ export function reportRoutes(ctx: AppContext) {
const rows = ctx.sqlite
.prepare(
`SELECT tag, categoryId, sum(net) AS net, count(DISTINCT txnId) AS n FROM (
SELECT tg.name AS tag, t.category_id AS categoryId, t.amount_cents AS net, t.id AS txnId
SELECT tg.name AS tag, 'txn' AS unitKind, t.id AS unit, t.category_id AS categoryId, t.amount_cents AS net, t.id AS txnId
FROM txn_tag tt JOIN tag tg ON tg.id = tt.tag_id JOIN txn t ON t.id = tt.transaction_id
WHERE t.is_split = 0 AND t.transfer_peer_id IS NULL AND t.date >= ? AND t.date <= ?
UNION ALL
SELECT tg.name, s.category_id, s.amount_cents, t.id
UNION
SELECT tg.name, 'part', s.id, s.category_id, s.amount_cents, t.id
FROM txn_tag tt JOIN tag tg ON tg.id = tt.tag_id JOIN txn t ON t.id = tt.transaction_id JOIN split s ON s.transaction_id = t.id
WHERE t.is_split = 1 AND t.date >= ? AND t.date <= ?
UNION
SELECT tg.name, 'part', s.id, s.category_id, s.amount_cents, t.id
FROM split_tag st JOIN tag tg ON tg.id = st.tag_id JOIN split s ON s.id = st.split_id JOIN txn t ON t.id = s.transaction_id
WHERE t.is_split = 1 AND t.date >= ? AND t.date <= ?
) GROUP BY tag, categoryId`,
)
.all(from, to, from, to) as { tag: string; categoryId: number | null; net: number; n: number }[];
.all(from, to, from, to, from, to) as { tag: string; categoryId: number | null; net: number; n: number }[];
const byTag = new Map<string, { tag: string; spent: number; income: number; count: number; byCategory: { categoryId: number | null; cents: number }[] }>();
for (const row of rows) {
const kind = row.categoryId === null ? (row.net < 0 ? 'expense' : 'income') : kinds.get(row.categoryId);

View File

@@ -5,7 +5,7 @@ import { logChange } from '../audit';
import { requireUser, requireWrite } from '../auth/guards';
import { rememberRule, Suggester } from '../categorize/suggest';
import type { AppContext } from '../context';
import { accounts, categories, rules, splits, subscriptions, tags, transactions, transactionTags, type TransactionRow } from '../db/schema';
import { accounts, categories, rules, splits, splitTags, subscriptions, transactions, transactionTags, type TransactionRow } from '../db/schema';
import { fail, nowIso, type Env } from '../http';
import { dedupeKey } from '../import/merchant';
import { unpair } from '../import/transfers';
@@ -19,6 +19,7 @@ import {
TransactionSchema,
validationHook,
} from '../schemas';
import { tagIdsFor } from '../tags';
import { newestFirst, toTransaction, toTransactions } from '../transactions/dto';
import { TransactionQuery, transactionWhere } from '../transactions/filter';
@@ -43,19 +44,7 @@ export function transactionRoutes(ctx: AppContext) {
function setTags(transactionId: number, names: string[]) {
ctx.db.delete(transactionTags).where(eq(transactionTags.transactionId, transactionId)).run();
// The first spelling of a tag wins: "Back to school" and "back to school" are one tag.
const byKey = new Map<string, string>();
for (const n of names) if (n.trim() && !byKey.has(n.trim().toLowerCase())) byKey.set(n.trim().toLowerCase(), n.trim());
const unique = [...byKey.values()];
for (const name of unique) {
const existing = ctx.db
.select()
.from(tags)
.where(sql`lower(${tags.name}) = ${name.toLowerCase()}`)
.get();
const tagId = existing?.id ?? ctx.db.insert(tags).values({ name }).returning().get().id;
ctx.db.insert(transactionTags).values({ transactionId, tagId }).onConflictDoNothing().run();
}
for (const tagId of tagIdsFor(ctx, names)) ctx.db.insert(transactionTags).values({ transactionId, tagId }).onConflictDoNothing().run();
}
/** Other transactions from the same merchant still waiting in Review, for "also file these?". */
@@ -214,7 +203,7 @@ export function transactionRoutes(ctx: AppContext) {
...jsonBody(
z.object({
parts: z
.array(z.object({ categoryId: z.number().int(), amountCents: Cents, note: z.string().trim().max(200).nullable().optional() }))
.array(z.object({ categoryId: z.number().int(), amountCents: Cents, note: z.string().trim().max(200).nullable().optional(), tags: TagList.optional() }))
.min(2, 'A split needs at least two parts.')
.max(20),
}),
@@ -236,7 +225,10 @@ export function transactionRoutes(ctx: AppContext) {
ctx.sqlite.transaction(() => {
if (t.transferPeerId) unpair(ctx, t.id);
ctx.db.delete(splits).where(eq(splits.transactionId, id)).run();
for (const p of parts) ctx.db.insert(splits).values({ transactionId: id, categoryId: p.categoryId, amountCents: p.amountCents, note: p.note ?? null }).run();
for (const p of parts) {
const part = ctx.db.insert(splits).values({ transactionId: id, categoryId: p.categoryId, amountCents: p.amountCents, note: p.note ?? null }).returning().get();
for (const tagId of tagIdsFor(ctx, p.tags ?? [])) ctx.db.insert(splitTags).values({ splitId: part.id, tagId }).onConflictDoNothing().run();
}
ctx.db
.update(transactions)
.set({ isSplit: true, categoryId: null, needsReview: false, suggestedBy: null, updatedAt: nowIso() })

View File

@@ -167,7 +167,7 @@ export const CategoryGroupSchema = z
.openapi('CategoryGroup');
export const SplitSchema = z
.object({ id: z.number().int(), categoryId: z.number().int(), amountCents: z.number().int(), note: z.string().nullable() })
.object({ id: z.number().int(), categoryId: z.number().int(), amountCents: z.number().int(), note: z.string().nullable(), tags: z.array(z.string()) })
.openapi('Split');
export const TransactionSchema = z

View File

@@ -5,7 +5,8 @@ import { Readable } from 'node:stream';
// Serves the built web app. Hashed files under /assets are cached for a year;
// everything else revalidates so a new version shows up straight away. Any
// path that isn't a file gets index.html, so the app can route it.
// path that isn't a file gets index.html, so the app can route it. Text files
// go out pre-compressed (brotli, else gzip) when the browser accepts it.
const TYPES: Record<string, string> = {
'.html': 'text/html; charset=utf-8',
@@ -31,13 +32,25 @@ function cacheControlFor(urlPath: string): string {
return 'public, max-age=86400';
}
function fileResponse(path: string, urlPath: string, head: boolean): Response {
const headers = {
function fileResponse(path: string, urlPath: string, head: boolean, acceptEncoding = ''): Response {
const headers: Record<string, string> = {
'Content-Type': TYPES[extname(path)] ?? 'application/octet-stream',
'Content-Length': String(statSync(path).size),
'Cache-Control': cacheControlFor(urlPath),
};
const body = head ? null : (Readable.toWeb(createReadStream(path)) as ReadableStream);
let file = path;
for (const [encoding, suffix] of [
['br', '.br'],
['gzip', '.gz'],
] as const) {
if (new RegExp(`\\b${encoding}\\b`).test(acceptEncoding) && existsSync(path + suffix)) {
file = path + suffix;
headers['Content-Encoding'] = encoding;
break;
}
}
if (existsSync(path + '.br') || existsSync(path + '.gz')) headers.Vary = 'Accept-Encoding';
headers['Content-Length'] = String(statSync(file).size);
const body = head ? null : (Readable.toWeb(createReadStream(file)) as ReadableStream);
return new Response(body, { headers });
}
@@ -52,12 +65,12 @@ export function webApp(root: string): MiddlewareHandler {
const candidate = normalize(join(root, urlPath));
if (candidate.startsWith(rootWithSep) && existsSync(candidate) && statSync(candidate).isFile()) {
return fileResponse(candidate, urlPath, method === 'HEAD');
return fileResponse(candidate, urlPath, method === 'HEAD', c.req.header('accept-encoding'));
}
if (extname(urlPath)) return c.text('Not found', 404);
if (!existsSync(index)) {
return c.text('The Spendlight web app is not built yet. Run "npm run build" (or "npm run dev:web" while developing).', 503);
}
return fileResponse(index, '/index.html', method === 'HEAD');
return fileResponse(index, '/index.html', method === 'HEAD', c.req.header('accept-encoding'));
};
}

17
server/src/tags.ts Normal file
View File

@@ -0,0 +1,17 @@
import { sql } from 'drizzle-orm';
import type { AppContext } from './context';
import { tags } from './db/schema';
/** Tag IDs for these names, creating any that are new. The first spelling of a tag wins: "Back to school" and "back to school" are one tag. */
export function tagIdsFor(ctx: AppContext, names: string[]): number[] {
const byKey = new Map<string, string>();
for (const n of names) if (n.trim() && !byKey.has(n.trim().toLowerCase())) byKey.set(n.trim().toLowerCase(), n.trim());
return [...byKey.values()].map((name) => {
const existing = ctx.db
.select()
.from(tags)
.where(sql`lower(${tags.name}) = ${name.toLowerCase()}`)
.get();
return existing?.id ?? ctx.db.insert(tags).values({ name }).returning().get().id;
});
}

View File

@@ -2,13 +2,21 @@ import type { Account, Category, CategoryGroup, PaletteColor, Transaction } from
import { and, asc, count, desc, eq, inArray, max } from 'drizzle-orm';
import { currentBalance } from '../accounts/balance';
import type { AppContext } from '../context';
import { accounts, categories, categoryGroups, imports, splits, tags, transactions, transactionTags, type AccountRow, type TransactionRow } from '../db/schema';
import { accounts, categories, categoryGroups, imports, splits, splitTags, tags, transactions, transactionTags, type AccountRow, type TransactionRow } from '../db/schema';
/** Full transactions, with splits, tags and the other half of any transfer, loaded in a few queries. */
export function toTransactions(ctx: AppContext, rows: TransactionRow[]): Transaction[] {
if (!rows.length) return [];
const ids = rows.map((r) => r.id);
const splitRows = ctx.db.select().from(splits).where(inArray(splits.transactionId, ids)).orderBy(asc(splits.id)).all();
const splitTagRows = splitRows.length
? ctx.db
.select({ splitId: splitTags.splitId, name: tags.name })
.from(splitTags)
.innerJoin(tags, eq(tags.id, splitTags.tagId))
.where(inArray(splitTags.splitId, splitRows.map((s) => s.id)))
.all()
: [];
const tagRows = ctx.db
.select({ transactionId: transactionTags.transactionId, name: tags.name })
.from(transactionTags)
@@ -38,7 +46,7 @@ export function toTransactions(ctx: AppContext, rows: TransactionRow[]): Transac
isSplit: r.isSplit,
splits: splitRows
.filter((s) => s.transactionId === r.id)
.map((s) => ({ id: s.id, categoryId: s.categoryId, amountCents: s.amountCents, note: s.note })),
.map((s) => ({ id: s.id, categoryId: s.categoryId, amountCents: s.amountCents, note: s.note, tags: splitTagRows.filter((x) => x.splitId === s.id).map((x) => x.name) })),
note: r.note,
tags: tagRows.filter((t) => t.transactionId === r.id).map((t) => t.name),
transferPeer: peers.find((p) => p.id === r.transferPeerId) ?? null,

View File

@@ -1,7 +1,7 @@
import { z } from '@hono/zod-openapi';
import { and, eq, exists, gte, isNull, like, lte, or, sql, type SQL } from 'drizzle-orm';
import type { AppContext } from '../context';
import { splits, tags, transactions, transactionTags } from '../db/schema';
import { splits, splitTags, tags, transactions, transactionTags } from '../db/schema';
import { DateString } from '../schemas';
/** The search and filters shared by the transaction list and its exports. */
@@ -36,14 +36,26 @@ export function transactionWhere(ctx: AppContext, q: TransactionFilter): SQL | u
);
}
if (q.tag) {
const name = sql`lower(${tags.name}) = ${q.tag.toLowerCase()}`;
// On the transaction itself, or on any part of a split.
where.push(
exists(
ctx.db
.select()
.from(transactionTags)
.innerJoin(tags, eq(tags.id, transactionTags.tagId))
.where(and(eq(transactionTags.transactionId, transactions.id), sql`lower(${tags.name}) = ${q.tag.toLowerCase()}`)),
),
or(
exists(
ctx.db
.select()
.from(transactionTags)
.innerJoin(tags, eq(tags.id, transactionTags.tagId))
.where(and(eq(transactionTags.transactionId, transactions.id), name)),
),
exists(
ctx.db
.select()
.from(splitTags)
.innerJoin(tags, eq(tags.id, splitTags.tagId))
.innerJoin(splits, eq(splits.id, splitTags.splitId))
.where(and(eq(splits.transactionId, transactions.id), name)),
),
)!,
);
}
if (q.from) where.push(gte(transactions.date, q.from));

View File

@@ -333,6 +333,38 @@ describe('managing categories, rules and tags', () => {
});
});
describe('tags on split parts', () => {
it('finds, totals and exports a tag on one part of a purchase', async () => {
const page = (await admin.get('/api/v1/transactions?q=Kroger&limit=1')).json;
const kroger = page.items[0];
const groceries = await category('Groceries');
const household = await category('Household Supplies');
const split = await admin.call('PUT', `/api/v1/transactions/${kroger.id}/split`, {
parts: [
{ categoryId: groceries, amountCents: kroger.amountCents + 1500 },
{ categoryId: household, amountCents: -1500, tags: ['Reimbursable'] },
],
});
expect(split.json.splits.map((x: { tags: string[] }) => x.tags)).toEqual([[], ['Reimbursable']]);
const found = (await admin.get('/api/v1/transactions?tag=reimbursable')).json;
expect(found.items.map((x: { id: number }) => x.id)).toEqual([kroger.id]);
const report = (await admin.get('/api/v1/reports/tags')).json as { tag: string; spent: number; count: number; byCategory: { categoryId: number }[] }[];
expect(report.find((r) => r.tag === 'Reimbursable')).toMatchObject({ spent: 1500, count: 1, byCategory: [{ categoryId: household, cents: 1500 }] });
expect((await admin.get('/api/v1/tags')).json).toEqual([{ id: expect.any(Number), name: 'Reimbursable', count: 1 }]);
// Tagging the whole purchase too doesn't count the tagged part twice.
await admin.patch(`/api/v1/transactions/${kroger.id}`, { tags: ['Reimbursable'] });
const both = (await admin.get('/api/v1/reports/tags')).json as { tag: string; spent: number }[];
expect(both.find((r) => r.tag === 'Reimbursable')!.spent).toBe(-kroger.amountCents);
const res = await t.app.request(`http://localhost/api/v1/export/transactions.csv?tag=Reimbursable`, { headers: { Cookie: `sl_session=${admin.cookie('sl_session')}` } });
const lines = Buffer.from(await res.arrayBuffer()).subarray(3).toString('utf8').trim().split('\r\n');
expect(lines).toHaveLength(3);
expect(lines.every((l, i) => i === 0 || l.includes('Reimbursable'))).toBe(true);
});
});
describe('export', () => {
it('writes CSV that defuses formulas and splits rows by part', async () => {
expect(buildCsv(['A', 'B'], [['=HYPERLINK("x")', { money: -1234 }], ['a,b', 5]])).toBe('\uFEFFA,B\r\n"\'=HYPERLINK(""x"")",-12.34\r\n"a,b",5\r\n');

88
server/test/ofx.test.ts Normal file
View File

@@ -0,0 +1,88 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { parseOfx } from '../src/import/ofx';
import { makeApp, setUpAdmin, type Client, type TestApp } from './helpers';
// Made-up OFX files in both flavors banks send: the older SGML style with
// unclosed tags (bank account), and XML (credit card, as in a .qfx).
const SGML = `OFXHEADER:100
DATA:OFXSGML
VERSION:102
SECURITY:NONE
ENCODING:USASCII
CHARSET:1252
<OFX>
<SIGNONMSGSRSV1><SONRS><STATUS><CODE>0<SEVERITY>INFO</STATUS><DTSERVER>20261007120000<LANGUAGE>ENG<FI><ORG>Bank of America<FID>5959</FI></SONRS></SIGNONMSGSRSV1>
<BANKMSGSRSV1><STMTTRNRS><TRNUID>1<STATUS><CODE>0<SEVERITY>INFO</STATUS>
<STMTRS><CURDEF>USD<BANKACCTFROM><BANKID>000000000<ACCTID>000099991234<ACCTTYPE>CHECKING</BANKACCTFROM>
<BANKTRANLIST><DTSTART>20260901<DTEND>20260930
<STMTTRN><TRNTYPE>CREDIT<DTPOSTED>20260905120000.000[-5:EST]<TRNAMT>3000.00<FITID>A1<NAME>ACME CORP PAYROLL<MEMO>PPD ID: 123</STMTTRN>
<STMTTRN><TRNTYPE>DEBIT<DTPOSTED>20260912<TRNAMT>-54.20<FITID>A2<NAME>KROGER #123 CHARLOTTE NC</STMTTRN>
<STMTTRN><TRNTYPE>XFER<DTPOSTED>20260915<TRNAMT>-300.00<FITID>A3<NAME>Online Banking transfer to SAV<MEMO>5678 Confirmation# 1</STMTTRN>
<STMTTRN><TRNTYPE>DEBIT<DTPOSTED>20260920<TRNAMT>-12,50<FITID>A4<NAME>JOES COFFEE &amp; BAGELS</STMTTRN>
</BANKTRANLIST><LEDGERBAL><BALAMT>4633.30<DTASOF>20260930</LEDGERBAL><AVAILBAL><BALAMT>4633.30<DTASOF>20260930</AVAILBAL></STMTRS></STMTTRNRS></BANKMSGSRSV1></OFX>
`;
const XML = `<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?OFX OFXHEADER="200" VERSION="220" SECURITY="NONE" OLDFILEUID="NONE" NEWFILEUID="NONE"?>
<OFX><CREDITCARDMSGSRSV1><CCSTMTTRNRS><TRNUID>1</TRNUID><CCSTMTRS><CURDEF>USD</CURDEF>
<CCACCTFROM><ACCTID>4111XXXXXXXX9876</ACCTID></CCACCTFROM>
<BANKTRANLIST><DTSTART>20260901</DTSTART><DTEND>20260930</DTEND>
<STMTTRN><TRNTYPE>DEBIT</TRNTYPE><DTPOSTED>20260903</DTPOSTED><DTUSER>20260902</DTUSER><TRNAMT>-23.99</TRNAMT><FITID>C1</FITID><NAME>AMAZON MKTPL*AB12CD3E4</NAME></STMTTRN>
<STMTTRN><TRNTYPE>CREDIT</TRNTYPE><DTPOSTED>20260907</DTPOSTED><TRNAMT>18.50</TRNAMT><FITID>C2</FITID><NAME>AMAZON MKTPL*QR56ST7U8</NAME><MEMO>Return</MEMO></STMTTRN>
<STMTTRN><TRNTYPE>DEBIT</TRNTYPE><DTPOSTED>20260912</DTPOSTED><TRNAMT>-15.49</TRNAMT><FITID>C3</FITID><NAME>NETFLIX.COM</NAME></STMTTRN>
</BANKTRANLIST><LEDGERBAL><BALAMT>-412.33</BALAMT><DTASOF>20260930</DTASOF></LEDGERBAL></CCSTMTRS></CCSTMTTRNRS></CREDITCARDMSGSRSV1></OFX>
`;
describe('reading OFX', () => {
it('reads the older SGML style', () => {
const s = parseOfx(SGML);
expect(s).toMatchObject({ kind: 'checking', institution: 'Bank of America', accountLast4: '1234', ledger: { cents: 463330, date: '2026-09-30' } });
expect(s.transactions.map((t) => [t.date, t.amountCents, t.name])).toEqual([
['2026-09-05', 300000, 'ACME CORP PAYROLL'],
['2026-09-12', -5420, 'KROGER #123 CHARLOTTE NC'],
['2026-09-15', -30000, 'Online Banking transfer to SAV'],
['2026-09-20', -1250, 'JOES COFFEE & BAGELS'],
]);
});
it('reads XML, using the purchase date when there is one', () => {
const s = parseOfx(XML);
expect(s).toMatchObject({ kind: 'credit_card', accountLast4: '9876' });
expect(s.transactions[0]).toMatchObject({ date: '2026-09-02', postedDate: '2026-09-03', amountCents: -2399 });
expect(s.transactions[1]).toMatchObject({ amountCents: 1850, memo: 'Return' });
});
});
describe('importing OFX', () => {
let t: TestApp;
let admin: Client;
beforeEach(async () => {
t = await makeApp();
admin = await setUpAdmin(t);
});
afterEach(() => t.cleanup());
it('imports with no mapping, matches the account by its last digits, and skips repeats', async () => {
const checking = (await admin.post('/api/v1/accounts', { name: 'Checking', kind: 'checking', institution: 'Bank of America', last4: '1234' })).json.id;
await admin.post('/api/v1/accounts', { name: 'Prime Visa', kind: 'credit_card', institution: 'Chase', last4: '9876' });
const preview = await admin.upload('/api/v1/imports/files', 'export.qfx', SGML);
expect(preview.json).toMatchObject({ layout: { key: 'ofx' }, account: { id: checking, confidence: 'certain' }, counts: { new: 4 } });
const done = await admin.post('/api/v1/imports', { stagingId: preview.json.stagingId, accountId: checking });
expect(done.status).toBe(200);
const page = (await admin.get(`/api/v1/transactions?accountId=${checking}`)).json;
expect(page.items.map((x: { merchant: string }) => x.merchant)).toContain('Kroger');
const account = ((await admin.get('/api/v1/accounts')).json as { id: number; balanceCents: number; balanceConfirmedOn: string }[]).find((a) => a.id === checking)!;
expect(account).toMatchObject({ balanceCents: 463330, balanceConfirmedOn: '2026-09-20' });
const again = await admin.upload('/api/v1/imports/files', 'export.qfx', SGML);
expect(again.json.counts).toMatchObject({ new: 0, duplicates: 4 });
const card = await admin.upload('/api/v1/imports/files', 'Statement.qfx', XML);
expect(card.json.account.confidence).toBe('certain');
expect(card.json.accountHint).toMatchObject({ kind: 'credit_card' });
});
});

128
server/test/restore.test.ts Normal file
View File

@@ -0,0 +1,128 @@
import Database from 'better-sqlite3';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { listBackups } from '../src/backups/backup';
import { makeApp, setUpAdmin, type Client, type TestApp } from './helpers';
import { discoverCsv } from './synthetic';
// The milestone 3 drill, automated: back up, change things, restore, and get
// everything back exactly. All data here is made up.
let t: TestApp;
let admin: Client;
const snapshot = async () => {
const page = (await admin.get('/api/v1/transactions?limit=500')).json;
return {
total: page.total,
sums: page.sums,
accounts: ((await admin.get('/api/v1/accounts')).json as { name: string; balanceCents: number }[]).map((a) => [a.name, a.balanceCents]),
budgets: (await admin.get('/api/v1/budgets?year=2026')).json,
subscriptions: ((await admin.get('/api/v1/subscriptions')).json as { name: string; status: string }[]).map((s) => [s.name, s.status]),
};
};
beforeEach(async () => {
t = await makeApp();
admin = await setUpAdmin(t);
const card = (await admin.post('/api/v1/accounts', { name: 'Discover', kind: 'credit_card', institution: 'Discover' })).json.id;
const preview = await admin.upload('/api/v1/imports/files', 'Discover-2026.csv', discoverCsv());
await admin.post('/api/v1/imports', { stagingId: preview.json.stagingId, accountId: card });
await admin.post('/api/v1/budgets/fill', { month: '2026-10', source: 'average3' });
});
afterEach(() => t.cleanup());
describe('restoring a backup', () => {
it('brings everything back, after saving a copy of how things were', async () => {
const before = await snapshot();
const backup = (await admin.post('/api/v1/backups')).json.name as string;
// Things go wrong: the import is undone and a budget is wiped.
const imports = (await admin.get('/api/v1/imports')).json as { id: number }[];
await admin.del(`/api/v1/imports/${imports[0]!.id}`);
await admin.put('/api/v1/budgets', { categoryId: before.budgets.items[0].categoryId, year: 2026, month: 1, amountCents: 0, scope: 'year' });
expect((await snapshot()).total).toBe(0);
const check = await admin.get(`/api/v1/backups/${backup}/check`);
expect(check.json.counts).toMatchObject({ transactions: before.total, accounts: 1, people: 1 });
const restored = await admin.post(`/api/v1/backups/${backup}/restore`);
expect(restored.status).toBe(200);
expect(restored.json.safetyCopy).toMatch(/^spendlight-pre-restore-/);
expect(await snapshot()).toEqual(before);
// The app keeps working on the restored data, and the restore can be undone.
expect((await admin.post('/api/v1/subscriptions/scan')).status).toBe(200);
const undo = await admin.post(`/api/v1/backups/${restored.json.safetyCopy}/restore`);
expect(undo.status).toBe(200);
expect((await snapshot()).total).toBe(0);
expect(listBackups(t.ctx.config.backupDir).filter((b) => b.kind === 'pre-restore')).toHaveLength(2);
});
it('keeps whoever restored signed in, and leaves only whole files in the backup folder', async () => {
const backup = (await admin.post('/api/v1/backups')).json.name as string;
// A sign-in newer than the backup.
const { Client } = await import('./helpers');
const fresh = new Client(t);
await fresh.post('/api/v1/auth/login', { username: 'Chris', password: 'correct-horse' });
await fresh.get(`/api/v1/backups/${backup}/check`);
expect((await fresh.post(`/api/v1/backups/${backup}/restore`)).status).toBe(200);
expect((await fresh.get('/api/v1/me')).status).toBe(200);
const names = (await import('node:fs')).readdirSync(t.ctx.config.backupDir);
expect(names.every((n) => /^spendlight-[a-z-]+-\d{8}-\d{6}\.db$/.test(n))).toBe(true);
});
it('accepts an uploaded backup, such as one from another machine', async () => {
const before = await snapshot();
const backup = (await admin.post('/api/v1/backups')).json.name as string;
const imports = (await admin.get('/api/v1/imports')).json as { id: number }[];
await admin.del(`/api/v1/imports/${imports[0]!.id}`);
const form = new FormData();
const bytes = (await import('node:fs')).readFileSync(join(t.ctx.config.backupDir, backup));
form.append('file', new File([bytes], backup));
const res = await t.app.request('http://localhost/api/v1/backups/restore-upload', { method: 'POST', body: form, headers: { Cookie: `sl_session=${admin.cookie('sl_session')}` } });
expect(res.status).toBe(200);
expect(await snapshot()).toEqual(before);
});
it('refuses files that aren’t a Spendlight backup', async () => {
const dir = t.ctx.config.backupDir;
const upload = async (name: string, data: Buffer | string) => {
const form = new FormData();
form.append('file', new File([data], name));
const res = await t.app.request('http://localhost/api/v1/backups/restore-upload', { method: 'POST', body: form, headers: { Cookie: `sl_session=${admin.cookie('sl_session')}` } });
return { status: res.status, json: (await res.json()) as { error: { code: string } } };
};
expect((await upload('notes.db', 'just some text')).json.error.code).toBe('not_sqlite');
// A SQLite file from some other app.
const other = join(dir, 'other.db');
const db = new Database(other);
db.exec('CREATE TABLE things (id INTEGER)');
db.close();
expect((await upload('other.db', (await import('node:fs')).readFileSync(other))).json.error.code).toBe('not_spendlight');
// A Spendlight backup from a newer version, with a database upgrade this one doesn't know.
const backup = (await admin.post('/api/v1/backups')).json.name as string;
const newer = new Database(join(dir, backup));
newer.prepare('INSERT INTO __drizzle_migrations (hash, created_at) VALUES (?, ?)').run('future', Date.now());
newer.close();
const tooNew = await admin.post(`/api/v1/backups/${backup}/restore`);
expect(tooNew.json.error.code).toBe('too_new');
// Nothing changed, and no safety copies were needed.
expect(listBackups(dir).filter((b) => b.kind === 'pre-restore')).toHaveLength(0);
expect((await admin.get('/api/v1/transactions?limit=1')).json.total).toBeGreaterThan(0);
});
it('is for admins only, and only for names in the backup folder', async () => {
await admin.post('/api/v1/people', { username: 'helper', displayName: 'Helper', access: 'full', isAdmin: false, password: 'helper-password-1' });
const { Client } = await import('./helpers');
const helper = new Client(t);
await helper.post('/api/v1/auth/login', { username: 'helper', password: 'helper-password-1' });
const backup = (await admin.post('/api/v1/backups')).json.name as string;
expect((await helper.post(`/api/v1/backups/${backup}/restore`)).status).toBe(403);
expect((await admin.post('/api/v1/backups/..%2Fdata%2Fspendlight.db/restore')).status).toBe(404);
});
});