feat: add shadow AI benchmarking for admin model comparison
Add a new admin-only feature that allows the platform owner to benchmark the production AI model against up to 2 alternate models (any OpenAI-compatible API) using real tenant data, without impacting users. Backend: - Shared AI caller utility (ai-caller.ts) for OpenAI-compatible endpoints - Shadow AI module with service, controller, and 3 entities - 6 admin API endpoints for model config CRUD, run trigger, and history - Auto-creates shadow_ai_models, shadow_runs, shadow_run_results tables - Exposes health-scores and investment-planning prompt builders for reuse Frontend: - New admin page at /admin/shadow-ai with 3 tabs: - Model Configuration (production + 2 alternate slots) - Run Comparison (tenant select, feature select, side-by-side results) - History (filterable run log with detail drill-down) - Full side-by-side output display with diff highlighting - Sidebar navigation link for AI Benchmarking Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
JoeBot
@@ -877,7 +877,7 @@ export class InvestmentPlanningService {
|
||||
|
||||
// ── Private: AI Prompt Construction ──
|
||||
|
||||
private buildPromptMessages(
|
||||
buildPromptMessages(
|
||||
snapshot: any,
|
||||
allRates: { cd: MarketRate[]; money_market: MarketRate[]; high_yield_savings: MarketRate[] },
|
||||
monthlyForecast: any,
|
||||
@@ -1059,6 +1059,285 @@ Based on this complete financial picture INCLUDING the 12-month cash flow foreca
|
||||
];
|
||||
}
|
||||
|
||||
// ── Schema-Based Prompt Building (for shadow AI benchmarking) ──
|
||||
|
||||
/**
|
||||
* Build investment recommendation prompt messages for a specific tenant schema.
|
||||
* Bypasses request-scoped TenantService by using DataSource directly.
|
||||
*/
|
||||
async buildPromptForSchema(schemaName: string): Promise<Array<{ role: string; content: string }>> {
|
||||
const qr = this.dataSource.createQueryRunner();
|
||||
try {
|
||||
await qr.connect();
|
||||
await qr.query(`SET search_path TO "${schemaName}"`);
|
||||
|
||||
const year = new Date().getFullYear();
|
||||
const currentMonth = new Date().getMonth() + 1;
|
||||
const monthNames = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec_amt'];
|
||||
const monthLabels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'];
|
||||
|
||||
// ── Gather financial snapshot data ──
|
||||
const [accountBalances, investmentAccounts, budgets, projects] = await Promise.all([
|
||||
qr.query(`
|
||||
SELECT a.id, a.account_number, a.name, a.account_type, a.fund_type, a.interest_rate,
|
||||
CASE
|
||||
WHEN a.account_type IN ('asset', 'expense')
|
||||
THEN COALESCE(SUM(jel.debit), 0) - COALESCE(SUM(jel.credit), 0)
|
||||
ELSE COALESCE(SUM(jel.credit), 0) - COALESCE(SUM(jel.debit), 0)
|
||||
END as balance
|
||||
FROM accounts a
|
||||
LEFT JOIN journal_entry_lines jel ON jel.account_id = a.id
|
||||
LEFT JOIN journal_entries je ON je.id = jel.journal_entry_id
|
||||
AND je.is_posted = true AND je.is_void = false
|
||||
WHERE a.is_active = true AND a.account_type IN ('asset', 'liability', 'equity')
|
||||
GROUP BY a.id, a.account_number, a.name, a.account_type, a.fund_type, a.interest_rate
|
||||
ORDER BY a.account_number
|
||||
`),
|
||||
qr.query(`
|
||||
SELECT id, name, institution, investment_type, fund_type,
|
||||
principal, interest_rate, maturity_date, purchase_date, current_value
|
||||
FROM investment_accounts WHERE is_active = true
|
||||
ORDER BY maturity_date NULLS LAST
|
||||
`),
|
||||
qr.query(
|
||||
`SELECT b.fund_type, a.account_type, a.name, a.account_number,
|
||||
(b.jan + b.feb + b.mar + b.apr + b.may + b.jun +
|
||||
b.jul + b.aug + b.sep + b.oct + b.nov + b.dec_amt) as annual_total
|
||||
FROM budgets b JOIN accounts a ON a.id = b.account_id
|
||||
WHERE b.fiscal_year = $1 ORDER BY a.account_type, a.account_number`,
|
||||
[year],
|
||||
),
|
||||
qr.query(`
|
||||
SELECT name, estimated_cost, target_year, target_month, fund_source,
|
||||
status, priority, current_fund_balance, funded_percentage
|
||||
FROM projects WHERE is_active = true AND status IN ('planned', 'approved', 'in_progress')
|
||||
ORDER BY target_year, target_month NULLS LAST, priority
|
||||
`),
|
||||
]);
|
||||
|
||||
// Cash flow context
|
||||
const [opCashResult, resCashResult, assessmentIncome] = await Promise.all([
|
||||
qr.query(`
|
||||
SELECT COALESCE(SUM(sub.bal), 0) as total FROM (
|
||||
SELECT COALESCE(SUM(jel.debit), 0) - COALESCE(SUM(jel.credit), 0) as bal
|
||||
FROM accounts a
|
||||
JOIN journal_entry_lines jel ON jel.account_id = a.id
|
||||
JOIN journal_entries je ON je.id = jel.journal_entry_id AND je.is_posted = true AND je.is_void = false
|
||||
WHERE a.account_type = 'asset' AND a.fund_type = 'operating' AND a.is_active = true
|
||||
GROUP BY a.id
|
||||
) sub
|
||||
`),
|
||||
qr.query(`
|
||||
SELECT COALESCE(SUM(sub.bal), 0) as total FROM (
|
||||
SELECT COALESCE(SUM(jel.debit), 0) - COALESCE(SUM(jel.credit), 0) as bal
|
||||
FROM accounts a
|
||||
JOIN journal_entry_lines jel ON jel.account_id = a.id
|
||||
JOIN journal_entries je ON je.id = jel.journal_entry_id AND je.is_posted = true AND je.is_void = false
|
||||
WHERE a.account_type = 'asset' AND a.fund_type = 'reserve' AND a.is_active = true
|
||||
GROUP BY a.id
|
||||
) sub
|
||||
`),
|
||||
qr.query(`
|
||||
SELECT COALESCE(SUM(ag.regular_assessment *
|
||||
(SELECT COUNT(*) FROM units u WHERE u.assessment_group_id = ag.id AND u.status = 'active')), 0) as monthly_assessment_income
|
||||
FROM assessment_groups ag WHERE ag.is_active = true
|
||||
`),
|
||||
]);
|
||||
|
||||
const operatingCash = accountBalances
|
||||
.filter((a: any) => a.fund_type === 'operating' && a.account_type === 'asset')
|
||||
.reduce((sum: number, a: any) => sum + parseFloat(a.balance || '0'), 0);
|
||||
const reserveCash = accountBalances
|
||||
.filter((a: any) => a.fund_type === 'reserve' && a.account_type === 'asset')
|
||||
.reduce((sum: number, a: any) => sum + parseFloat(a.balance || '0'), 0);
|
||||
const operatingInvestments = investmentAccounts
|
||||
.filter((i: any) => i.fund_type === 'operating')
|
||||
.reduce((sum: number, i: any) => sum + parseFloat(i.current_value || i.principal || '0'), 0);
|
||||
const reserveInvestments = investmentAccounts
|
||||
.filter((i: any) => i.fund_type === 'reserve')
|
||||
.reduce((sum: number, i: any) => sum + parseFloat(i.current_value || i.principal || '0'), 0);
|
||||
|
||||
const snapshot = {
|
||||
summary: {
|
||||
operating_cash: operatingCash,
|
||||
reserve_cash: reserveCash,
|
||||
operating_investments: operatingInvestments,
|
||||
reserve_investments: reserveInvestments,
|
||||
total_operating: operatingCash + operatingInvestments,
|
||||
total_reserve: reserveCash + reserveInvestments,
|
||||
total_all: operatingCash + reserveCash + operatingInvestments + reserveInvestments,
|
||||
},
|
||||
account_balances: accountBalances,
|
||||
investment_accounts: investmentAccounts,
|
||||
budgets,
|
||||
projects,
|
||||
cash_flow_context: {
|
||||
current_operating_cash: parseFloat(opCashResult[0]?.total || '0'),
|
||||
current_reserve_cash: parseFloat(resCashResult[0]?.total || '0'),
|
||||
budget_summary: await qr.query(
|
||||
`SELECT b.fund_type, a.account_type,
|
||||
SUM(b.jan + b.feb + b.mar + b.apr + b.may + b.jun +
|
||||
b.jul + b.aug + b.sep + b.oct + b.nov + b.dec_amt) as annual_total
|
||||
FROM budgets b JOIN accounts a ON a.id = b.account_id
|
||||
WHERE b.fiscal_year = $1 GROUP BY b.fund_type, a.account_type`,
|
||||
[year],
|
||||
),
|
||||
monthly_assessment_income: parseFloat(assessmentIncome[0]?.monthly_assessment_income || '0'),
|
||||
},
|
||||
};
|
||||
|
||||
// ── Build monthly forecast ──
|
||||
const [opCashRows2, resCashRows2, opInvRows, resInvRows] = await Promise.all([
|
||||
qr.query(`SELECT COALESCE(SUM(sub.bal), 0) as total FROM (
|
||||
SELECT COALESCE(SUM(jel.debit), 0) - COALESCE(SUM(jel.credit), 0) as bal
|
||||
FROM accounts a JOIN journal_entry_lines jel ON jel.account_id = a.id
|
||||
JOIN journal_entries je ON je.id = jel.journal_entry_id AND je.is_posted = true AND je.is_void = false
|
||||
WHERE a.account_type = 'asset' AND a.fund_type = 'operating' AND a.is_active = true GROUP BY a.id
|
||||
) sub`),
|
||||
qr.query(`SELECT COALESCE(SUM(sub.bal), 0) as total FROM (
|
||||
SELECT COALESCE(SUM(jel.debit), 0) - COALESCE(SUM(jel.credit), 0) as bal
|
||||
FROM accounts a JOIN journal_entry_lines jel ON jel.account_id = a.id
|
||||
JOIN journal_entries je ON je.id = jel.journal_entry_id AND je.is_posted = true AND je.is_void = false
|
||||
WHERE a.account_type = 'asset' AND a.fund_type = 'reserve' AND a.is_active = true GROUP BY a.id
|
||||
) sub`),
|
||||
qr.query(`SELECT COALESCE(SUM(current_value), 0) as total FROM investment_accounts WHERE fund_type = 'operating' AND is_active = true`),
|
||||
qr.query(`SELECT COALESCE(SUM(current_value), 0) as total FROM investment_accounts WHERE fund_type = 'reserve' AND is_active = true`),
|
||||
]);
|
||||
|
||||
let runOpCash = parseFloat(opCashRows2[0]?.total || '0');
|
||||
let runResCash = parseFloat(resCashRows2[0]?.total || '0');
|
||||
let runOpInv = parseFloat(opInvRows[0]?.total || '0');
|
||||
let runResInv = parseFloat(resInvRows[0]?.total || '0');
|
||||
|
||||
const assessmentGroups = await qr.query(`
|
||||
SELECT ag.frequency, ag.regular_assessment, ag.special_assessment,
|
||||
(SELECT COUNT(*) FROM units u WHERE u.assessment_group_id = ag.id AND u.status = 'active') as unit_count
|
||||
FROM assessment_groups ag WHERE ag.is_active = true
|
||||
`);
|
||||
|
||||
const getAssessmentIncome = (month: number): { operating: number; reserve: number } => {
|
||||
let operating = 0, reserve = 0;
|
||||
for (const g of assessmentGroups) {
|
||||
const units = parseInt(g.unit_count) || 0;
|
||||
const regular = parseFloat(g.regular_assessment) || 0;
|
||||
const special = parseFloat(g.special_assessment) || 0;
|
||||
const freq = g.frequency || 'monthly';
|
||||
let applies = false;
|
||||
if (freq === 'monthly') applies = true;
|
||||
else if (freq === 'quarterly') applies = [1,4,7,10].includes(month);
|
||||
else if (freq === 'annual') applies = month === 1;
|
||||
if (applies) { operating += regular * units; reserve += special * units; }
|
||||
}
|
||||
return { operating, reserve };
|
||||
};
|
||||
|
||||
const budgetsByYearMonth: Record<string, { opIncome: number; opExpense: number; resIncome: number; resExpense: number }> = {};
|
||||
for (const yr of [year, year + 1]) {
|
||||
const budgetRows = await qr.query(
|
||||
`SELECT b.fund_type, a.account_type,
|
||||
b.jan, b.feb, b.mar, b.apr, b.may, b.jun, b.jul, b.aug, b.sep, b.oct, b.nov, b.dec_amt
|
||||
FROM budgets b JOIN accounts a ON a.id = b.account_id WHERE b.fiscal_year = $1`, [yr],
|
||||
);
|
||||
for (let m = 0; m < 12; m++) {
|
||||
const key = `${yr}-${m + 1}`;
|
||||
if (!budgetsByYearMonth[key]) budgetsByYearMonth[key] = { opIncome: 0, opExpense: 0, resIncome: 0, resExpense: 0 };
|
||||
for (const row of budgetRows) {
|
||||
const amt = parseFloat(row[monthNames[m]]) || 0;
|
||||
if (amt === 0) continue;
|
||||
const isOp = row.fund_type === 'operating';
|
||||
if (row.account_type === 'income') { if (isOp) budgetsByYearMonth[key].opIncome += amt; else budgetsByYearMonth[key].resIncome += amt; }
|
||||
else if (row.account_type === 'expense') { if (isOp) budgetsByYearMonth[key].opExpense += amt; else budgetsByYearMonth[key].resExpense += amt; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const maturities = await qr.query(`
|
||||
SELECT fund_type, current_value, maturity_date, interest_rate, purchase_date
|
||||
FROM investment_accounts WHERE is_active = true AND maturity_date IS NOT NULL AND maturity_date > CURRENT_DATE
|
||||
`);
|
||||
const maturityIndex: Record<string, { operating: number; reserve: number }> = {};
|
||||
for (const inv of maturities) {
|
||||
const d = new Date(inv.maturity_date);
|
||||
const key = `${d.getFullYear()}-${d.getMonth() + 1}`;
|
||||
if (!maturityIndex[key]) maturityIndex[key] = { operating: 0, reserve: 0 };
|
||||
const val = parseFloat(inv.current_value) || 0;
|
||||
const rate = parseFloat(inv.interest_rate) || 0;
|
||||
const purchaseDate = inv.purchase_date ? new Date(inv.purchase_date) : new Date();
|
||||
const matDate = new Date(inv.maturity_date);
|
||||
const daysHeld = Math.max((matDate.getTime() - purchaseDate.getTime()) / 86400000, 1);
|
||||
const interestEarned = val * (rate / 100) * (daysHeld / 365);
|
||||
const maturityTotal = val + interestEarned;
|
||||
if (inv.fund_type === 'operating') maturityIndex[key].operating += maturityTotal;
|
||||
else maturityIndex[key].reserve += maturityTotal;
|
||||
}
|
||||
|
||||
const projectExpenses = await qr.query(`
|
||||
SELECT estimated_cost, target_year, target_month, fund_source
|
||||
FROM projects WHERE is_active = true AND status IN ('planned', 'in_progress')
|
||||
AND target_year IS NOT NULL AND estimated_cost > 0
|
||||
`);
|
||||
const projectIndex: Record<string, { operating: number; reserve: number }> = {};
|
||||
for (const p of projectExpenses) {
|
||||
const yr2 = parseInt(p.target_year);
|
||||
const mo = parseInt(p.target_month) || 6;
|
||||
const key = `${yr2}-${mo}`;
|
||||
if (!projectIndex[key]) projectIndex[key] = { operating: 0, reserve: 0 };
|
||||
const cost = parseFloat(p.estimated_cost) || 0;
|
||||
if (p.fund_source === 'operating') projectIndex[key].operating += cost;
|
||||
else projectIndex[key].reserve += cost;
|
||||
}
|
||||
|
||||
const datapoints: any[] = [];
|
||||
for (let i = 0; i < 12; i++) {
|
||||
const fYear = year + Math.floor((currentMonth - 1 + i) / 12);
|
||||
const fMonth = ((currentMonth - 1 + i) % 12) + 1;
|
||||
const key = `${fYear}-${fMonth}`;
|
||||
const label = `${monthLabels[fMonth - 1]} ${fYear}`;
|
||||
const assessments = getAssessmentIncome(fMonth);
|
||||
const budget = budgetsByYearMonth[key] || { opIncome: 0, opExpense: 0, resIncome: 0, resExpense: 0 };
|
||||
const maturity = maturityIndex[key] || { operating: 0, reserve: 0 };
|
||||
const project = projectIndex[key] || { operating: 0, reserve: 0 };
|
||||
const opIncomeMonth = budget.opIncome > 0 ? budget.opIncome : assessments.operating;
|
||||
const resIncomeMonth = budget.resIncome > 0 ? budget.resIncome : assessments.reserve;
|
||||
runOpCash += opIncomeMonth - budget.opExpense - project.operating + maturity.operating;
|
||||
runResCash += resIncomeMonth - budget.resExpense - project.reserve + maturity.reserve;
|
||||
if (maturity.operating > 0) runOpInv = Math.max(0, runOpInv - (maturity.operating * 0.96));
|
||||
if (maturity.reserve > 0) runResInv = Math.max(0, runResInv - (maturity.reserve * 0.96));
|
||||
datapoints.push({
|
||||
month: label,
|
||||
operating_cash: Math.round(runOpCash * 100) / 100,
|
||||
operating_investments: Math.round(runOpInv * 100) / 100,
|
||||
reserve_cash: Math.round(runResCash * 100) / 100,
|
||||
reserve_investments: Math.round(runResInv * 100) / 100,
|
||||
op_income: Math.round(opIncomeMonth * 100) / 100,
|
||||
op_expense: Math.round(budget.opExpense * 100) / 100,
|
||||
res_income: Math.round(resIncomeMonth * 100) / 100,
|
||||
res_expense: Math.round(budget.resExpense * 100) / 100,
|
||||
project_cost_op: Math.round(project.operating * 100) / 100,
|
||||
project_cost_res: Math.round(project.reserve * 100) / 100,
|
||||
maturity_op: Math.round(maturity.operating * 100) / 100,
|
||||
maturity_res: Math.round(maturity.reserve * 100) / 100,
|
||||
});
|
||||
}
|
||||
|
||||
const assessmentSchedule = assessmentGroups.map((g: any) => ({
|
||||
frequency: g.frequency || 'monthly',
|
||||
regular_per_unit: parseFloat(g.regular_assessment) || 0,
|
||||
special_per_unit: parseFloat(g.special_assessment) || 0,
|
||||
units: parseInt(g.unit_count) || 0,
|
||||
total_regular: (parseFloat(g.regular_assessment) || 0) * (parseInt(g.unit_count) || 0),
|
||||
total_special: (parseFloat(g.special_assessment) || 0) * (parseInt(g.unit_count) || 0),
|
||||
}));
|
||||
|
||||
const monthlyForecast = { datapoints, assessment_schedule: assessmentSchedule };
|
||||
const allRates = await this.getMarketRates();
|
||||
|
||||
return this.buildPromptMessages(snapshot, allRates, monthlyForecast);
|
||||
} finally {
|
||||
await qr.release();
|
||||
}
|
||||
}
|
||||
|
||||
// ── Private: AI API Call ──
|
||||
|
||||
private async callAI(messages: Array<{ role: string; content: string }>): Promise<AIResponse> {
|
||||
|
||||
Reference in New Issue
Block a user