import { useState, useEffect } from 'react'; import { Title, Text, Card, SimpleGrid, Group, Stack, Badge, Loader, Center, Tabs, TextInput, Button, PasswordInput, Select, Table, Accordion, Switch, Paper, RingProgress, Divider, Alert, Code, ScrollArea, Box, Tooltip, ActionIcon, } from '@mantine/core'; import { IconScale, IconSettings, IconPlayerPlay, IconHistory, IconCheck, IconX, IconAlertTriangle, IconClock, IconTrash, IconRefresh, IconArrowRight, IconChevronDown, } from '@tabler/icons-react'; import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import api from '../../services/api'; // ── Interfaces ── interface ShadowModel { id: string; slot: string; name: string; api_url: string; api_key: string; model_name: string; is_active: boolean; created_at: string; updated_at: string; } interface ShadowRunResult { id: string; run_id: string; model_role: string; model_name: string; api_url: string; raw_response: string; parsed_response: any; response_time_ms: number; token_usage: any; status: string; error_message: string; created_at: string; } interface ShadowRun { id: string; tenant_id: string; tenant_name: string; feature: string; status: string; prompt_messages: any; started_at: string; completed_at: string; created_at: string; results: ShadowRunResult[]; result_count?: string; success_count?: string; } interface AdminOrg { id: string; name: string; status: string; } // ── Helper Functions ── const featureLabels: Record = { operating_health: 'Operating Health', reserve_health: 'Reserve Health', investment_recommendations: 'Investment Recommendations', }; const roleLabels: Record = { production: 'Production', alternate_a: 'Alternate A', alternate_b: 'Alternate B', }; const statusColor: Record = { running: 'blue', completed: 'green', partial: 'yellow', failed: 'red', pending: 'gray', success: 'green', error: 'red', }; function formatDuration(ms: number | null): string { if (!ms) return '-'; if (ms < 1000) return `${ms}ms`; return `${(ms / 1000).toFixed(1)}s`; } function formatDate(d: string): string { if (!d) return '-'; return new Date(d).toLocaleString(); } // ── Model Configuration Tab ── function ModelConfigTab() { const queryClient = useQueryClient(); const { data: models, isLoading } = useQuery({ queryKey: ['shadow-ai-models'], queryFn: () => api.get('/admin/shadow-ai/models').then((r) => r.data), }); const modelA = models?.find((m) => m.slot === 'A'); const modelB = models?.find((m) => m.slot === 'B'); return ( Configure alternate AI models to benchmark against the production model. Each model can use any OpenAI-compatible API endpoint. ); } function ProductionModelCard() { return ( Production Model Active Configured via environment variables Production model settings are managed through server environment variables and cannot be changed from the UI. ); } function ModelSlotCard({ slot, model, isLoading }: { slot: string; model?: ShadowModel; isLoading: boolean }) { const queryClient = useQueryClient(); const [name, setName] = useState(''); const [apiUrl, setApiUrl] = useState(''); const [apiKey, setApiKey] = useState(''); const [modelName, setModelName] = useState(''); const [isActive, setIsActive] = useState(true); useEffect(() => { if (model) { setName(model.name); setApiUrl(model.api_url); setApiKey(model.api_key); setModelName(model.model_name); setIsActive(model.is_active); } }, [model]); const saveMutation = useMutation({ mutationFn: () => api.put(`/admin/shadow-ai/models/${slot}`, { name, apiUrl, apiKey, modelName, isActive }), onSuccess: () => queryClient.invalidateQueries({ queryKey: ['shadow-ai-models'] }), }); const deleteMutation = useMutation({ mutationFn: () => api.delete(`/admin/shadow-ai/models/${slot}`), onSuccess: () => { setName(''); setApiUrl(''); setApiKey(''); setModelName(''); setIsActive(true); queryClient.invalidateQueries({ queryKey: ['shadow-ai-models'] }); }, }); if (isLoading) return
; return ( Alternate {slot} {model ? ( {isActive ? 'Active' : 'Inactive'} ) : ( Not configured )} setName(e.target.value)} size="sm" /> setApiUrl(e.target.value)} size="sm" /> setApiKey(e.target.value)} size="sm" /> setModelName(e.target.value)} size="sm" /> setIsActive(e.currentTarget.checked)} /> {model && ( )} {saveMutation.isError && Failed to save} {saveMutation.isSuccess && Saved} ); } // ── Run Comparison Tab ── function RunComparisonTab() { const queryClient = useQueryClient(); const [tenantId, setTenantId] = useState(null); const [feature, setFeature] = useState(null); const [activeRunId, setActiveRunId] = useState(null); const { data: orgs } = useQuery({ queryKey: ['admin-orgs'], queryFn: () => api.get('/admin/organizations').then((r) => r.data), }); const triggerMutation = useMutation({ mutationFn: () => api.post('/admin/shadow-ai/runs', { tenantId, feature }), onSuccess: (res) => { setActiveRunId(res.data.runId); }, }); const { data: activeRun } = useQuery({ queryKey: ['shadow-ai-run', activeRunId], queryFn: () => api.get(`/admin/shadow-ai/runs/${activeRunId}`).then((r) => r.data), enabled: !!activeRunId, refetchInterval: (query) => { const run = query.state.data; return run?.status === 'running' ? 3000 : false; }, }); const orgOptions = (orgs || []) .filter((o) => o.status === 'active') .map((o) => ({ value: o.id, label: o.name })); const featureOptions = [ { value: 'operating_health', label: 'Operating Health Score' }, { value: 'reserve_health', label: 'Reserve Health Score' }, { value: 'investment_recommendations', label: 'Investment Recommendations' }, ]; return ( Run Shadow Comparison {triggerMutation.isError && ( }> Failed to start comparison. Ensure at least one alternate model is configured. )} {activeRun && ( {featureLabels[activeRun.feature] || activeRun.feature} {activeRun.status} {activeRun.tenant_name && ( Tenant: {activeRun.tenant_name} )} {activeRun.status === 'running' && (
Running models... This may take a few minutes. {(activeRun.results || []).map((r) => ( {roleLabels[r.model_role]}: {r.status} ))}
)} {activeRun.status !== 'running' && activeRun.results && ( )}
)}
); } // ── History Tab ── function HistoryTab() { const [selectedRunId, setSelectedRunId] = useState(null); const [tenantFilter, setTenantFilter] = useState(null); const [featureFilter, setFeatureFilter] = useState(null); const { data: orgs } = useQuery({ queryKey: ['admin-orgs'], queryFn: () => api.get('/admin/organizations').then((r) => r.data), }); const { data: historyData, isLoading } = useQuery({ queryKey: ['shadow-ai-runs', tenantFilter, featureFilter], queryFn: () => { const params = new URLSearchParams(); if (tenantFilter) params.set('tenantId', tenantFilter); if (featureFilter) params.set('feature', featureFilter); params.set('limit', '50'); return api.get(`/admin/shadow-ai/runs?${params}`).then((r) => r.data); }, }); const { data: selectedRun } = useQuery({ queryKey: ['shadow-ai-run', selectedRunId], queryFn: () => api.get(`/admin/shadow-ai/runs/${selectedRunId}`).then((r) => r.data), enabled: !!selectedRunId, }); const orgOptions = [ { value: '', label: 'All Tenants' }, ...(orgs || []).map((o) => ({ value: o.id, label: o.name })), ]; const featureOptions = [ { value: '', label: 'All Features' }, { value: 'operating_health', label: 'Operating Health' }, { value: 'reserve_health', label: 'Reserve Health' }, { value: 'investment_recommendations', label: 'Investment Recommendations' }, ]; const runs: ShadowRun[] = historyData?.runs || []; return ( setFeatureFilter(v || null)} clearable w={200} /> {isLoading ? (
) : runs.length === 0 ? ( No shadow runs found. ) : ( Date Tenant Feature Status Models Duration {runs.map((run) => { const duration = run.completed_at && run.started_at ? new Date(run.completed_at).getTime() - new Date(run.started_at).getTime() : null; return ( setSelectedRunId(run.id)} bg={selectedRunId === run.id ? 'var(--mantine-color-blue-light)' : undefined} > {formatDate(run.created_at)} {run.tenant_name || '-'} {featureLabels[run.feature] || run.feature} {run.status} {run.success_count || '0'}/{run.result_count || '0'} {formatDuration(duration)} ); })}
)} {selectedRun && selectedRun.results && ( {featureLabels[selectedRun.feature] || selectedRun.feature} {selectedRun.status} {selectedRun.tenant_name} | {formatDate(selectedRun.created_at)} )}
); } // ── Comparison Results Component ── function ComparisonResults({ results, feature }: { results: ShadowRunResult[]; feature: string }) { const isHealthScore = feature === 'operating_health' || feature === 'reserve_health'; // Collect all parsed values for diff highlighting const parsedValues = results .filter((r) => r.status === 'success' && r.parsed_response) .map((r) => r.parsed_response); return ( {results.map((result) => ( ))} ); } function ResultCard({ result, isHealthScore, allParsed, }: { result: ShadowRunResult; isHealthScore: boolean; allParsed: any[]; }) { const roleColor: Record = { production: 'green', alternate_a: 'blue', alternate_b: 'violet', }; return ( {roleLabels[result.model_role]} : result.status === 'error' ? : } > {result.status} {result.model_name} {result.response_time_ms && ( {formatDuration(result.response_time_ms)} )} {result.token_usage && ( Tokens: {result.token_usage.prompt_tokens || '?'} prompt / {result.token_usage.completion_tokens || '?'} completion )} {result.status === 'error' && ( }> {result.error_message || 'Unknown error'} )} {result.status === 'success' && result.parsed_response && ( isHealthScore ? : )} {result.status === 'success' && ( Raw JSON Response {JSON.stringify(result.parsed_response, null, 2)} )} ); } // ── Health Score Display ── function HealthScoreDisplay({ data, allParsed }: { data: any; allParsed: any[] }) { const score = data.score ?? data.raw_text; const label = data.label || ''; const summary = data.summary || ''; const factors = data.factors || []; const recommendations = data.recommendations || []; // Check if score differs from other models const scores = allParsed.map((p) => p.score).filter((s) => typeof s === 'number'); const scoreDiffers = scores.length > 1 && !scores.every((s) => s === scores[0]); const labelColor: Record = { Excellent: 'green', Good: 'teal', Fair: 'yellow', 'Needs Attention': 'orange', 'At Risk': 'red', Critical: 'red', }; return ( {typeof score === 'number' && ( {score} } /> )} {label && ( {label} )} {summary && {summary}} {factors.length > 0 && ( <> Factors {factors.map((f: any, i: number) => ( {f.impact} {f.name}: {f.detail} ))} )} {recommendations.length > 0 && ( <> Recommendations {recommendations.map((r: any, i: number) => ( {r.priority} {r.text} ))} )} ); } // ── Investment Display ── function InvestmentDisplay({ data, allParsed }: { data: any; allParsed: any[] }) { const recommendations = data.recommendations || []; const overall = data.overall_assessment || ''; const riskNotes = data.risk_notes || []; const recCounts = allParsed.map((p) => (p.recommendations || []).length); const countDiffers = recCounts.length > 1 && !recCounts.every((c) => c === recCounts[0]); const typeColors: Record = { cd_ladder: 'violet', new_investment: 'blue', reallocation: 'teal', maturity_action: 'orange', liquidity_warning: 'red', general: 'gray', }; return ( {overall && ( {overall} )} {recommendations.length > 0 && ( <> Recommendations {recommendations.length} {recommendations.map((rec: any, i: number) => ( {rec.type} {rec.priority} {rec.fund_type && {rec.fund_type}} {rec.title} {rec.summary} {rec.suggested_amount && ( Amount: ${rec.suggested_amount.toLocaleString()} {rec.suggested_rate ? ` | Rate: ${rec.suggested_rate}%` : ''} {rec.suggested_term ? ` | Term: ${rec.suggested_term}` : ''} )} ))} )} {riskNotes.length > 0 && ( <> Risk Notes {riskNotes.map((note: string, i: number) => ( {note} ))} )} ); } // ── Main Page ── export function AdminShadowAiPage() { return ( AI Benchmarking Compare AI model outputs side-by-side using real tenant data. Configure alternate models, run shadow comparisons, and review historical results. }> Model Configuration }> Run Comparison }> History ); }