feat: reliability enhancements for AI services and capital planning

1. Health Scores — separate operating/reserve refresh - Added POST /health-scores/calculate/operating and /calculate/reserve - Each health card now has its own Refresh button - On failure, shows cached (last good) data with "last analysis failed" watermark instead of blank "Error calculating score" - Backend getLatestScores returns latest complete score + failure flag 2. Investment Planning — increased AI timeout to 5 minutes - Backend callAI timeout: 180s → 300s - Frontend axios timeout: set explicitly to 300s (was browser default) - Host nginx proxy_read_timeout: 180s → 300s - Loading message updated to reflect longer wait times 3. Capital Planning — Unscheduled column moved to rightmost position - Kanban column order: current year → future → unscheduled (was leftmost) - Puts immediate/near-term projects front and center Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-03 12:02:30 -05:00
parent 467fdd2a6c
commit 337b6061b2
7 changed files with 175 additions and 47 deletions
--- a/backend/src/modules/health-scores/health-scores.controller.ts
+++ b/backend/src/modules/health-scores/health-scores.controller.ts
@@ -19,7 +19,7 @@ export class HealthScoresController {
  }

  @Post('calculate')
-  @ApiOperation({ summary: 'Trigger health score recalculation for current tenant' })
+  @ApiOperation({ summary: 'Trigger both health score recalculations (used by scheduler)' })
  @AllowViewer()
  async calculate(@Req() req: any) {
    const schema = req.user?.orgSchema;
@@ -29,4 +29,22 @@ export class HealthScoresController {
    ]);
    return { operating, reserve };
  }
+
+  @Post('calculate/operating')
+  @ApiOperation({ summary: 'Recalculate operating fund health score only' })
+  @AllowViewer()
+  async calculateOperating(@Req() req: any) {
+    const schema = req.user?.orgSchema;
+    const operating = await this.service.calculateScore(schema, 'operating');
+    return { operating };
+  }
+
+  @Post('calculate/reserve')
+  @ApiOperation({ summary: 'Recalculate reserve fund health score only' })
+  @AllowViewer()
+  async calculateReserve(@Req() req: any) {
+    const schema = req.user?.orgSchema;
+    const reserve = await this.service.calculateScore(schema, 'reserve');
+    return { reserve };
+  }
 }
--- a/backend/src/modules/health-scores/health-scores.service.ts
+++ b/backend/src/modules/health-scores/health-scores.service.ts
@@ -47,23 +47,49 @@ export class HealthScoresService {

  // ── Public API ──

-  async getLatestScores(schema: string): Promise<{ operating: HealthScore | null; reserve: HealthScore | null }> {
+  async getLatestScores(schema: string): Promise<{
+    operating: HealthScore | null;
+    reserve: HealthScore | null;
+    operating_last_failed: boolean;
+    reserve_last_failed: boolean;
+  }> {
    const qr = this.dataSource.createQueryRunner();
    try {
      await qr.connect();
      await qr.query(`SET search_path TO "${schema}"`);

-      const operating = await qr.query(
-        `SELECT * FROM health_scores WHERE score_type = 'operating' ORDER BY calculated_at DESC LIMIT 1`,
-      );
-      const reserve = await qr.query(
-        `SELECT * FROM health_scores WHERE score_type = 'reserve' ORDER BY calculated_at DESC LIMIT 1`,
-      );
+      // For each score type, return the latest *successful* score for display,
+      // and flag whether the most recent attempt (any status) was an error.
+      const result = { operating: null as HealthScore | null, reserve: null as HealthScore | null, operating_last_failed: false, reserve_last_failed: false };

-      return {
-        operating: operating[0] || null,
-        reserve: reserve[0] || null,
-      };
+      for (const scoreType of ['operating', 'reserve'] as const) {
+        // Most recent row (any status)
+        const latest = await qr.query(
+          `SELECT * FROM health_scores WHERE score_type = $1 ORDER BY calculated_at DESC LIMIT 1`,
+          [scoreType],
+        );
+        const latestRow = latest[0] || null;
+
+        if (!latestRow) {
+          // No scores at all
+          continue;
+        }
+
+        if (latestRow.status === 'error') {
+          // Most recent attempt failed — return the latest *complete* score instead
+          const lastGood = await qr.query(
+            `SELECT * FROM health_scores WHERE score_type = $1 AND status = 'complete' ORDER BY calculated_at DESC LIMIT 1`,
+            [scoreType],
+          );
+          result[scoreType] = lastGood[0] || latestRow; // fall back to error row if no good score exists
+          result[`${scoreType}_last_failed`] = true;
+        } else {
+          result[scoreType] = latestRow;
+          result[`${scoreType}_last_failed`] = false;
+        }
+      }
+
+      return result;
    } finally {
      await qr.release();
    }
--- a/backend/src/modules/investment-planning/investment-planning.service.ts
+++ b/backend/src/modules/investment-planning/investment-planning.service.ts
@@ -873,7 +873,7 @@ Based on this complete financial picture INCLUDING the 12-month cash flow foreca
            'Content-Type': 'application/json',
            'Content-Length': Buffer.byteLength(bodyString, 'utf-8'),
          },
-          timeout: 180000, // 3 minute timeout
+          timeout: 300000, // 5 minute timeout
        };

        const req = https.request(options, (res) => {
@@ -887,7 +887,7 @@ Based on this complete financial picture INCLUDING the 12-month cash flow foreca
        req.on('error', (err) => reject(err));
        req.on('timeout', () => {
          req.destroy();
-          reject(new Error(`Request timed out after 180s`));
+          reject(new Error(`Request timed out after 300s`));
        });

        req.write(bodyString);
--- a/frontend/src/pages/capital-projects/CapitalProjectsPage.tsx
+++ b/frontend/src/pages/capital-projects/CapitalProjectsPage.tsx
@@ -430,13 +430,13 @@ export function CapitalProjectsPage() {
  // Merge base years with any extra years from projects (excluding FUTURE_YEAR for now)
  const regularYears = [...new Set([...baseYears, ...projectYears.filter((y) => y !== FUTURE_YEAR)])].sort();
  const years = [
-    ...(hasUnscheduledProjects ? [UNSCHEDULED] : []),
    ...regularYears,
    ...(hasFutureProjects ? [FUTURE_YEAR] : []),
+    ...(hasUnscheduledProjects ? [UNSCHEDULED] : []),
  ];

-  // Kanban columns: Unscheduled + current..current+4 + Future
-  const kanbanYears = [UNSCHEDULED, ...baseYears, FUTURE_YEAR];
+  // Kanban columns: current..current+4 + Future + Unscheduled (rightmost)
+  const kanbanYears = [...baseYears, FUTURE_YEAR, UNSCHEDULED];

  // ---- Loading state ----

--- a/frontend/src/pages/dashboard/DashboardPage.tsx
+++ b/frontend/src/pages/dashboard/DashboardPage.tsx
@@ -16,6 +16,7 @@ import {
  IconRefresh,
  IconInfoCircle,
 } from '@tabler/icons-react';
+import { useState } from 'react';
 import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
 import { useAuthStore } from '../../stores/authStore';
 import api from '../../services/api';
@@ -39,6 +40,8 @@ interface HealthScore {
 interface HealthScoresData {
  operating: HealthScore | null;
  reserve: HealthScore | null;
+  operating_last_failed?: boolean;
+  reserve_last_failed?: boolean;
 }

 function getScoreColor(score: number): string {
@@ -55,14 +58,37 @@ function TrajectoryIcon({ trajectory }: { trajectory: string | null }) {
  return null;
 }

-function HealthScoreCard({ score, title, icon }: { score: HealthScore | null; title: string; icon: React.ReactNode }) {
+function HealthScoreCard({
+  score,
+  title,
+  icon,
+  isRefreshing,
+  onRefresh,
+  lastFailed,
+}: {
+  score: HealthScore | null;
+  title: string;
+  icon: React.ReactNode;
+  isRefreshing?: boolean;
+  onRefresh?: () => void;
+  lastFailed?: boolean;
+}) {
+  // No score at all yet
  if (!score) {
    return (
      <Card withBorder padding="lg" radius="md">
        <Group justify="space-between" mb="xs">
          <Text size="xs" c="dimmed" tt="uppercase" fw={700}>{title} Health</Text>
+          <Group gap={6}>
+            {onRefresh && (
+              <Tooltip label={`Recalculate ${title.toLowerCase()} score`}>
+                <Button variant="subtle" size="compact-xs" leftSection={<IconRefresh size={14} />}
+                  loading={isRefreshing} onClick={onRefresh}>Refresh</Button>
+              </Tooltip>
+            )}
            {icon}
          </Group>
+        </Group>
        <Center h={100}>
          <Text c="dimmed" size="sm">No health score yet</Text>
        </Center>
@@ -70,6 +96,7 @@ function HealthScoreCard({ score, title, icon }: { score: HealthScore | null; ti
    );
  }

+  // Pending — missing data, can't calculate
  if (score.status === 'pending') {
    const missingItems = Array.isArray(score.missing_data) ? score.missing_data :
      (typeof score.missing_data === 'string' ? JSON.parse(score.missing_data) : []);
@@ -77,8 +104,16 @@ function HealthScoreCard({ score, title, icon }: { score: HealthScore | null; ti
      <Card withBorder padding="lg" radius="md">
        <Group justify="space-between" mb="xs">
          <Text size="xs" c="dimmed" tt="uppercase" fw={700}>{title} Health</Text>
+          <Group gap={6}>
+            {onRefresh && (
+              <Tooltip label={`Recalculate ${title.toLowerCase()} score`}>
+                <Button variant="subtle" size="compact-xs" leftSection={<IconRefresh size={14} />}
+                  loading={isRefreshing} onClick={onRefresh}>Refresh</Button>
+              </Tooltip>
+            )}
            {icon}
          </Group>
+        </Group>
        <Center>
          <Stack align="center" gap="xs">
            <Badge color="gray" variant="light" size="lg">Pending</Badge>
@@ -92,20 +127,38 @@ function HealthScoreCard({ score, title, icon }: { score: HealthScore | null; ti
    );
  }

-  if (score.status === 'error') {
+  // For error status, we still render the score data (cached from the previous
+  // successful run) rather than blanking the card with "Error calculating score".
+  // A small watermark under the timestamp tells the user it's stale.
+  const showAsError = score.status === 'error' && score.score === 0 && !score.summary;
+
+  // Pure error with no cached data to fall back on
+  if (showAsError) {
    return (
      <Card withBorder padding="lg" radius="md">
        <Group justify="space-between" mb="xs">
          <Text size="xs" c="dimmed" tt="uppercase" fw={700}>{title} Health</Text>
+          <Group gap={6}>
+            {onRefresh && (
+              <Tooltip label={`Retry ${title.toLowerCase()} score`}>
+                <Button variant="subtle" size="compact-xs" leftSection={<IconRefresh size={14} />}
+                  loading={isRefreshing} onClick={onRefresh}>Retry</Button>
+              </Tooltip>
+            )}
            {icon}
          </Group>
+        </Group>
        <Center h={100}>
+          <Stack align="center" gap={4}>
            <Badge color="red" variant="light">Error calculating score</Badge>
+            <Text size="xs" c="dimmed">Click Retry to try again</Text>
+          </Stack>
        </Center>
      </Card>
    );
  }

+  // Normal display — works for both 'complete' and 'error' (with cached data)
  const color = getScoreColor(score.score);
  const factors = Array.isArray(score.factors) ? score.factors :
    (typeof score.factors === 'string' ? JSON.parse(score.factors) : []);
@@ -116,8 +169,16 @@ function HealthScoreCard({ score, title, icon }: { score: HealthScore | null; ti
    <Card withBorder padding="lg" radius="md">
      <Group justify="space-between" mb="xs">
        <Text size="xs" c="dimmed" tt="uppercase" fw={700}>{title} Health</Text>
+        <Group gap={6}>
+          {onRefresh && (
+            <Tooltip label={`Recalculate ${title.toLowerCase()} score`}>
+              <Button variant="subtle" size="compact-xs" leftSection={<IconRefresh size={14} />}
+                loading={isRefreshing} onClick={onRefresh}>Refresh</Button>
+            </Tooltip>
+          )}
          {icon}
        </Group>
+      </Group>
      <Group align="flex-start" gap="lg">
        <RingProgress
          size={120}
@@ -215,9 +276,16 @@ function HealthScoreCard({ score, title, icon }: { score: HealthScore | null; ti
        </Stack>
      </Group>
      {score.calculated_at && (
-        <Text size="10px" c="dimmed" ta="right" mt={6} style={{ opacity: 0.7 }}>
+        <Stack gap={0} mt={6} align="flex-end">
+          <Text size="10px" c="dimmed" style={{ opacity: 0.7 }}>
            Last updated {new Date(score.calculated_at).toLocaleDateString()} at {new Date(score.calculated_at).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })}
          </Text>
+          {lastFailed && (
+            <Text size="10px" c="orange" fw={500} style={{ opacity: 0.85 }}>
+              last analysis failed — showing cached data
+            </Text>
+          )}
+        </Stack>
      )}
    </Card>
  );
@@ -245,6 +313,10 @@ export function DashboardPage() {
  const currentOrg = useAuthStore((s) => s.currentOrg);
  const queryClient = useQueryClient();

+  // Track whether last refresh attempt failed (per score type)
+  const [operatingFailed, setOperatingFailed] = useState(false);
+  const [reserveFailed, setReserveFailed] = useState(false);
+
  const { data, isLoading } = useQuery<DashboardData>({
    queryKey: ['dashboard'],
    queryFn: async () => { const { data } = await api.get('/reports/dashboard'); return data; },
@@ -257,9 +329,28 @@ export function DashboardPage() {
    enabled: !!currentOrg,
  });

-  const recalcMutation = useMutation({
-    mutationFn: () => api.post('/health-scores/calculate'),
+  // Separate mutations for each score type
+  const recalcOperatingMutation = useMutation({
+    mutationFn: () => api.post('/health-scores/calculate/operating'),
    onSuccess: () => {
+      setOperatingFailed(false);
+      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
+    },
+    onError: () => {
+      setOperatingFailed(true);
+      // Still refresh to get whatever the backend saved (could be cached data)
+      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
+    },
+  });
+
+  const recalcReserveMutation = useMutation({
+    mutationFn: () => api.post('/health-scores/calculate/reserve'),
+    onSuccess: () => {
+      setReserveFailed(false);
+      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
+    },
+    onError: () => {
+      setReserveFailed(true);
      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
    },
  });
@@ -290,20 +381,7 @@ export function DashboardPage() {
        <Center h={200}><Loader /></Center>
      ) : (
        <>
-          <Group justify="space-between" align="center">
          <Text size="sm" fw={600} c="dimmed">AI Health Scores</Text>
-            <Tooltip label="Recalculate health scores now">
-              <Button
-                variant="subtle"
-                size="compact-xs"
-                leftSection={<IconRefresh size={14} />}
-                loading={recalcMutation.isPending}
-                onClick={() => recalcMutation.mutate()}
-              >
-                Refresh
-              </Button>
-            </Tooltip>
-          </Group>
          <SimpleGrid cols={{ base: 1, md: 2 }}>
            <HealthScoreCard
              score={healthScores?.operating || null}
@@ -313,6 +391,9 @@ export function DashboardPage() {
                  <IconHeartbeat size={20} />
                </ThemeIcon>
              }
+              isRefreshing={recalcOperatingMutation.isPending}
+              onRefresh={() => recalcOperatingMutation.mutate()}
+              lastFailed={operatingFailed || !!healthScores?.operating_last_failed}
            />
            <HealthScoreCard
              score={healthScores?.reserve || null}
@@ -322,6 +403,9 @@ export function DashboardPage() {
                  <IconHeartbeat size={20} />
                </ThemeIcon>
              }
+              isRefreshing={recalcReserveMutation.isPending}
+              onRefresh={() => recalcReserveMutation.mutate()}
+              lastFailed={reserveFailed || !!healthScores?.reserve_last_failed}
            />
          </SimpleGrid>

--- a/frontend/src/pages/investment-planning/InvestmentPlanningPage.tsx
+++ b/frontend/src/pages/investment-planning/InvestmentPlanningPage.tsx
@@ -373,7 +373,7 @@ export function InvestmentPlanningPage() {
  // AI recommendation (on-demand)
  const aiMutation = useMutation({
    mutationFn: async () => {
-      const { data } = await api.post('/investment-planning/recommendations');
+      const { data } = await api.post('/investment-planning/recommendations', {}, { timeout: 300000 });
      return data as AIResponse;
    },
    onSuccess: (data) => {
@@ -663,7 +663,7 @@ export function InvestmentPlanningPage() {
                Analyzing your financial data and market rates...
              </Text>
              <Text c="dimmed" size="xs">
-                This may take up to 30 seconds
+                This may take a few minutes for complex tenant data
              </Text>
            </Stack>
          </Center>
--- a/nginx/host-production.conf
+++ b/nginx/host-production.conf
@@ -74,10 +74,10 @@ server {
        proxy_send_timeout 15s;
    }

-    # AI endpoints — longer timeouts (LLM calls can take 30-120s)
+    # AI endpoints — longer timeouts (LLM calls can take minutes)
    location /api/investment-planning/recommendations {
        proxy_pass http://127.0.0.1:3000;
-        proxy_read_timeout 180s;
+        proxy_read_timeout 300s;
        proxy_connect_timeout 10s;
        proxy_send_timeout 30s;
    }