feat: async AI calls, 10-min timeout, and failure messaging

- Make all AI endpoints (health scores + investment recommendations)
  fire-and-forget: POST returns immediately, frontend polls for results
- Extend AI API timeout from 2-5 min to 10 min for both services
- Add "last analysis failed — showing cached data" message to the
  Investment Recommendations panel (matches health score widgets)
- Add status/error_message columns to ai_recommendations table
- Remove nginx AI timeout overrides (no longer needed)
- Users can now navigate away during AI processing without interruption

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

frontend/src/pages/dashboard/DashboardPage.tsx

@@ -16,8 +16,8 @@ import {
   IconRefresh,
   IconInfoCircle,
 } from '@tabler/icons-react';
-import { useState } from 'react';
-import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
+import { useState, useCallback } from 'react';
+import { useQuery, useQueryClient } from '@tanstack/react-query';
 import { useAuthStore } from '../../stores/authStore';
 import api from '../../services/api';
 
@@ -313,9 +313,9 @@ export function DashboardPage() {
   const currentOrg = useAuthStore((s) => s.currentOrg);
   const queryClient = useQueryClient();
 
-  // Track whether last refresh attempt failed (per score type)
-  const [operatingFailed, setOperatingFailed] = useState(false);
-  const [reserveFailed, setReserveFailed] = useState(false);
+  // Track whether a refresh is in progress (per score type) for async polling
+  const [operatingRefreshing, setOperatingRefreshing] = useState(false);
+  const [reserveRefreshing, setReserveRefreshing] = useState(false);
 
   const { data, isLoading } = useQuery<DashboardData>({
     queryKey: ['dashboard'],
@@ -327,33 +327,66 @@ export function DashboardPage() {
     queryKey: ['health-scores'],
     queryFn: async () => { const { data } = await api.get('/health-scores/latest'); return data; },
     enabled: !!currentOrg,
+    // Poll every 3 seconds while a refresh is in progress
+    refetchInterval: (operatingRefreshing || reserveRefreshing) ? 3000 : false,
   });
 
-  // Separate mutations for each score type
-  const recalcOperatingMutation = useMutation({
-    mutationFn: () => api.post('/health-scores/calculate/operating'),
-    onSuccess: () => {
-      setOperatingFailed(false);
-      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
-    },
-    onError: () => {
-      setOperatingFailed(true);
-      // Still refresh to get whatever the backend saved (could be cached data)
-      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
-    },
-  });
+  // Async refresh handlers — trigger the backend and poll for results
+  const handleRefreshOperating = useCallback(async () => {
+    const prevId = healthScores?.operating?.id;
+    setOperatingRefreshing(true);
+    try {
+      await api.post('/health-scores/calculate/operating');
+    } catch {
+      // Trigger failed at network level — polling will pick up any backend-saved error
+    }
+    // Start polling — watch for the health score to change (new id or updated timestamp)
+    const pollUntilDone = () => {
+      const checkInterval = setInterval(async () => {
+        try {
+          const { data: latest } = await api.get('/health-scores/latest');
+          const newScore = latest?.operating;
+          if (newScore && newScore.id !== prevId) {
+            setOperatingRefreshing(false);
+            queryClient.setQueryData(['health-scores'], latest);
+            clearInterval(checkInterval);
+          }
+        } catch {
+          // Keep polling
+        }
+      }, 3000);
+      // Safety timeout — stop polling after 11 minutes
+      setTimeout(() => { clearInterval(checkInterval); setOperatingRefreshing(false); }, 660000);
+    };
+    pollUntilDone();
+  }, [healthScores?.operating?.id, queryClient]);
 
-  const recalcReserveMutation = useMutation({
-    mutationFn: () => api.post('/health-scores/calculate/reserve'),
-    onSuccess: () => {
-      setReserveFailed(false);
-      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
-    },
-    onError: () => {
-      setReserveFailed(true);
-      queryClient.invalidateQueries({ queryKey: ['health-scores'] });
-    },
-  });
+  const handleRefreshReserve = useCallback(async () => {
+    const prevId = healthScores?.reserve?.id;
+    setReserveRefreshing(true);
+    try {
+      await api.post('/health-scores/calculate/reserve');
+    } catch {
+      // Trigger failed at network level
+    }
+    const pollUntilDone = () => {
+      const checkInterval = setInterval(async () => {
+        try {
+          const { data: latest } = await api.get('/health-scores/latest');
+          const newScore = latest?.reserve;
+          if (newScore && newScore.id !== prevId) {
+            setReserveRefreshing(false);
+            queryClient.setQueryData(['health-scores'], latest);
+            clearInterval(checkInterval);
+          }
+        } catch {
+          // Keep polling
+        }
+      }, 3000);
+      setTimeout(() => { clearInterval(checkInterval); setReserveRefreshing(false); }, 660000);
+    };
+    pollUntilDone();
+  }, [healthScores?.reserve?.id, queryClient]);
 
   const fmt = (v: string | number) =>
     parseFloat(String(v || '0')).toLocaleString('en-US', { style: 'currency', currency: 'USD' });
@@ -391,9 +424,9 @@ export function DashboardPage() {
                   <IconHeartbeat size={20} />
                 </ThemeIcon>
               }
-              isRefreshing={recalcOperatingMutation.isPending}
-              onRefresh={() => recalcOperatingMutation.mutate()}
-              lastFailed={operatingFailed || !!healthScores?.operating_last_failed}
+              isRefreshing={operatingRefreshing}
+              onRefresh={handleRefreshOperating}
+              lastFailed={!!healthScores?.operating_last_failed}
             />
             <HealthScoreCard
               score={healthScores?.reserve || null}
@@ -403,9 +436,9 @@ export function DashboardPage() {
                   <IconHeartbeat size={20} />
                 </ThemeIcon>
               }
-              isRefreshing={recalcReserveMutation.isPending}
-              onRefresh={() => recalcReserveMutation.mutate()}
-              lastFailed={reserveFailed || !!healthScores?.reserve_last_failed}
+              isRefreshing={reserveRefreshing}
+              onRefresh={handleRefreshReserve}
+              lastFailed={!!healthScores?.reserve_last_failed}
             />
           </SimpleGrid>