Add a new admin-only feature that allows the platform owner to benchmark the production AI model against up to 2 alternate models (any OpenAI-compatible API) using real tenant data, without impacting users. Backend: - Shared AI caller utility (ai-caller.ts) for OpenAI-compatible endpoints - Shadow AI module with service, controller, and 3 entities - 6 admin API endpoints for model config CRUD, run trigger, and history - Auto-creates shadow_ai_models, shadow_runs, shadow_run_results tables - Exposes health-scores and investment-planning prompt builders for reuse Frontend: - New admin page at /admin/shadow-ai with 3 tabs: - Model Configuration (production + 2 alternate slots) - Run Comparison (tenant select, feature select, side-by-side results) - History (filterable run log with detail drill-down) - Full side-by-side output display with diff highlighting - Sidebar navigation link for AI Benchmarking Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
119 lines
3.3 KiB
TypeScript
119 lines
3.3 KiB
TypeScript
import {
|
|
Controller,
|
|
Get,
|
|
Put,
|
|
Post,
|
|
Delete,
|
|
Body,
|
|
Param,
|
|
Query,
|
|
UseGuards,
|
|
Req,
|
|
ForbiddenException,
|
|
BadRequestException,
|
|
NotFoundException,
|
|
} from '@nestjs/common';
|
|
import { ApiTags, ApiBearerAuth } from '@nestjs/swagger';
|
|
import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard';
|
|
import { UsersService } from '../users/users.service';
|
|
import { ShadowAiService } from './shadow-ai.service';
|
|
|
|
@ApiTags('admin/shadow-ai')
|
|
@Controller('admin/shadow-ai')
|
|
@ApiBearerAuth()
|
|
@UseGuards(JwtAuthGuard)
|
|
export class ShadowAiController {
|
|
constructor(
|
|
private shadowAiService: ShadowAiService,
|
|
private usersService: UsersService,
|
|
) {}
|
|
|
|
private async requireSuperadmin(req: any) {
|
|
const user = await this.usersService.findById(req.user.userId || req.user.sub);
|
|
if (!user?.isSuperadmin) {
|
|
throw new ForbiddenException('Superadmin access required');
|
|
}
|
|
return user;
|
|
}
|
|
|
|
// ── Model Configuration ──
|
|
|
|
@Get('models')
|
|
async getModels(@Req() req: any) {
|
|
await this.requireSuperadmin(req);
|
|
return this.shadowAiService.getModels();
|
|
}
|
|
|
|
@Put('models/:slot')
|
|
async upsertModel(
|
|
@Req() req: any,
|
|
@Param('slot') slot: string,
|
|
@Body() body: { name: string; apiUrl: string; apiKey: string; modelName: string; isActive?: boolean },
|
|
) {
|
|
await this.requireSuperadmin(req);
|
|
if (!['A', 'B'].includes(slot)) {
|
|
throw new BadRequestException('Slot must be A or B');
|
|
}
|
|
if (!body.name || !body.apiUrl || !body.apiKey || !body.modelName) {
|
|
throw new BadRequestException('name, apiUrl, apiKey, and modelName are required');
|
|
}
|
|
return this.shadowAiService.upsertModel(slot, body);
|
|
}
|
|
|
|
@Delete('models/:slot')
|
|
async deleteModel(@Req() req: any, @Param('slot') slot: string) {
|
|
await this.requireSuperadmin(req);
|
|
if (!['A', 'B'].includes(slot)) {
|
|
throw new BadRequestException('Slot must be A or B');
|
|
}
|
|
return this.shadowAiService.deleteModel(slot);
|
|
}
|
|
|
|
// ── Shadow Runs ──
|
|
|
|
@Post('runs')
|
|
async triggerRun(
|
|
@Req() req: any,
|
|
@Body() body: { tenantId: string; feature: string },
|
|
) {
|
|
const user = await this.requireSuperadmin(req);
|
|
const validFeatures = ['operating_health', 'reserve_health', 'investment_recommendations'];
|
|
if (!validFeatures.includes(body.feature)) {
|
|
throw new BadRequestException(`Feature must be one of: ${validFeatures.join(', ')}`);
|
|
}
|
|
if (!body.tenantId) {
|
|
throw new BadRequestException('tenantId is required');
|
|
}
|
|
return this.shadowAiService.triggerRun(
|
|
body.tenantId,
|
|
body.feature as any,
|
|
user.id,
|
|
);
|
|
}
|
|
|
|
@Get('runs')
|
|
async getRunHistory(
|
|
@Req() req: any,
|
|
@Query('page') page?: string,
|
|
@Query('limit') limit?: string,
|
|
@Query('tenantId') tenantId?: string,
|
|
@Query('feature') feature?: string,
|
|
) {
|
|
await this.requireSuperadmin(req);
|
|
return this.shadowAiService.getRunHistory({
|
|
page: page ? parseInt(page) : undefined,
|
|
limit: limit ? parseInt(limit) : undefined,
|
|
tenantId,
|
|
feature,
|
|
});
|
|
}
|
|
|
|
@Get('runs/:id')
|
|
async getRunDetail(@Req() req: any, @Param('id') id: string) {
|
|
await this.requireSuperadmin(req);
|
|
const detail = await this.shadowAiService.getRunDetail(id);
|
|
if (!detail) throw new NotFoundException('Shadow run not found');
|
|
return detail;
|
|
}
|
|
}
|