feat: add shadow AI benchmarking for admin model comparison
Add a new admin-only feature that allows the platform owner to benchmark the production AI model against up to 2 alternate models (any OpenAI-compatible API) using real tenant data, without impacting users. Backend: - Shared AI caller utility (ai-caller.ts) for OpenAI-compatible endpoints - Shadow AI module with service, controller, and 3 entities - 6 admin API endpoints for model config CRUD, run trigger, and history - Auto-creates shadow_ai_models, shadow_runs, shadow_run_results tables - Exposes health-scores and investment-planning prompt builders for reuse Frontend: - New admin page at /admin/shadow-ai with 3 tabs: - Model Configuration (production + 2 alternate slots) - Run Comparison (tenant select, feature select, side-by-side results) - History (filterable run log with detail drill-down) - Full side-by-side output display with diff highlighting - Sidebar navigation link for AI Benchmarking Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
} from 'typeorm';
|
||||
|
||||
@Entity({ schema: 'shared', name: 'shadow_ai_models' })
|
||||
export class ShadowAiModel {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ type: 'varchar', length: 10, unique: true })
|
||||
slot: string;
|
||||
|
||||
@Column({ type: 'varchar', length: 100 })
|
||||
name: string;
|
||||
|
||||
@Column({ name: 'api_url', type: 'varchar', length: 500 })
|
||||
apiUrl: string;
|
||||
|
||||
@Column({ name: 'api_key', type: 'varchar', length: 500 })
|
||||
apiKey: string;
|
||||
|
||||
@Column({ name: 'model_name', type: 'varchar', length: 200 })
|
||||
modelName: string;
|
||||
|
||||
@Column({ name: 'is_active', type: 'boolean', default: true })
|
||||
isActive: boolean;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at', type: 'timestamptz' })
|
||||
createdAt: Date;
|
||||
|
||||
@UpdateDateColumn({ name: 'updated_at', type: 'timestamptz' })
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { ShadowRun } from './shadow-run.entity';
|
||||
|
||||
@Entity({ schema: 'shared', name: 'shadow_run_results' })
|
||||
export class ShadowRunResult {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'run_id', type: 'uuid' })
|
||||
runId: string;
|
||||
|
||||
@Column({ name: 'model_role', type: 'varchar', length: 20 })
|
||||
modelRole: string;
|
||||
|
||||
@Column({ name: 'model_name', type: 'varchar', length: 200 })
|
||||
modelName: string;
|
||||
|
||||
@Column({ name: 'api_url', type: 'varchar', length: 500 })
|
||||
apiUrl: string;
|
||||
|
||||
@Column({ name: 'raw_response', type: 'text', nullable: true })
|
||||
rawResponse: string;
|
||||
|
||||
@Column({ name: 'parsed_response', type: 'jsonb', nullable: true })
|
||||
parsedResponse: any;
|
||||
|
||||
@Column({ name: 'response_time_ms', type: 'integer', nullable: true })
|
||||
responseTimeMs: number;
|
||||
|
||||
@Column({ name: 'token_usage', type: 'jsonb', nullable: true })
|
||||
tokenUsage: any;
|
||||
|
||||
@Column({ type: 'varchar', length: 20, default: 'pending' })
|
||||
status: string;
|
||||
|
||||
@Column({ name: 'error_message', type: 'text', nullable: true })
|
||||
errorMessage: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at', type: 'timestamptz' })
|
||||
createdAt: Date;
|
||||
|
||||
@ManyToOne(() => ShadowRun, (run) => run.results, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'run_id' })
|
||||
run: ShadowRun;
|
||||
}
|
||||
44
backend/src/modules/shadow-ai/entities/shadow-run.entity.ts
Normal file
44
backend/src/modules/shadow-ai/entities/shadow-run.entity.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
OneToMany,
|
||||
} from 'typeorm';
|
||||
import { ShadowRunResult } from './shadow-run-result.entity';
|
||||
|
||||
@Entity({ schema: 'shared', name: 'shadow_runs' })
|
||||
export class ShadowRun {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'tenant_id', type: 'uuid' })
|
||||
tenantId: string;
|
||||
|
||||
@Column({ type: 'varchar', length: 30 })
|
||||
feature: string;
|
||||
|
||||
@Column({ type: 'varchar', length: 20, default: 'running' })
|
||||
status: string;
|
||||
|
||||
@Column({ name: 'triggered_by', type: 'uuid', nullable: true })
|
||||
triggeredBy: string;
|
||||
|
||||
@Column({ name: 'prompt_messages', type: 'jsonb' })
|
||||
promptMessages: any;
|
||||
|
||||
@Column({ name: 'started_at', type: 'timestamptz', default: () => 'NOW()' })
|
||||
startedAt: Date;
|
||||
|
||||
@Column({ name: 'completed_at', type: 'timestamptz', nullable: true })
|
||||
completedAt: Date;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at', type: 'timestamptz' })
|
||||
createdAt: Date;
|
||||
|
||||
@OneToMany(() => ShadowRunResult, (result) => result.run, { eager: true })
|
||||
results: ShadowRunResult[];
|
||||
|
||||
// Virtual field populated via JOIN
|
||||
tenantName?: string;
|
||||
}
|
||||
Reference in New Issue
Block a user