feat: add shadow AI benchmarking for admin model comparison
Add a new admin-only feature that allows the platform owner to benchmark the production AI model against up to 2 alternate models (any OpenAI-compatible API) using real tenant data, without impacting users. Backend: - Shared AI caller utility (ai-caller.ts) for OpenAI-compatible endpoints - Shadow AI module with service, controller, and 3 entities - 6 admin API endpoints for model config CRUD, run trigger, and history - Auto-creates shadow_ai_models, shadow_runs, shadow_run_results tables - Exposes health-scores and investment-planning prompt builders for reuse Frontend: - New admin page at /admin/shadow-ai with 3 tabs: - Model Configuration (production + 2 alternate slots) - Run Comparison (tenant select, feature select, side-by-side results) - History (filterable run log with detail drill-down) - Full side-by-side output display with diff highlighting - Sidebar navigation link for AI Benchmarking Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
import {
|
||||
Entity,
|
||||
PrimaryGeneratedColumn,
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
ManyToOne,
|
||||
JoinColumn,
|
||||
} from 'typeorm';
|
||||
import { ShadowRun } from './shadow-run.entity';
|
||||
|
||||
@Entity({ schema: 'shared', name: 'shadow_run_results' })
|
||||
export class ShadowRunResult {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id: string;
|
||||
|
||||
@Column({ name: 'run_id', type: 'uuid' })
|
||||
runId: string;
|
||||
|
||||
@Column({ name: 'model_role', type: 'varchar', length: 20 })
|
||||
modelRole: string;
|
||||
|
||||
@Column({ name: 'model_name', type: 'varchar', length: 200 })
|
||||
modelName: string;
|
||||
|
||||
@Column({ name: 'api_url', type: 'varchar', length: 500 })
|
||||
apiUrl: string;
|
||||
|
||||
@Column({ name: 'raw_response', type: 'text', nullable: true })
|
||||
rawResponse: string;
|
||||
|
||||
@Column({ name: 'parsed_response', type: 'jsonb', nullable: true })
|
||||
parsedResponse: any;
|
||||
|
||||
@Column({ name: 'response_time_ms', type: 'integer', nullable: true })
|
||||
responseTimeMs: number;
|
||||
|
||||
@Column({ name: 'token_usage', type: 'jsonb', nullable: true })
|
||||
tokenUsage: any;
|
||||
|
||||
@Column({ type: 'varchar', length: 20, default: 'pending' })
|
||||
status: string;
|
||||
|
||||
@Column({ name: 'error_message', type: 'text', nullable: true })
|
||||
errorMessage: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at', type: 'timestamptz' })
|
||||
createdAt: Date;
|
||||
|
||||
@ManyToOne(() => ShadowRun, (run) => run.results, { onDelete: 'CASCADE' })
|
||||
@JoinColumn({ name: 'run_id' })
|
||||
run: ShadowRun;
|
||||
}
|
||||
Reference in New Issue
Block a user