04e15037f2
Self-hosted OpenAI-compatible proxy that aggregates the free tiers of fourteen LLM providers — Google, Groq, Cerebras, SambaNova, NVIDIA, Mistral, OpenRouter, GitHub Models, Hugging Face, Cohere, Cloudflare, Zhipu, Moonshot, MiniMax — behind a single /v1/chat/completions endpoint. Server: - Express + SQLite, per-provider adapters with streaming and non-streaming support, automatic fallover on 429/5xx, per-key RPM/RPD/TPM/TPD tracking, sticky sessions for multi-turn, AES-256-GCM encrypted key storage, unified bearer-token auth, periodic health checks. Client: - React + Vite + shadcn/ui admin dashboard: keys, fallback chain (drag to reorder, color-coded per-provider monthly token budget), playground, analytics with per-provider breakdowns. Tooling: - GitHub Actions CI (server tests + client build), MIT license, README with provider-by-provider ToS review. For personal experimentation, not production.
170 lines
3.2 KiB
TypeScript
170 lines
3.2 KiB
TypeScript
// ---- Platform & Model Types ----
|
|
|
|
export type Platform =
|
|
| 'google'
|
|
| 'groq'
|
|
| 'cerebras'
|
|
| 'sambanova'
|
|
| 'nvidia'
|
|
| 'mistral'
|
|
| 'openrouter'
|
|
| 'github'
|
|
| 'huggingface'
|
|
| 'cohere'
|
|
| 'cloudflare'
|
|
| 'zhipu'
|
|
| 'moonshot'
|
|
| 'minimax';
|
|
|
|
export interface Model {
|
|
id: number;
|
|
platform: Platform;
|
|
modelId: string;
|
|
displayName: string;
|
|
intelligenceRank: number;
|
|
speedRank: number;
|
|
sizeLabel: string;
|
|
rpmLimit: number | null;
|
|
rpdLimit: number | null;
|
|
tpmLimit: number | null;
|
|
tpdLimit: number | null;
|
|
monthlyTokenBudget: string;
|
|
contextWindow: number | null;
|
|
enabled: boolean;
|
|
}
|
|
|
|
export type KeyStatus = 'healthy' | 'rate_limited' | 'invalid' | 'error' | 'unknown';
|
|
|
|
export interface ApiKey {
|
|
id: number;
|
|
platform: Platform;
|
|
label: string;
|
|
maskedKey: string;
|
|
status: KeyStatus;
|
|
enabled: boolean;
|
|
createdAt: string;
|
|
lastCheckedAt: string | null;
|
|
}
|
|
|
|
export interface ApiKeyCreate {
|
|
platform: Platform;
|
|
key: string;
|
|
label?: string;
|
|
}
|
|
|
|
// ---- Fallback Config ----
|
|
|
|
export interface FallbackEntry {
|
|
modelId: number;
|
|
platform: Platform;
|
|
displayName: string;
|
|
intelligenceRank: number;
|
|
speedRank: number;
|
|
priority: number;
|
|
enabled: boolean;
|
|
}
|
|
|
|
// ---- OpenAI-Compatible Types ----
|
|
|
|
export interface ChatMessage {
|
|
role: 'system' | 'user' | 'assistant';
|
|
content: string;
|
|
}
|
|
|
|
export interface ChatCompletionRequest {
|
|
model?: string;
|
|
messages: ChatMessage[];
|
|
temperature?: number;
|
|
max_tokens?: number;
|
|
stream?: boolean;
|
|
top_p?: number;
|
|
}
|
|
|
|
export interface ChatCompletionChoice {
|
|
index: number;
|
|
message: ChatMessage;
|
|
finish_reason: string | null;
|
|
}
|
|
|
|
export interface TokenUsage {
|
|
prompt_tokens: number;
|
|
completion_tokens: number;
|
|
total_tokens: number;
|
|
}
|
|
|
|
export interface ChatCompletionResponse {
|
|
id: string;
|
|
object: 'chat.completion';
|
|
created: number;
|
|
model: string;
|
|
choices: ChatCompletionChoice[];
|
|
usage: TokenUsage;
|
|
_routed_via?: {
|
|
platform: Platform;
|
|
model: string;
|
|
};
|
|
}
|
|
|
|
export interface ChatCompletionChunk {
|
|
id: string;
|
|
object: 'chat.completion.chunk';
|
|
created: number;
|
|
model: string;
|
|
choices: {
|
|
index: number;
|
|
delta: Partial<ChatMessage>;
|
|
finish_reason: string | null;
|
|
}[];
|
|
}
|
|
|
|
// ---- Analytics Types ----
|
|
|
|
export interface AnalyticsSummary {
|
|
totalRequests: number;
|
|
successRate: number;
|
|
totalInputTokens: number;
|
|
totalOutputTokens: number;
|
|
avgLatencyMs: number;
|
|
estimatedCostSavings: number;
|
|
}
|
|
|
|
export interface PlatformStats {
|
|
platform: Platform;
|
|
requests: number;
|
|
successRate: number;
|
|
avgLatencyMs: number;
|
|
totalInputTokens: number;
|
|
totalOutputTokens: number;
|
|
}
|
|
|
|
export interface TimelinePoint {
|
|
timestamp: string;
|
|
requests: number;
|
|
successCount: number;
|
|
failureCount: number;
|
|
}
|
|
|
|
export interface RequestLog {
|
|
id: number;
|
|
platform: Platform;
|
|
modelId: string;
|
|
status: 'success' | 'error';
|
|
inputTokens: number;
|
|
outputTokens: number;
|
|
latencyMs: number;
|
|
error: string | null;
|
|
createdAt: string;
|
|
}
|
|
|
|
// ---- Rate Limit Types ----
|
|
|
|
export interface RateLimitStatus {
|
|
platform: Platform;
|
|
modelId: string;
|
|
rpm: { used: number; limit: number | null };
|
|
rpd: { used: number; limit: number | null };
|
|
tpm: { used: number; limit: number | null };
|
|
available: boolean;
|
|
nextResetAt: string | null;
|
|
}
|