feat(router): explain WHY the pool is empty on exhaustion (#423) (#444)

'All models exhausted. Add more API keys or wait for rate limits to reset.'
gave the caller nothing to act on. The per-model routing diagnostics already
existed but were only logged server-side.

summarizeExhaustion() rolls them into a short, client-safe summary — aggregate
bucket counts (rate-limited/on cooldown, no usable key, prompt too large,
lacks vision/tools, ...), no key material or per-key detail — and appends a
soonest-reset ETA from getSoonestCooldownExpiry(). The message now reads e.g.
'All models exhausted: 12 routes checked (8 rate-limited or on cooldown,
2 no usable key configured, 2 prompt too large for the model). Add more API
keys or wait for rate limits to reset. Soonest reset ~3m.'

Classifies off the whole diagnostic line since model ids can contain ':'.
Keeps the 'All models exhausted' prefix so existing matchers/tests hold.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

server/src/__tests__/services/exhaustion-summary.test.ts

@@ -0,0 +1,80 @@
+import { describe, it, expect } from 'vitest';
+import { summarizeExhaustion, formatResetEta } from '../../services/router.js';
+
+// #423: "All models exhausted" gave the caller nothing to act on. The summary
+// rolls the per-model routing diagnostics into aggregate, client-safe buckets
+// plus a soonest-reset ETA.
+
+describe('formatResetEta', () => {
+  const now = 1_000_000_000_000;
+  it('formats seconds under 90s', () => {
+    expect(formatResetEta(now + 12_000, now)).toBe('~12s');
+  });
+  it('formats minutes from 90s up', () => {
+    expect(formatResetEta(now + 4 * 60_000, now)).toBe('~4m');
+  });
+  it('formats hours past 90m', () => {
+    expect(formatResetEta(now + 3 * 60 * 60_000, now)).toBe('~3h');
+  });
+  it('returns null for lapsed or missing timestamps', () => {
+    expect(formatResetEta(now - 1000, now)).toBeNull();
+    expect(formatResetEta(null, now)).toBeNull();
+    expect(formatResetEta(undefined, now)).toBeNull();
+  });
+});
+
+describe('summarizeExhaustion', () => {
+  const now = 1_000_000_000_000;
+
+  it('keeps the "All models exhausted" prefix (backwards-compatible)', () => {
+    expect(summarizeExhaustion([], null, now)).toMatch(/^All models exhausted/);
+    expect(summarizeExhaustion(undefined, null, now)).toMatch(/^All models exhausted/);
+  });
+
+  it('buckets rate-limit and cooldown reasons together', () => {
+    const diag = [
+      'groq/llama-3.3-70b: 2 key(s) — cooldown:1, rpm/rpd-limit:1',
+      'google/gemini-1.5-flash: 1 key(s) — tpm/tpd-limit:1',
+    ];
+    const msg = summarizeExhaustion(diag, null, now);
+    expect(msg).toContain('2 routes checked');
+    expect(msg).toContain('2 rate-limited or on cooldown');
+  });
+
+  it('distinguishes "no usable key" from rate limits', () => {
+    const diag = [
+      'cohere/command-r: no enabled+healthy key for platform',
+      'groq/llama: 1 key(s) — cooldown:1',
+    ];
+    const msg = summarizeExhaustion(diag, null, now);
+    expect(msg).toContain('1 no usable key configured');
+    expect(msg).toContain('1 rate-limited or on cooldown');
+  });
+
+  it('classifies prompt-too-large lines, not as rate limits', () => {
+    const diag = [
+      'groq/gpt-oss-120b: tpm_limit 8000 < estimated 33476',
+      'google/flash: context 32768 < estimated 40000',
+    ];
+    const msg = summarizeExhaustion(diag, null, now);
+    expect(msg).toContain('2 prompt too large for the model');
+    expect(msg).not.toContain('rate-limited');
+  });
+
+  it('handles model ids that contain a colon', () => {
+    const diag = ['custom/qwen3:4b: no vision support'];
+    const msg = summarizeExhaustion(diag, null, now);
+    expect(msg).toContain('1 model lacks vision');
+  });
+
+  it('appends the soonest-reset ETA when a cooldown is active', () => {
+    const diag = ['groq/llama: 1 key(s) — cooldown:1'];
+    const msg = summarizeExhaustion(diag, now + 3 * 60_000, now);
+    expect(msg).toContain('Soonest reset ~3m.');
+  });
+
+  it('omits the ETA when nothing is cooling down', () => {
+    const diag = ['cohere/command-r: no enabled+healthy key for platform'];
+    expect(summarizeExhaustion(diag, null, now)).not.toContain('Soonest reset');
+  });
+});

server/src/services/ratelimit.ts

@@ -477,6 +477,22 @@ export function isOnCooldown(platform: string, modelId: string, keyId: number):
   return true;
 }
 
+/**
+ * Soonest moment any active cooldown expires, in ms since epoch, or null when
+ * nothing is cooling down. Used to tell an exhausted caller roughly when to
+ * retry (#423) instead of the bare "wait for rate limits to reset".
+ */
+export function getSoonestCooldownExpiry(now = Date.now()): number | null {
+  return withDb(db => {
+    const row = db.prepare(`
+      SELECT MIN(expires_at_ms) AS soonest
+        FROM rate_limit_cooldowns
+       WHERE expires_at_ms > ?
+    `).get(now) as { soonest: number | null } | undefined;
+    return row?.soonest ?? null;
+  }) ?? null;
+}
+
 export function getRateLimitStatus(
   platform: string,
   modelId: string,

server/src/services/router.ts

@@ -1,7 +1,7 @@
 import { getDb, getSetting, setSetting } from '../db/index.js';
 import { getProvider, hasProvider, resolveProvider } from '../providers/index.js';
 import { decrypt } from '../lib/crypto.js';
-import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider } from './ratelimit.js';
+import { canMakeRequest, canUseTokens, isOnCooldown, canUseProvider, getSoonestCooldownExpiry } from './ratelimit.js';
 import {
   BANDIT_PRESETS, DEFAULT_STRATEGY, type RoutingStrategy, type RoutingWeights,
   reliabilityPosterior, expectedReliability, sampleBeta,
@@ -28,6 +28,66 @@ class RouteError extends Error {
   }
 }
 
+// Human-readable retry ETA from a cooldown expiry timestamp (#423). Null when
+// nothing is cooling down or it already lapsed.
+export function formatResetEta(soonestResetMs: number | null | undefined, now = Date.now()): string | null {
+  if (soonestResetMs == null) return null;
+  const deltaMs = soonestResetMs - now;
+  if (deltaMs <= 0) return null;
+  const secs = Math.round(deltaMs / 1000);
+  if (secs < 90) return `~${secs}s`;
+  const mins = Math.round(secs / 60);
+  if (mins < 90) return `~${mins}m`;
+  return `~${Math.round(mins / 60)}h`;
+}
+
+const EXHAUSTION_ADVICE = 'Add more API keys or wait for rate limits to reset.';
+
+// Roll the per-model diagnostics (see RouteError.diagnostics) up into a short,
+// client-safe summary so an exhausted caller learns WHY the pool was empty
+// instead of a bare "All models exhausted" (#423). Buckets are aggregate
+// counts only — no key material, no per-key detail. Classifies off the whole
+// line (model ids can contain ':' so splitting label from reason is unsafe).
+export function summarizeExhaustion(
+  diag: string[] | undefined,
+  soonestResetMs?: number | null,
+  now = Date.now(),
+): string {
+  const eta = formatResetEta(soonestResetMs, now);
+  const etaSuffix = eta ? ` Soonest reset ${eta}.` : '';
+  if (!diag || diag.length === 0) {
+    return `All models exhausted. ${EXHAUSTION_ADVICE}${etaSuffix}`;
+  }
+
+  const counts: Record<string, number> = {};
+  const bump = (bucket: string) => { counts[bucket] = (counts[bucket] ?? 0) + 1; };
+  for (const line of diag) {
+    const l = line.toLowerCase();
+    if (l.includes('no provider registered')) bump('unsupported provider');
+    else if (/no enabled\+healthy key|no usable key|decrypt-error/.test(l)) bump('no usable key configured');
+    else if (l.includes('< estimated')) bump('prompt too large for the model');
+    else if (l.includes('no vision support')) bump('model lacks vision');
+    else if (l.includes('no tool-calling support')) bump('model lacks tool-calling');
+    else if (/ruled out|already-failed/.test(l)) bump('failed earlier this request');
+    else if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap/.test(l)) bump('rate-limited or on cooldown');
+    else bump('unavailable');
+  }
+  // Most actionable buckets first.
+  const order = [
+    'rate-limited or on cooldown',
+    'no usable key configured',
+    'prompt too large for the model',
+    'model lacks vision',
+    'model lacks tool-calling',
+    'failed earlier this request',
+    'unsupported provider',
+    'unavailable',
+  ];
+  const parts = order.filter(b => counts[b]).map(b => `${counts[b]} ${b}`);
+  const total = diag.length;
+  return `All models exhausted: ${total} route${total === 1 ? '' : 's'} checked (${parts.join(', ')}). ${EXHAUSTION_ADVICE}${etaSuffix}`;
+}
+
 interface KeyRow {
   id: number;
   platform: string;
@@ -916,7 +976,7 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
     if (route) return route;
   }
 
-  throw new RouteError('All models exhausted. Add more API keys or wait for rate limits to reset.', 429, diag);
+  throw new RouteError(summarizeExhaustion(diag, getSoonestCooldownExpiry()), 429, diag);
 }
 
 /**