fix(channels): retry transient gateway activation

This commit is contained in:
Tyler Slaton
2026-08-03 13:42:00 -07:00
committed by Tyler Slaton
parent 503fc7c593
commit 82eaecdfda
5 changed files with 439 additions and 75 deletions
@@ -0,0 +1,76 @@
import { expect, test, vi } from "vitest";
import {
connectRealtimeGateway,
RealtimeGatewayUnreachableError,
} from "./realtime-gateway.js";
interface RetryabilitySetup {
connection: ReturnType<typeof connectRealtimeGateway>;
diagnosticFetch: ReturnType<typeof vi.fn<typeof fetch>>;
}
/** Build one socket that cannot upgrade and an HTTP diagnosis for its host. */
function setupRetryability(status: number): RetryabilitySetup {
class RefusedWebSocket {
static readonly CONNECTING = 0;
static readonly OPEN = 1;
static readonly CLOSING = 2;
static readonly CLOSED = 3;
readyState = RefusedWebSocket.CONNECTING;
binaryType = "arraybuffer";
onopen: (() => void) | null = null;
onmessage: ((event: { data: string }) => void) | null = null;
onerror: ((error: Error) => void) | null = null;
onclose: ((event: { code: number }) => void) | null = null;
constructor() {
queueMicrotask(() => this.onerror?.(new Error("upgrade refused")));
}
send(): void {}
close(): void {
this.readyState = RefusedWebSocket.CLOSED;
queueMicrotask(() => this.onclose?.({ code: 1000 }));
}
}
const diagnosticFetch = vi.fn<typeof fetch>(async () =>
Promise.resolve(new Response(null, { status })),
);
const connection = connectRealtimeGateway({
wsUrl: "wss://gateway.example/channels",
apiKey: "cpk-test",
projectId: 7,
join: {
protocol: "channel_delivery_v1",
runtimeInstanceId: "rti_1",
channels: [{ channelName: "support", adapter: "slack" }],
},
connectTimeoutMs: 10,
diagnosticFetch,
webSocket: RefusedWebSocket,
});
return { connection, diagnosticFetch };
}
test("an HTTP 502 gateway diagnosis is retryable", async () => {
const { connection, diagnosticFetch } = setupRetryability(502);
const error = await connection.catch((cause: unknown) => cause);
expect(diagnosticFetch).toHaveBeenCalledOnce();
expect(error).toBeInstanceOf(RealtimeGatewayUnreachableError);
expect((error as RealtimeGatewayUnreachableError).retryable).toBe(true);
});
test("an HTTP 403 gateway diagnosis is terminal", async () => {
const { connection, diagnosticFetch } = setupRetryability(403);
const error = await connection.catch((cause: unknown) => cause);
expect(diagnosticFetch).toHaveBeenCalledOnce();
expect(error).toBeInstanceOf(RealtimeGatewayUnreachableError);
expect((error as RealtimeGatewayUnreachableError).retryable).toBe(false);
});
@@ -139,12 +139,15 @@ export interface RealtimeGatewayConnectionDetail {
* endpoint that was tried and the underlying transport error so the message
* points at the misconfigured URL rather than at a stopwatch (OSS-623).
*
* An unreachable host is a caller configuration error that must reject
* `ready()`.
* Permanent reachability failures such as NXDOMAIN reject `ready()`. Transient
* failures such as an HTTP 5xx response carry `retryable=true`, allowing the
* runtime manager to attempt a fresh initial connection with backoff.
*/
export class RealtimeGatewayUnreachableError extends Error {
/** Cross-package marker, mirroring the `code` convention of its siblings. */
readonly code = "GATEWAY_UNREACHABLE";
/** Whether reconnecting later can succeed without changing configuration. */
readonly retryable: boolean;
/** The gateway endpoint that was tried, minus any query string. */
readonly endpoint: string;
/**
@@ -158,14 +161,13 @@ export class RealtimeGatewayUnreachableError extends Error {
* @param transportError - Rendered transport failure, if one was reported.
* @param connectTimeoutMs - The elapsed connect window, named when no
* transport error was reported (the only signal the caller has left).
* @param options - Standard error options; carries the raw transport error
* as `cause`.
* @param options - Retry classification plus the raw transport error.
*/
constructor(
endpoint: string,
transportError: string | undefined,
connectTimeoutMs: number,
options?: { cause?: unknown },
options?: { cause?: unknown; retryable?: boolean },
) {
super(
`realtime gateway unreachable: the socket never connected to ${endpoint} ` +
@@ -178,6 +180,7 @@ export class RealtimeGatewayUnreachableError extends Error {
this.name = "RealtimeGatewayUnreachableError";
this.endpoint = endpoint;
this.transportError = transportError;
this.retryable = options?.retryable ?? false;
}
}
@@ -570,12 +573,19 @@ export async function connectRealtimeGateway(
? await diagnosis
: await startDiagnosis();
const cause = probed?.cause ?? lastTransportRaw;
const retryable =
probed === undefined ||
(!probed.nonRetryable &&
(probed.status === undefined || probed.status >= 500));
failConnect(
new RealtimeGatewayUnreachableError(
endpoint,
probed?.text ?? lastTransportError,
connectTimeoutMs,
cause !== undefined ? { cause } : undefined,
{
...(cause !== undefined ? { cause } : {}),
retryable,
},
),
);
})();
@@ -155,8 +155,7 @@ describe("ChannelManager connection health (onStateChange)", () => {
expect(mgr.status().overall).toBe("stopped");
});
it("logs the drop cause and backs off while the session stays down (OSS-670)", async () => {
vi.useFakeTimers();
it("logs the drop cause and keeps logging while the session is down (OSS-670)", async () => {
const handle = observableHandle();
const engine: ActivateChannelEngine = vi.fn(async () => handle);
const logs: string[] = [];
@@ -166,46 +165,31 @@ describe("ChannelManager connection health (onStateChange)", () => {
channels: [createChannel({ identifyUser: "platform", name: "support" })],
activateChannel: engine,
log: (m: string) => logs.push(m),
reconnectLogIntervalMs: 30_000,
reconnectLogIntervalMs: 5,
});
try {
mgr.activate();
await mgr.ready();
mgr.activate();
await mgr.ready();
handle.fireState("reconnecting", {
reason: "read ECONNRESET",
code: "ECONNRESET",
});
expect(logs.some((m) => m.includes("ECONNRESET"))).toBe(true);
handle.fireState("reconnecting", {
reason: "read ECONNRESET",
code: "ECONNRESET",
});
expect(logs.some((m) => m.includes("ECONNRESET"))).toBe(true);
const stillDownLogs = () => logs.filter((m) => m.includes("still down"));
// Still down: the operator must keep hearing about it.
await new Promise((r) => setTimeout(r, 20));
expect(logs.filter((m) => m.includes("still down")).length).toBeGreaterThan(
0,
);
await vi.advanceTimersByTimeAsync(30_000);
expect(stillDownLogs()).toHaveLength(1);
// The next reminder waits twice as long instead of repeating every 30s.
await vi.advanceTimersByTimeAsync(30_000);
expect(stillDownLogs()).toHaveLength(1);
await vi.advanceTimersByTimeAsync(30_000);
expect(stillDownLogs()).toHaveLength(2);
// Each reminder doubles the delay again: 30s, 60s, then 120s.
await vi.advanceTimersByTimeAsync(119_999);
expect(stillDownLogs()).toHaveLength(2);
await vi.advanceTimersByTimeAsync(1);
expect(stillDownLogs()).toHaveLength(3);
const before = logs.length;
handle.fireState("online");
await vi.advanceTimersByTimeAsync(15 * 60_000);
// Recovery cancels future reminders; only "back online" lands after it.
expect(
logs.slice(before).filter((m) => m.includes("still down")),
).toHaveLength(0);
await mgr.stop();
} finally {
vi.useRealTimers();
}
const before = logs.length;
handle.fireState("online");
await new Promise((r) => setTimeout(r, 20));
// Recovery stops the repeat: only the "back online" line lands after it.
expect(
logs.slice(before).filter((m) => m.includes("still down")),
).toHaveLength(0);
await mgr.stop();
});
it("says retries continue when the give-up window elapses (OSS-670)", async () => {
@@ -0,0 +1,187 @@
import { expect, test, vi } from "vitest";
import { createChannel } from "@copilotkit/channels";
import { RealtimeGatewayUnreachableError } from "@copilotkit/channels-intelligence";
import { CopilotKitIntelligence } from "../../intelligence-platform";
import { ChannelManager } from "../channel-manager";
import type { ActivateChannelEngine, ChannelsHandle } from "../channel-manager";
interface RecoverySetup {
manager: ChannelManager;
activateChannel: ReturnType<typeof vi.fn<ActivateChannelEngine>>;
cleanup(): Promise<void>;
}
type ConnectionState = "online" | "reconnecting" | "gave_up";
interface ReconnectLoggingSetup {
manager: ChannelManager;
logs: string[];
fireState(state: ConnectionState): void;
cleanup(): Promise<void>;
}
/** Create a gateway error with the cross-package retry contract. */
function gatewayError(retryable: boolean): Error {
return new RealtimeGatewayUnreachableError(
"wss://runtime.example/channels",
"the gateway host answered HTTP 502",
30_000,
{ retryable },
);
}
/** Build one isolated manager whose first gateway activation is unavailable. */
function setupRecovery(firstError = gatewayError(true)): RecoverySetup {
vi.useFakeTimers();
const handle: ChannelsHandle = {
metadata: {},
stop: vi.fn(async () => {}),
};
const activateChannel = vi
.fn<ActivateChannelEngine>()
.mockRejectedValueOnce(firstError)
.mockResolvedValue(handle);
const manager = new ChannelManager({
intelligence: new CopilotKitIntelligence({
apiUrl: "https://runtime.example",
wsUrl: "wss://runtime.example",
apiKey: "cpk-42_short_long",
}),
channels: [createChannel({ identifyUser: "platform", name: "support" })],
activateChannel,
});
return {
manager,
activateChannel,
async cleanup() {
await manager.stop();
vi.useRealTimers();
},
};
}
/** Build one isolated online manager whose connection state is test-driven. */
function setupReconnectLogging(): ReconnectLoggingSetup {
vi.useFakeTimers();
let stateListener: ((state: ConnectionState) => void) | undefined;
const handle: ChannelsHandle = {
metadata: {},
stop: vi.fn(async () => {}),
onStateChange(listener: (state: ConnectionState) => void) {
stateListener = listener;
},
};
const logs: string[] = [];
const manager = new ChannelManager({
intelligence: new CopilotKitIntelligence({
apiUrl: "https://runtime.example",
wsUrl: "wss://runtime.example",
apiKey: "cpk-42_short_long",
}),
channels: [createChannel({ identifyUser: "platform", name: "support" })],
activateChannel: vi.fn(async () => handle),
log: (message: string) => logs.push(message),
});
return {
manager,
logs,
fireState(state: ConnectionState) {
stateListener?.(state);
},
async cleanup() {
await manager.stop();
vi.useRealTimers();
},
};
}
test("a transient initial gateway outage retries until the channel is online", async () => {
const { manager, activateChannel, cleanup } = setupRecovery();
try {
manager.activate();
await vi.advanceTimersByTimeAsync(0);
expect(activateChannel).toHaveBeenCalledTimes(1);
expect(manager.status().channels.support).toBe("reconnecting");
await vi.advanceTimersByTimeAsync(999);
expect(activateChannel).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(1);
await expect(manager.ready()).resolves.toBeUndefined();
expect(activateChannel).toHaveBeenCalledTimes(2);
expect(manager.status().channels.support).toBe("online");
} finally {
await cleanup();
}
});
test("a permanent initial gateway error remains terminal", async () => {
const { manager, activateChannel, cleanup } = setupRecovery(
gatewayError(false),
);
try {
manager.activate();
await vi.advanceTimersByTimeAsync(0);
expect(manager.status().channels.support).toBe("error");
await expect(manager.ready()).rejects.toThrow(AggregateError);
await vi.advanceTimersByTimeAsync(60_000);
expect(activateChannel).toHaveBeenCalledTimes(1);
} finally {
await cleanup();
}
});
test("stopping the manager cancels a pending activation retry", async () => {
const { manager, activateChannel, cleanup } = setupRecovery();
try {
manager.activate();
await vi.advanceTimersByTimeAsync(0);
expect(manager.status().channels.support).toBe("reconnecting");
await manager.stop();
await vi.advanceTimersByTimeAsync(60_000);
expect(activateChannel).toHaveBeenCalledTimes(1);
expect(manager.status().channels.support).toBe("stopped");
} finally {
await cleanup();
}
});
test("a prolonged established-session outage backs off reminder logs", async () => {
const { manager, logs, fireState, cleanup } = setupReconnectLogging();
try {
manager.activate();
await manager.ready();
fireState("reconnecting");
const stillDownLogs = () =>
logs.filter((line) => line.includes("still down"));
await vi.advanceTimersByTimeAsync(30_000);
expect(stillDownLogs()).toHaveLength(1);
await vi.advanceTimersByTimeAsync(30_000);
expect(stillDownLogs()).toHaveLength(1);
await vi.advanceTimersByTimeAsync(30_000);
expect(stillDownLogs()).toHaveLength(2);
await vi.advanceTimersByTimeAsync(120_000);
expect(stillDownLogs()).toHaveLength(3);
fireState("online");
await vi.advanceTimersByTimeAsync(15 * 60_000);
expect(stillDownLogs()).toHaveLength(3);
} finally {
await cleanup();
}
});
@@ -194,6 +194,12 @@ interface ChannelEntry {
reconnectLogTimer?: ReturnType<typeof setTimeout>;
/** Delay before the next reminder; doubles after each emitted reminder. */
reconnectLogDelayMs?: number;
/** Next retry after a transient initial activation failure. */
activationRetryTimer?: ReturnType<typeof setTimeout>;
/** Delay before the next activation retry; doubles after each failed attempt. */
activationRetryDelayMs?: number;
/** Reject the retry wrapper when teardown cancels a pending retry. */
cancelActivationRetry?: () => void;
}
/**
@@ -717,6 +723,16 @@ function isSetupRequired(err: unknown): boolean {
);
}
/** Whether a failed initial activation can recover without new configuration. */
function isRetryableActivationError(err: unknown): boolean {
return (
typeof err === "object" &&
err !== null &&
(err as { code?: unknown }).code === "GATEWAY_UNREACHABLE" &&
(err as { retryable?: unknown }).retryable === true
);
}
/**
* Whether `err` is a Node/runtime module-resolution failure — i.e. the error
* a dynamic `import()` throws when the target package is not installed.
@@ -740,6 +756,12 @@ const DEFAULT_RECONNECT_LOG_INTERVAL_MS = 30_000;
/** Longest delay (ms) between reminders during one continuous outage. */
const DEFAULT_RECONNECT_LOG_MAX_INTERVAL_MS = 15 * 60_000;
/** First delay (ms) before retrying a transient initial activation failure. */
const DEFAULT_ACTIVATION_RETRY_DELAY_MS = 1_000;
/** Longest delay (ms) between transient initial activation attempts. */
const DEFAULT_ACTIVATION_RETRY_MAX_DELAY_MS = 30_000;
/**
* Reject with `timeoutMessage` after `timeoutMs` if `inner` has not settled,
* otherwise pass `inner` through. When `timeoutMs` is undefined, `inner` is
@@ -785,17 +807,18 @@ function withTimeout<T>(
* {@link activate} starts it and a second call is a no-op. Activation throws
* SYNCHRONOUSLY (a {@link ChannelConfigError}) only for a misconfiguration it
* can detect up front — a duplicate or missing Channel name. Every OTHER
* activation failure is recorded as the Channel's status (`error`, or
* `setup_required` for a missing provider) and surfaced through {@link status}
* and {@link ready} rather than thrown.
* permanent activation failure is recorded as the Channel's status (`error`,
* or `setup_required` for a missing provider) and surfaced through
* {@link status} and {@link ready} rather than thrown. A retryable initial
* gateway outage stays unsettled and retries until it connects or the manager
* stops.
*
* Reconnection is NOT handled here — it is delegated to the Phoenix connection
* Established-session reconnection is delegated to the Phoenix connection
* layer that backs the launcher. When a managed control socket drops, Phoenix's
* `Socket` reconnects and rejoins with the same Runtime declaration. Active
* deliveries request fresh one-use join tokens through that control link. A
* re-activation here would be both redundant AND broken: re-invoking the engine
* on an already-started `Channel` throws in `channel.addAdapter` (started=true).
* The manager therefore never re-activates on a drop.
* `Socket` reconnects and rejoins with the same Runtime declaration. The manager
* never re-activates an already-started Channel. It does retry a transient
* INITIAL gateway activation failure: that happens before the launcher adds or
* starts the managed adapter, so a later attempt is safe.
*
* It DOES, however, reflect real connection health through the session's
* `onStateChange` observer so {@link ChannelManager.status} stays honest rather
@@ -865,8 +888,8 @@ export class ChannelManager implements ChannelsControl {
/**
* Start activation of every declared Channel (lazy + idempotent). Mints a
* distinct runtime instance id per Channel, derives its activation config,
* and calls the engine. Records each Channel as `connecting`, transitioning
* to `online`/`setup_required`/`error` as its activation settles.
* and calls the engine. Transient gateway failures retry with exponential
* backoff; other outcomes transition to `online`/`setup_required`/`error`.
*/
activate(): void {
// Short-circuit on BOTH latches: `activated` makes activation idempotent,
@@ -903,26 +926,8 @@ export class ChannelManager implements ChannelsControl {
// promise is always considered handled — ready() still sees the reason.
settled.catch(() => {});
// Invoke the engine synchronously so activation is observably started the
// moment activate() returns (callers assert the engine was called and see
// `connecting` before awaiting ready). A synchronous config/engine throw is
// turned into a rejected activation so it becomes this channel's status
// rather than throwing out of activate().
let activation: Promise<ChannelsHandle>;
let config: ChannelActivationConfig | undefined;
try {
config = deriveChannelActivationConfig({
intelligence: this.intelligence,
channel,
runtimeInstanceId,
});
activation = this.activateChannel(config, channel);
} catch (err) {
activation = Promise.reject(err);
}
// The deferred `.then` callbacks capture `entry` and run only after the
// literal has fully initialized, so referencing it here is safe.
// The deferred activation callbacks capture `entry` and run only after
// the literal has fully initialized, so referencing it there is safe.
const entry: ChannelEntry = {
status: "connecting",
handle: undefined,
@@ -930,6 +935,21 @@ export class ChannelManager implements ChannelsControl {
settled,
};
// Invoke the engine synchronously so activation is observably started the
// moment activate() returns. Only a typed transient gateway failure is
// retried; config errors stay on the existing terminal path.
let activation: Promise<ChannelsHandle>;
try {
const config = deriveChannelActivationConfig({
intelligence: this.intelligence,
channel,
runtimeInstanceId,
});
activation = this.activateWithRetry(config, channel, name, entry);
} catch (err) {
activation = Promise.reject(err);
}
// Anchor the settle handlers. Both branches route every teardown through
// the idempotent `stopEntry`, so a late settle can never resurrect a
// `stopped` entry and a handle is torn down at most once. The handlers
@@ -1012,6 +1032,72 @@ export class ChannelManager implements ChannelsControl {
}
}
/**
* Retry only transient failures from the pre-adapter gateway connection.
* Permanent errors reject on the first attempt; teardown cancels a pending
* timer while preserving the existing late-settle handling for in-flight work.
*/
private activateWithRetry(
config: ChannelActivationConfig,
channel: Channel,
name: string,
entry: ChannelEntry,
): Promise<ChannelsHandle> {
return new Promise<ChannelsHandle>((resolve, reject) => {
const attempt = (): void => {
let activation: Promise<ChannelsHandle>;
try {
activation = this.activateChannel(config, channel);
} catch (err) {
activation = Promise.reject(err);
}
activation.then(
(handle) => {
this.clearActivationRetry(entry);
resolve(handle);
},
(err: unknown) => {
if (this.stopped || !isRetryableActivationError(err)) {
this.clearActivationRetry(entry);
reject(err);
return;
}
const delayMs =
entry.activationRetryDelayMs ?? DEFAULT_ACTIVATION_RETRY_DELAY_MS;
entry.status = "reconnecting";
entry.activationRetryDelayMs = Math.min(
delayMs * 2,
DEFAULT_ACTIVATION_RETRY_MAX_DELAY_MS,
);
this.log?.(
`channel "${name}" failed to activate; retrying in ${delayMs}ms`,
err,
);
const timer = setTimeout(() => {
entry.activationRetryTimer = undefined;
entry.cancelActivationRetry = undefined;
if (this.stopped || entry.status === "stopped") {
reject(err);
return;
}
entry.status = "connecting";
attempt();
}, delayMs);
(timer as unknown as { unref?: () => void }).unref?.();
entry.activationRetryTimer = timer;
entry.cancelActivationRetry = () => {
this.clearActivationRetry(entry);
reject(err);
};
},
);
};
attempt();
});
}
/**
* Throw if two declared Channels share a `name`. `entries` is keyed by name,
* so a duplicate would overwrite the first Channel's entry and leak its live
@@ -1277,6 +1363,26 @@ export class ChannelManager implements ChannelsControl {
entry.reconnectLogDelayMs = undefined;
}
/** Cancel a pending transient activation retry and reset its backoff. */
private clearActivationRetry(entry: ChannelEntry): void {
if (entry.activationRetryTimer !== undefined) {
clearTimeout(entry.activationRetryTimer);
entry.activationRetryTimer = undefined;
}
entry.activationRetryDelayMs = undefined;
entry.cancelActivationRetry = undefined;
}
/** Cancel a scheduled activation retry and settle its wrapper. */
private cancelActivationRetry(entry: ChannelEntry): void {
const cancel = entry.cancelActivationRetry;
if (cancel) {
cancel();
} else {
this.clearActivationRetry(entry);
}
}
/**
* Drive a single entry to its terminal `stopped` state, tearing down its
* handle AT MOST ONCE. Idempotent: it always sets `status = "stopped"`, and
@@ -1314,6 +1420,7 @@ export class ChannelManager implements ChannelsControl {
// An unref'd interval would not hold the process open, but a stopped
// manager must not keep logging about a session it no longer owns.
this.clearReconnectLog(entry);
this.cancelActivationRetry(entry);
if (entry.handle && !entry.handleStopped) {
entry.handleStopped = true;
const handle = entry.handle;