fix(channels): retry transient gateway activation
This commit is contained in:
committed by
Tyler Slaton
parent
503fc7c593
commit
82eaecdfda
@@ -0,0 +1,76 @@
|
||||
import { expect, test, vi } from "vitest";
|
||||
import {
|
||||
connectRealtimeGateway,
|
||||
RealtimeGatewayUnreachableError,
|
||||
} from "./realtime-gateway.js";
|
||||
|
||||
interface RetryabilitySetup {
|
||||
connection: ReturnType<typeof connectRealtimeGateway>;
|
||||
diagnosticFetch: ReturnType<typeof vi.fn<typeof fetch>>;
|
||||
}
|
||||
|
||||
/** Build one socket that cannot upgrade and an HTTP diagnosis for its host. */
|
||||
function setupRetryability(status: number): RetryabilitySetup {
|
||||
class RefusedWebSocket {
|
||||
static readonly CONNECTING = 0;
|
||||
static readonly OPEN = 1;
|
||||
static readonly CLOSING = 2;
|
||||
static readonly CLOSED = 3;
|
||||
readyState = RefusedWebSocket.CONNECTING;
|
||||
binaryType = "arraybuffer";
|
||||
onopen: (() => void) | null = null;
|
||||
onmessage: ((event: { data: string }) => void) | null = null;
|
||||
onerror: ((error: Error) => void) | null = null;
|
||||
onclose: ((event: { code: number }) => void) | null = null;
|
||||
|
||||
constructor() {
|
||||
queueMicrotask(() => this.onerror?.(new Error("upgrade refused")));
|
||||
}
|
||||
|
||||
send(): void {}
|
||||
|
||||
close(): void {
|
||||
this.readyState = RefusedWebSocket.CLOSED;
|
||||
queueMicrotask(() => this.onclose?.({ code: 1000 }));
|
||||
}
|
||||
}
|
||||
|
||||
const diagnosticFetch = vi.fn<typeof fetch>(async () =>
|
||||
Promise.resolve(new Response(null, { status })),
|
||||
);
|
||||
const connection = connectRealtimeGateway({
|
||||
wsUrl: "wss://gateway.example/channels",
|
||||
apiKey: "cpk-test",
|
||||
projectId: 7,
|
||||
join: {
|
||||
protocol: "channel_delivery_v1",
|
||||
runtimeInstanceId: "rti_1",
|
||||
channels: [{ channelName: "support", adapter: "slack" }],
|
||||
},
|
||||
connectTimeoutMs: 10,
|
||||
diagnosticFetch,
|
||||
webSocket: RefusedWebSocket,
|
||||
});
|
||||
|
||||
return { connection, diagnosticFetch };
|
||||
}
|
||||
|
||||
test("an HTTP 502 gateway diagnosis is retryable", async () => {
|
||||
const { connection, diagnosticFetch } = setupRetryability(502);
|
||||
|
||||
const error = await connection.catch((cause: unknown) => cause);
|
||||
|
||||
expect(diagnosticFetch).toHaveBeenCalledOnce();
|
||||
expect(error).toBeInstanceOf(RealtimeGatewayUnreachableError);
|
||||
expect((error as RealtimeGatewayUnreachableError).retryable).toBe(true);
|
||||
});
|
||||
|
||||
test("an HTTP 403 gateway diagnosis is terminal", async () => {
|
||||
const { connection, diagnosticFetch } = setupRetryability(403);
|
||||
|
||||
const error = await connection.catch((cause: unknown) => cause);
|
||||
|
||||
expect(diagnosticFetch).toHaveBeenCalledOnce();
|
||||
expect(error).toBeInstanceOf(RealtimeGatewayUnreachableError);
|
||||
expect((error as RealtimeGatewayUnreachableError).retryable).toBe(false);
|
||||
});
|
||||
@@ -139,12 +139,15 @@ export interface RealtimeGatewayConnectionDetail {
|
||||
* endpoint that was tried and the underlying transport error so the message
|
||||
* points at the misconfigured URL rather than at a stopwatch (OSS-623).
|
||||
*
|
||||
* An unreachable host is a caller configuration error that must reject
|
||||
* `ready()`.
|
||||
* Permanent reachability failures such as NXDOMAIN reject `ready()`. Transient
|
||||
* failures such as an HTTP 5xx response carry `retryable=true`, allowing the
|
||||
* runtime manager to attempt a fresh initial connection with backoff.
|
||||
*/
|
||||
export class RealtimeGatewayUnreachableError extends Error {
|
||||
/** Cross-package marker, mirroring the `code` convention of its siblings. */
|
||||
readonly code = "GATEWAY_UNREACHABLE";
|
||||
/** Whether reconnecting later can succeed without changing configuration. */
|
||||
readonly retryable: boolean;
|
||||
/** The gateway endpoint that was tried, minus any query string. */
|
||||
readonly endpoint: string;
|
||||
/**
|
||||
@@ -158,14 +161,13 @@ export class RealtimeGatewayUnreachableError extends Error {
|
||||
* @param transportError - Rendered transport failure, if one was reported.
|
||||
* @param connectTimeoutMs - The elapsed connect window, named when no
|
||||
* transport error was reported (the only signal the caller has left).
|
||||
* @param options - Standard error options; carries the raw transport error
|
||||
* as `cause`.
|
||||
* @param options - Retry classification plus the raw transport error.
|
||||
*/
|
||||
constructor(
|
||||
endpoint: string,
|
||||
transportError: string | undefined,
|
||||
connectTimeoutMs: number,
|
||||
options?: { cause?: unknown },
|
||||
options?: { cause?: unknown; retryable?: boolean },
|
||||
) {
|
||||
super(
|
||||
`realtime gateway unreachable: the socket never connected to ${endpoint} ` +
|
||||
@@ -178,6 +180,7 @@ export class RealtimeGatewayUnreachableError extends Error {
|
||||
this.name = "RealtimeGatewayUnreachableError";
|
||||
this.endpoint = endpoint;
|
||||
this.transportError = transportError;
|
||||
this.retryable = options?.retryable ?? false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -570,12 +573,19 @@ export async function connectRealtimeGateway(
|
||||
? await diagnosis
|
||||
: await startDiagnosis();
|
||||
const cause = probed?.cause ?? lastTransportRaw;
|
||||
const retryable =
|
||||
probed === undefined ||
|
||||
(!probed.nonRetryable &&
|
||||
(probed.status === undefined || probed.status >= 500));
|
||||
failConnect(
|
||||
new RealtimeGatewayUnreachableError(
|
||||
endpoint,
|
||||
probed?.text ?? lastTransportError,
|
||||
connectTimeoutMs,
|
||||
cause !== undefined ? { cause } : undefined,
|
||||
{
|
||||
...(cause !== undefined ? { cause } : {}),
|
||||
retryable,
|
||||
},
|
||||
),
|
||||
);
|
||||
})();
|
||||
|
||||
@@ -155,8 +155,7 @@ describe("ChannelManager connection health (onStateChange)", () => {
|
||||
expect(mgr.status().overall).toBe("stopped");
|
||||
});
|
||||
|
||||
it("logs the drop cause and backs off while the session stays down (OSS-670)", async () => {
|
||||
vi.useFakeTimers();
|
||||
it("logs the drop cause and keeps logging while the session is down (OSS-670)", async () => {
|
||||
const handle = observableHandle();
|
||||
const engine: ActivateChannelEngine = vi.fn(async () => handle);
|
||||
const logs: string[] = [];
|
||||
@@ -166,46 +165,31 @@ describe("ChannelManager connection health (onStateChange)", () => {
|
||||
channels: [createChannel({ identifyUser: "platform", name: "support" })],
|
||||
activateChannel: engine,
|
||||
log: (m: string) => logs.push(m),
|
||||
reconnectLogIntervalMs: 30_000,
|
||||
reconnectLogIntervalMs: 5,
|
||||
});
|
||||
try {
|
||||
mgr.activate();
|
||||
await mgr.ready();
|
||||
mgr.activate();
|
||||
await mgr.ready();
|
||||
|
||||
handle.fireState("reconnecting", {
|
||||
reason: "read ECONNRESET",
|
||||
code: "ECONNRESET",
|
||||
});
|
||||
expect(logs.some((m) => m.includes("ECONNRESET"))).toBe(true);
|
||||
handle.fireState("reconnecting", {
|
||||
reason: "read ECONNRESET",
|
||||
code: "ECONNRESET",
|
||||
});
|
||||
expect(logs.some((m) => m.includes("ECONNRESET"))).toBe(true);
|
||||
|
||||
const stillDownLogs = () => logs.filter((m) => m.includes("still down"));
|
||||
// Still down: the operator must keep hearing about it.
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
expect(logs.filter((m) => m.includes("still down")).length).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(stillDownLogs()).toHaveLength(1);
|
||||
|
||||
// The next reminder waits twice as long instead of repeating every 30s.
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(stillDownLogs()).toHaveLength(1);
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(stillDownLogs()).toHaveLength(2);
|
||||
|
||||
// Each reminder doubles the delay again: 30s, 60s, then 120s.
|
||||
await vi.advanceTimersByTimeAsync(119_999);
|
||||
expect(stillDownLogs()).toHaveLength(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(stillDownLogs()).toHaveLength(3);
|
||||
|
||||
const before = logs.length;
|
||||
handle.fireState("online");
|
||||
await vi.advanceTimersByTimeAsync(15 * 60_000);
|
||||
// Recovery cancels future reminders; only "back online" lands after it.
|
||||
expect(
|
||||
logs.slice(before).filter((m) => m.includes("still down")),
|
||||
).toHaveLength(0);
|
||||
await mgr.stop();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
const before = logs.length;
|
||||
handle.fireState("online");
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
// Recovery stops the repeat: only the "back online" line lands after it.
|
||||
expect(
|
||||
logs.slice(before).filter((m) => m.includes("still down")),
|
||||
).toHaveLength(0);
|
||||
await mgr.stop();
|
||||
});
|
||||
|
||||
it("says retries continue when the give-up window elapses (OSS-670)", async () => {
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { createChannel } from "@copilotkit/channels";
|
||||
import { RealtimeGatewayUnreachableError } from "@copilotkit/channels-intelligence";
|
||||
import { CopilotKitIntelligence } from "../../intelligence-platform";
|
||||
import { ChannelManager } from "../channel-manager";
|
||||
import type { ActivateChannelEngine, ChannelsHandle } from "../channel-manager";
|
||||
|
||||
interface RecoverySetup {
|
||||
manager: ChannelManager;
|
||||
activateChannel: ReturnType<typeof vi.fn<ActivateChannelEngine>>;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
type ConnectionState = "online" | "reconnecting" | "gave_up";
|
||||
|
||||
interface ReconnectLoggingSetup {
|
||||
manager: ChannelManager;
|
||||
logs: string[];
|
||||
fireState(state: ConnectionState): void;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
/** Create a gateway error with the cross-package retry contract. */
|
||||
function gatewayError(retryable: boolean): Error {
|
||||
return new RealtimeGatewayUnreachableError(
|
||||
"wss://runtime.example/channels",
|
||||
"the gateway host answered HTTP 502",
|
||||
30_000,
|
||||
{ retryable },
|
||||
);
|
||||
}
|
||||
|
||||
/** Build one isolated manager whose first gateway activation is unavailable. */
|
||||
function setupRecovery(firstError = gatewayError(true)): RecoverySetup {
|
||||
vi.useFakeTimers();
|
||||
const handle: ChannelsHandle = {
|
||||
metadata: {},
|
||||
stop: vi.fn(async () => {}),
|
||||
};
|
||||
const activateChannel = vi
|
||||
.fn<ActivateChannelEngine>()
|
||||
.mockRejectedValueOnce(firstError)
|
||||
.mockResolvedValue(handle);
|
||||
const manager = new ChannelManager({
|
||||
intelligence: new CopilotKitIntelligence({
|
||||
apiUrl: "https://runtime.example",
|
||||
wsUrl: "wss://runtime.example",
|
||||
apiKey: "cpk-42_short_long",
|
||||
}),
|
||||
channels: [createChannel({ identifyUser: "platform", name: "support" })],
|
||||
activateChannel,
|
||||
});
|
||||
|
||||
return {
|
||||
manager,
|
||||
activateChannel,
|
||||
async cleanup() {
|
||||
await manager.stop();
|
||||
vi.useRealTimers();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Build one isolated online manager whose connection state is test-driven. */
|
||||
function setupReconnectLogging(): ReconnectLoggingSetup {
|
||||
vi.useFakeTimers();
|
||||
let stateListener: ((state: ConnectionState) => void) | undefined;
|
||||
const handle: ChannelsHandle = {
|
||||
metadata: {},
|
||||
stop: vi.fn(async () => {}),
|
||||
onStateChange(listener: (state: ConnectionState) => void) {
|
||||
stateListener = listener;
|
||||
},
|
||||
};
|
||||
const logs: string[] = [];
|
||||
const manager = new ChannelManager({
|
||||
intelligence: new CopilotKitIntelligence({
|
||||
apiUrl: "https://runtime.example",
|
||||
wsUrl: "wss://runtime.example",
|
||||
apiKey: "cpk-42_short_long",
|
||||
}),
|
||||
channels: [createChannel({ identifyUser: "platform", name: "support" })],
|
||||
activateChannel: vi.fn(async () => handle),
|
||||
log: (message: string) => logs.push(message),
|
||||
});
|
||||
|
||||
return {
|
||||
manager,
|
||||
logs,
|
||||
fireState(state: ConnectionState) {
|
||||
stateListener?.(state);
|
||||
},
|
||||
async cleanup() {
|
||||
await manager.stop();
|
||||
vi.useRealTimers();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("a transient initial gateway outage retries until the channel is online", async () => {
|
||||
const { manager, activateChannel, cleanup } = setupRecovery();
|
||||
|
||||
try {
|
||||
manager.activate();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(activateChannel).toHaveBeenCalledTimes(1);
|
||||
expect(manager.status().channels.support).toBe("reconnecting");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(999);
|
||||
expect(activateChannel).toHaveBeenCalledTimes(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await expect(manager.ready()).resolves.toBeUndefined();
|
||||
expect(activateChannel).toHaveBeenCalledTimes(2);
|
||||
expect(manager.status().channels.support).toBe("online");
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a permanent initial gateway error remains terminal", async () => {
|
||||
const { manager, activateChannel, cleanup } = setupRecovery(
|
||||
gatewayError(false),
|
||||
);
|
||||
|
||||
try {
|
||||
manager.activate();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(manager.status().channels.support).toBe("error");
|
||||
await expect(manager.ready()).rejects.toThrow(AggregateError);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(activateChannel).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("stopping the manager cancels a pending activation retry", async () => {
|
||||
const { manager, activateChannel, cleanup } = setupRecovery();
|
||||
|
||||
try {
|
||||
manager.activate();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(manager.status().channels.support).toBe("reconnecting");
|
||||
|
||||
await manager.stop();
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
|
||||
expect(activateChannel).toHaveBeenCalledTimes(1);
|
||||
expect(manager.status().channels.support).toBe("stopped");
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a prolonged established-session outage backs off reminder logs", async () => {
|
||||
const { manager, logs, fireState, cleanup } = setupReconnectLogging();
|
||||
|
||||
try {
|
||||
manager.activate();
|
||||
await manager.ready();
|
||||
fireState("reconnecting");
|
||||
const stillDownLogs = () =>
|
||||
logs.filter((line) => line.includes("still down"));
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(stillDownLogs()).toHaveLength(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(stillDownLogs()).toHaveLength(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(stillDownLogs()).toHaveLength(2);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
expect(stillDownLogs()).toHaveLength(3);
|
||||
|
||||
fireState("online");
|
||||
await vi.advanceTimersByTimeAsync(15 * 60_000);
|
||||
expect(stillDownLogs()).toHaveLength(3);
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
});
|
||||
@@ -194,6 +194,12 @@ interface ChannelEntry {
|
||||
reconnectLogTimer?: ReturnType<typeof setTimeout>;
|
||||
/** Delay before the next reminder; doubles after each emitted reminder. */
|
||||
reconnectLogDelayMs?: number;
|
||||
/** Next retry after a transient initial activation failure. */
|
||||
activationRetryTimer?: ReturnType<typeof setTimeout>;
|
||||
/** Delay before the next activation retry; doubles after each failed attempt. */
|
||||
activationRetryDelayMs?: number;
|
||||
/** Reject the retry wrapper when teardown cancels a pending retry. */
|
||||
cancelActivationRetry?: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -717,6 +723,16 @@ function isSetupRequired(err: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/** Whether a failed initial activation can recover without new configuration. */
|
||||
function isRetryableActivationError(err: unknown): boolean {
|
||||
return (
|
||||
typeof err === "object" &&
|
||||
err !== null &&
|
||||
(err as { code?: unknown }).code === "GATEWAY_UNREACHABLE" &&
|
||||
(err as { retryable?: unknown }).retryable === true
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `err` is a Node/runtime module-resolution failure — i.e. the error
|
||||
* a dynamic `import()` throws when the target package is not installed.
|
||||
@@ -740,6 +756,12 @@ const DEFAULT_RECONNECT_LOG_INTERVAL_MS = 30_000;
|
||||
/** Longest delay (ms) between reminders during one continuous outage. */
|
||||
const DEFAULT_RECONNECT_LOG_MAX_INTERVAL_MS = 15 * 60_000;
|
||||
|
||||
/** First delay (ms) before retrying a transient initial activation failure. */
|
||||
const DEFAULT_ACTIVATION_RETRY_DELAY_MS = 1_000;
|
||||
|
||||
/** Longest delay (ms) between transient initial activation attempts. */
|
||||
const DEFAULT_ACTIVATION_RETRY_MAX_DELAY_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Reject with `timeoutMessage` after `timeoutMs` if `inner` has not settled,
|
||||
* otherwise pass `inner` through. When `timeoutMs` is undefined, `inner` is
|
||||
@@ -785,17 +807,18 @@ function withTimeout<T>(
|
||||
* {@link activate} starts it and a second call is a no-op. Activation throws
|
||||
* SYNCHRONOUSLY (a {@link ChannelConfigError}) only for a misconfiguration it
|
||||
* can detect up front — a duplicate or missing Channel name. Every OTHER
|
||||
* activation failure is recorded as the Channel's status (`error`, or
|
||||
* `setup_required` for a missing provider) and surfaced through {@link status}
|
||||
* and {@link ready} rather than thrown.
|
||||
* permanent activation failure is recorded as the Channel's status (`error`,
|
||||
* or `setup_required` for a missing provider) and surfaced through
|
||||
* {@link status} and {@link ready} rather than thrown. A retryable initial
|
||||
* gateway outage stays unsettled and retries until it connects or the manager
|
||||
* stops.
|
||||
*
|
||||
* Reconnection is NOT handled here — it is delegated to the Phoenix connection
|
||||
* Established-session reconnection is delegated to the Phoenix connection
|
||||
* layer that backs the launcher. When a managed control socket drops, Phoenix's
|
||||
* `Socket` reconnects and rejoins with the same Runtime declaration. Active
|
||||
* deliveries request fresh one-use join tokens through that control link. A
|
||||
* re-activation here would be both redundant AND broken: re-invoking the engine
|
||||
* on an already-started `Channel` throws in `channel.addAdapter` (started=true).
|
||||
* The manager therefore never re-activates on a drop.
|
||||
* `Socket` reconnects and rejoins with the same Runtime declaration. The manager
|
||||
* never re-activates an already-started Channel. It does retry a transient
|
||||
* INITIAL gateway activation failure: that happens before the launcher adds or
|
||||
* starts the managed adapter, so a later attempt is safe.
|
||||
*
|
||||
* It DOES, however, reflect real connection health through the session's
|
||||
* `onStateChange` observer so {@link ChannelManager.status} stays honest rather
|
||||
@@ -865,8 +888,8 @@ export class ChannelManager implements ChannelsControl {
|
||||
/**
|
||||
* Start activation of every declared Channel (lazy + idempotent). Mints a
|
||||
* distinct runtime instance id per Channel, derives its activation config,
|
||||
* and calls the engine. Records each Channel as `connecting`, transitioning
|
||||
* to `online`/`setup_required`/`error` as its activation settles.
|
||||
* and calls the engine. Transient gateway failures retry with exponential
|
||||
* backoff; other outcomes transition to `online`/`setup_required`/`error`.
|
||||
*/
|
||||
activate(): void {
|
||||
// Short-circuit on BOTH latches: `activated` makes activation idempotent,
|
||||
@@ -903,26 +926,8 @@ export class ChannelManager implements ChannelsControl {
|
||||
// promise is always considered handled — ready() still sees the reason.
|
||||
settled.catch(() => {});
|
||||
|
||||
// Invoke the engine synchronously so activation is observably started the
|
||||
// moment activate() returns (callers assert the engine was called and see
|
||||
// `connecting` before awaiting ready). A synchronous config/engine throw is
|
||||
// turned into a rejected activation so it becomes this channel's status
|
||||
// rather than throwing out of activate().
|
||||
let activation: Promise<ChannelsHandle>;
|
||||
let config: ChannelActivationConfig | undefined;
|
||||
try {
|
||||
config = deriveChannelActivationConfig({
|
||||
intelligence: this.intelligence,
|
||||
channel,
|
||||
runtimeInstanceId,
|
||||
});
|
||||
activation = this.activateChannel(config, channel);
|
||||
} catch (err) {
|
||||
activation = Promise.reject(err);
|
||||
}
|
||||
|
||||
// The deferred `.then` callbacks capture `entry` and run only after the
|
||||
// literal has fully initialized, so referencing it here is safe.
|
||||
// The deferred activation callbacks capture `entry` and run only after
|
||||
// the literal has fully initialized, so referencing it there is safe.
|
||||
const entry: ChannelEntry = {
|
||||
status: "connecting",
|
||||
handle: undefined,
|
||||
@@ -930,6 +935,21 @@ export class ChannelManager implements ChannelsControl {
|
||||
settled,
|
||||
};
|
||||
|
||||
// Invoke the engine synchronously so activation is observably started the
|
||||
// moment activate() returns. Only a typed transient gateway failure is
|
||||
// retried; config errors stay on the existing terminal path.
|
||||
let activation: Promise<ChannelsHandle>;
|
||||
try {
|
||||
const config = deriveChannelActivationConfig({
|
||||
intelligence: this.intelligence,
|
||||
channel,
|
||||
runtimeInstanceId,
|
||||
});
|
||||
activation = this.activateWithRetry(config, channel, name, entry);
|
||||
} catch (err) {
|
||||
activation = Promise.reject(err);
|
||||
}
|
||||
|
||||
// Anchor the settle handlers. Both branches route every teardown through
|
||||
// the idempotent `stopEntry`, so a late settle can never resurrect a
|
||||
// `stopped` entry and a handle is torn down at most once. The handlers
|
||||
@@ -1012,6 +1032,72 @@ export class ChannelManager implements ChannelsControl {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry only transient failures from the pre-adapter gateway connection.
|
||||
* Permanent errors reject on the first attempt; teardown cancels a pending
|
||||
* timer while preserving the existing late-settle handling for in-flight work.
|
||||
*/
|
||||
private activateWithRetry(
|
||||
config: ChannelActivationConfig,
|
||||
channel: Channel,
|
||||
name: string,
|
||||
entry: ChannelEntry,
|
||||
): Promise<ChannelsHandle> {
|
||||
return new Promise<ChannelsHandle>((resolve, reject) => {
|
||||
const attempt = (): void => {
|
||||
let activation: Promise<ChannelsHandle>;
|
||||
try {
|
||||
activation = this.activateChannel(config, channel);
|
||||
} catch (err) {
|
||||
activation = Promise.reject(err);
|
||||
}
|
||||
activation.then(
|
||||
(handle) => {
|
||||
this.clearActivationRetry(entry);
|
||||
resolve(handle);
|
||||
},
|
||||
(err: unknown) => {
|
||||
if (this.stopped || !isRetryableActivationError(err)) {
|
||||
this.clearActivationRetry(entry);
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
const delayMs =
|
||||
entry.activationRetryDelayMs ?? DEFAULT_ACTIVATION_RETRY_DELAY_MS;
|
||||
entry.status = "reconnecting";
|
||||
entry.activationRetryDelayMs = Math.min(
|
||||
delayMs * 2,
|
||||
DEFAULT_ACTIVATION_RETRY_MAX_DELAY_MS,
|
||||
);
|
||||
this.log?.(
|
||||
`channel "${name}" failed to activate; retrying in ${delayMs}ms`,
|
||||
err,
|
||||
);
|
||||
const timer = setTimeout(() => {
|
||||
entry.activationRetryTimer = undefined;
|
||||
entry.cancelActivationRetry = undefined;
|
||||
if (this.stopped || entry.status === "stopped") {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
entry.status = "connecting";
|
||||
attempt();
|
||||
}, delayMs);
|
||||
(timer as unknown as { unref?: () => void }).unref?.();
|
||||
entry.activationRetryTimer = timer;
|
||||
entry.cancelActivationRetry = () => {
|
||||
this.clearActivationRetry(entry);
|
||||
reject(err);
|
||||
};
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
attempt();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw if two declared Channels share a `name`. `entries` is keyed by name,
|
||||
* so a duplicate would overwrite the first Channel's entry and leak its live
|
||||
@@ -1277,6 +1363,26 @@ export class ChannelManager implements ChannelsControl {
|
||||
entry.reconnectLogDelayMs = undefined;
|
||||
}
|
||||
|
||||
/** Cancel a pending transient activation retry and reset its backoff. */
|
||||
private clearActivationRetry(entry: ChannelEntry): void {
|
||||
if (entry.activationRetryTimer !== undefined) {
|
||||
clearTimeout(entry.activationRetryTimer);
|
||||
entry.activationRetryTimer = undefined;
|
||||
}
|
||||
entry.activationRetryDelayMs = undefined;
|
||||
entry.cancelActivationRetry = undefined;
|
||||
}
|
||||
|
||||
/** Cancel a scheduled activation retry and settle its wrapper. */
|
||||
private cancelActivationRetry(entry: ChannelEntry): void {
|
||||
const cancel = entry.cancelActivationRetry;
|
||||
if (cancel) {
|
||||
cancel();
|
||||
} else {
|
||||
this.clearActivationRetry(entry);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive a single entry to its terminal `stopped` state, tearing down its
|
||||
* handle AT MOST ONCE. Idempotent: it always sets `status = "stopped"`, and
|
||||
@@ -1314,6 +1420,7 @@ export class ChannelManager implements ChannelsControl {
|
||||
// An unref'd interval would not hold the process open, but a stopped
|
||||
// manager must not keep logging about a session it no longer owns.
|
||||
this.clearReconnectLog(entry);
|
||||
this.cancelActivationRetry(entry);
|
||||
if (entry.handle && !entry.handleStopped) {
|
||||
entry.handleStopped = true;
|
||||
const handle = entry.handle;
|
||||
|
||||
Reference in New Issue
Block a user