8465ac5ac3
## What Wires the run-ops split into the webapp: database topology, environment flags, split-mode gating, and the control-plane resolver/cache layer that the run-store and run-engine seams from the previous PR plug into. - **DB topology & env** (`apps/webapp/app/db.server.ts`, `env.server.ts`, `entry.server.tsx`): adds the run-ops database clients/topology and the environment variables that configure and gate the split. - **runOpsMigration module** (new `apps/webapp/app/v3/runOpsMigration/`): the webapp-side machinery — `splitMode.server.ts`, `controlPlaneResolver.server.ts` + `controlPlaneCache.server.ts`, `readThrough.server.ts`, `crossSeamGuard.server.ts`, `distinctDbSentinel.server.ts`, id-minting helpers (`mintBatchFriendlyId`, `runOpsMintKind`, `resolveInheritedMintKind`), `runOpsCascadeCleanup.server.ts`, the split read gate, and route/unblock catalogs. - **Store/engine wiring** (`app/v3/runStore.server.ts`, `runEngine.server.ts`, `runEngineHandlers.server.ts` + new `runEngineHandlersShared.server.ts`): points the webapp's store/engine construction at the resolver, and factors shared handler logic out so both seams use one path. - **Read-path touch-ups**: `runtimeEnvironment.server.ts`, `eventRepository/index.server.ts`, `taskRunHeartbeatFailed.server.ts`, `engineVersion.server.ts` route their run/environment lookups read-through the resolver. - `413a94511` — interlocks split mode against the native realtime backend so the two aren't enabled in an incompatible combination (see `.server-changes/run-ops-split-realtime-interlock.md`). - `dc74c57fd` — drops the earlier "known-migrated" read layer; residency is determined by id-shape only. ## Why PR5 of the run-ops split stack. This is the webapp foundation layer: it stands up the DB topology, flags, and resolver/cache the rest of the stack depends on, and repoints webapp read paths through the resolver. Additive when the split is not enabled (existing single-DB behavior preserved behind flags); behavior-changing on the read-through paths and the realtime interlock. ## Tests New vitest coverage across `apps/webapp/test/` and colocated `*.server.test.ts` files: db topology, split mode, split read gate, cross-seam guard, mint cutover / flip latency, control-plane cache, control-plane resolver, distinct-db sentinel, read-through loaders (route loaders, run-detail loaders, `findEnvironmentFromRun`), and the run-engine handlers. Testcontainers-backed; no mocks. `pnpm-lock.yaml` synced for the two new webapp deps. ## Notes Draft, **stacked on #4116** (`runops/pr04-store-engine`). Review that first; this diff is against it. Server-change / changeset note to be added at stack-assembly time. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
325 lines
11 KiB
TypeScript
325 lines
11 KiB
TypeScript
import { createReadableStreamFromReadable, type EntryContext } from "@remix-run/node"; // or cloudflare/deno
|
|
import { RemixServer } from "@remix-run/react";
|
|
import * as Sentry from "@sentry/remix";
|
|
import { wrapHandleErrorWithSentry } from "@sentry/remix";
|
|
import { addTenantContextToEvent } from "~/utils/sentryTenantContext.server";
|
|
import { parseAcceptLanguage } from "intl-parse-accept-language";
|
|
import isbot from "isbot";
|
|
import { renderToPipeableStream } from "react-dom/server";
|
|
import { PassThrough } from "stream";
|
|
import * as Worker from "~/services/worker.server";
|
|
import { initMollifierDrainerWorker } from "~/v3/mollifierDrainerWorker.server";
|
|
import { initMollifierStaleSweepWorker } from "~/v3/mollifierStaleSweepWorker.server";
|
|
import { initBillingLimitWorker } from "~/v3/billingLimitWorker.server";
|
|
import { bootstrap } from "./bootstrap";
|
|
import { LocaleContextProvider } from "./components/primitives/LocaleProvider";
|
|
import type { OperatingSystemPlatform } from "./components/primitives/OperatingSystemProvider";
|
|
import { OperatingSystemContextProvider } from "./components/primitives/OperatingSystemProvider";
|
|
import { assertRunOpsSplitSentinel, Prisma } from "./db.server";
|
|
import { env } from "./env.server";
|
|
import { eventLoopMonitor } from "./eventLoopMonitor.server";
|
|
import { logger } from "./services/logger.server";
|
|
import { resourceMonitor } from "./services/resourceMonitor.server";
|
|
import { singleton } from "./utils/singleton";
|
|
import { remoteBuildsEnabled } from "./v3/remoteImageBuilder.server";
|
|
import {
|
|
registerRunEngineEventBusHandlers,
|
|
setupBatchQueueCallbacks,
|
|
} from "./v3/runEngineHandlers.server";
|
|
import { registerRunChangeNotifierHandlers } from "./services/realtime/runChangeNotifierHandlers.server";
|
|
// Touch the sessions replication singleton at entry so it boots deterministically
|
|
// on webapp startup. The singleton's initializer wires start (gated on
|
|
// `clickhouseFactory.isReady()`) and SIGTERM/SIGINT shutdown — mirrors
|
|
// runsReplicationInstance.
|
|
//
|
|
// IMPORTANT: do NOT replace this with `void sessionsReplicationInstance;`.
|
|
// `apps/webapp/package.json` declares `"sideEffects": false`, so esbuild
|
|
// treats `void <identifier>;` as a pure expression statement and tree-shakes
|
|
// the entire import — the singleton's initializer never fires and the
|
|
// sessions→ClickHouse logical replication slot stops being consumed. Assigning
|
|
// to globalThis is an unambiguous side effect the bundler must preserve. See
|
|
// TRI-9864 for the incident write-up.
|
|
import { sessionsReplicationInstance } from "./services/sessionsReplicationInstance.server";
|
|
(globalThis as Record<string, unknown>).__sessionsReplicationInstance = sessionsReplicationInstance;
|
|
import { globalFlagsRegistry } from "./v3/globalFlagsRegistry.server";
|
|
(globalThis as Record<string, unknown>).__globalFlagsRegistry = globalFlagsRegistry;
|
|
import { workerRegionRegistry } from "./v3/workerRegions.server";
|
|
(globalThis as Record<string, unknown>).__workerRegionRegistry = workerRegionRegistry;
|
|
|
|
const ABORT_DELAY = 30000;
|
|
|
|
export default function handleRequest(
|
|
request: Request,
|
|
responseStatusCode: number,
|
|
responseHeaders: Headers,
|
|
remixContext: EntryContext
|
|
) {
|
|
const url = new URL(request.url);
|
|
|
|
if (url.pathname.startsWith("/login")) {
|
|
responseHeaders.set("X-Frame-Options", "SAMEORIGIN");
|
|
responseHeaders.set("Content-Security-Policy", "frame-ancestors 'self'");
|
|
}
|
|
|
|
const acceptLanguage = request.headers.get("accept-language");
|
|
const locales = parseAcceptLanguage(acceptLanguage, {
|
|
validate: Intl.DateTimeFormat.supportedLocalesOf,
|
|
});
|
|
|
|
//get whether it's a mac or pc from the headers
|
|
const platform: OperatingSystemPlatform = request.headers.get("user-agent")?.includes("Mac")
|
|
? "mac"
|
|
: "windows";
|
|
|
|
// If the request is from a bot, we want to wait for the full
|
|
// response to render before sending it to the client. This
|
|
// ensures that bots can see the full page content.
|
|
if (isbot(request.headers.get("user-agent"))) {
|
|
return handleBotRequest(
|
|
request,
|
|
responseStatusCode,
|
|
responseHeaders,
|
|
remixContext,
|
|
locales,
|
|
platform
|
|
);
|
|
}
|
|
|
|
return handleBrowserRequest(
|
|
request,
|
|
responseStatusCode,
|
|
responseHeaders,
|
|
remixContext,
|
|
locales,
|
|
platform
|
|
);
|
|
}
|
|
|
|
function handleBotRequest(
|
|
request: Request,
|
|
responseStatusCode: number,
|
|
responseHeaders: Headers,
|
|
remixContext: EntryContext,
|
|
locales: string[],
|
|
platform: OperatingSystemPlatform
|
|
) {
|
|
return new Promise((resolve, reject) => {
|
|
let shellRendered = false;
|
|
// Timer handle is cleared in every terminal callback so the abort closure
|
|
// (which captures the full React render tree + remixContext) doesn't pin
|
|
// memory for 30s per successful request. See react-router PR #14200.
|
|
let abortTimer: NodeJS.Timeout | undefined;
|
|
const { pipe, abort } = renderToPipeableStream(
|
|
<OperatingSystemContextProvider platform={platform}>
|
|
<LocaleContextProvider locales={locales}>
|
|
<RemixServer context={remixContext} url={request.url} abortDelay={ABORT_DELAY} />,
|
|
</LocaleContextProvider>
|
|
</OperatingSystemContextProvider>,
|
|
{
|
|
onAllReady() {
|
|
shellRendered = true;
|
|
const body = new PassThrough();
|
|
const stream = createReadableStreamFromReadable(body);
|
|
|
|
responseHeaders.set("Content-Type", "text/html");
|
|
|
|
resolve(
|
|
new Response(stream, {
|
|
headers: responseHeaders,
|
|
status: responseStatusCode,
|
|
})
|
|
);
|
|
|
|
pipe(body);
|
|
clearTimeout(abortTimer);
|
|
},
|
|
onShellError(error: unknown) {
|
|
clearTimeout(abortTimer);
|
|
reject(error);
|
|
},
|
|
onError(error: unknown) {
|
|
responseStatusCode = 500;
|
|
// Log streaming rendering errors from inside the shell. Don't log
|
|
// errors encountered during initial shell rendering since they'll
|
|
// reject and get logged in handleDocumentRequest.
|
|
if (shellRendered) {
|
|
console.error(error);
|
|
}
|
|
},
|
|
}
|
|
);
|
|
|
|
abortTimer = setTimeout(abort, ABORT_DELAY);
|
|
});
|
|
}
|
|
|
|
function handleBrowserRequest(
|
|
request: Request,
|
|
responseStatusCode: number,
|
|
responseHeaders: Headers,
|
|
remixContext: EntryContext,
|
|
locales: string[],
|
|
platform: OperatingSystemPlatform
|
|
) {
|
|
return new Promise((resolve, reject) => {
|
|
let shellRendered = false;
|
|
// Timer handle is cleared in every terminal callback so the abort closure
|
|
// (which captures the full React render tree + remixContext) doesn't pin
|
|
// memory for 30s per successful request. See react-router PR #14200.
|
|
let abortTimer: NodeJS.Timeout | undefined;
|
|
const { pipe, abort } = renderToPipeableStream(
|
|
<OperatingSystemContextProvider platform={platform}>
|
|
<LocaleContextProvider locales={locales}>
|
|
<RemixServer context={remixContext} url={request.url} abortDelay={ABORT_DELAY} />
|
|
</LocaleContextProvider>
|
|
</OperatingSystemContextProvider>,
|
|
{
|
|
onShellReady() {
|
|
shellRendered = true;
|
|
const body = new PassThrough();
|
|
const stream = createReadableStreamFromReadable(body);
|
|
|
|
responseHeaders.set("Content-Type", "text/html");
|
|
|
|
resolve(
|
|
new Response(stream, {
|
|
headers: responseHeaders,
|
|
status: responseStatusCode,
|
|
})
|
|
);
|
|
|
|
pipe(body);
|
|
clearTimeout(abortTimer);
|
|
},
|
|
onShellError(error: unknown) {
|
|
clearTimeout(abortTimer);
|
|
reject(error);
|
|
},
|
|
onError(error: unknown) {
|
|
responseStatusCode = 500;
|
|
// Log streaming rendering errors from inside the shell. Don't log
|
|
// errors encountered during initial shell rendering since they'll
|
|
// reject and get logged in handleDocumentRequest.
|
|
if (shellRendered) {
|
|
console.error(error);
|
|
}
|
|
},
|
|
}
|
|
);
|
|
|
|
abortTimer = setTimeout(abort, ABORT_DELAY);
|
|
});
|
|
}
|
|
|
|
export const handleError = wrapHandleErrorWithSentry((error, { request }) => {
|
|
if (request instanceof Request) {
|
|
logger.debug("Error in handleError", {
|
|
error,
|
|
request: {
|
|
url: request.url,
|
|
method: request.method,
|
|
},
|
|
});
|
|
} else {
|
|
logger.debug("Error in handleError", {
|
|
error,
|
|
});
|
|
}
|
|
});
|
|
|
|
Worker.init().catch((error) => {
|
|
logError(error);
|
|
});
|
|
|
|
initMollifierDrainerWorker();
|
|
initMollifierStaleSweepWorker();
|
|
initBillingLimitWorker();
|
|
|
|
bootstrap().catch((error) => {
|
|
logError(error);
|
|
});
|
|
|
|
function logError(error: unknown, request?: Request) {
|
|
console.error(error);
|
|
|
|
if (error instanceof Error && error.message.startsWith("There are locked jobs present")) {
|
|
console.log("⚠️ graphile-worker migration issue detected!");
|
|
}
|
|
}
|
|
|
|
process.on("uncaughtException", (error, origin) => {
|
|
if (
|
|
error instanceof Prisma.PrismaClientKnownRequestError ||
|
|
error instanceof Prisma.PrismaClientUnknownRequestError
|
|
) {
|
|
// Don't exit the process if the error is a Prisma error
|
|
logger.error("uncaughtException prisma error", {
|
|
error,
|
|
prismaMessage: error.message,
|
|
code: "code" in error ? error.code : undefined,
|
|
meta: "meta" in error ? error.meta : undefined,
|
|
stack: error.stack,
|
|
origin,
|
|
});
|
|
} else {
|
|
logger.error("uncaughtException", {
|
|
error: { name: error.name, message: error.message, stack: error.stack },
|
|
origin,
|
|
});
|
|
}
|
|
|
|
process.exit(1);
|
|
});
|
|
|
|
// Boot-time run-ops split interlock. Async, so it runs as a
|
|
// fire-and-forget at startup; a flag-on-but-sentinel-fails misconfig crashes
|
|
// the process loudly before any run-ops routing is wired.
|
|
singleton("AssertRunOpsSplitSentinel", () => {
|
|
assertRunOpsSplitSentinel().catch((error) => {
|
|
logger.error("Run-ops split sentinel assertion failed; refusing to start", { error });
|
|
process.exit(1);
|
|
});
|
|
return true;
|
|
});
|
|
|
|
singleton("RunEngineEventBusHandlers", registerRunEngineEventBusHandlers);
|
|
singleton("SetupBatchQueueCallbacks", setupBatchQueueCallbacks);
|
|
// Attach the realtime run-changed publish delegations to the engine event bus.
|
|
// No-ops (registers nothing) unless REALTIME_BACKEND_NATIVE_ENABLED=1.
|
|
singleton("RunChangeNotifierHandlers", registerRunChangeNotifierHandlers);
|
|
|
|
// Wrapped in singleton() so Remix's dev-mode CJS reloads don't append
|
|
// duplicate copies of the processor — Sentry's processor list lives in
|
|
// node_modules and persists across module reloads. Idempotent at runtime
|
|
// (the processor is a pure read+stamp), but the pattern matches the rest
|
|
// of this file.
|
|
singleton("SentryTenantContextProcessor", () => {
|
|
if (env.SENTRY_DSN) {
|
|
Sentry.addEventProcessor(addTenantContextToEvent);
|
|
}
|
|
// Return a truthy value — `singleton()` uses `??=` so a `void`
|
|
// callback would re-execute (and re-register) on every dev reload.
|
|
return true;
|
|
});
|
|
|
|
export { apiRateLimiter } from "./services/apiRateLimit.server";
|
|
export { engineRateLimiter } from "./services/engineRateLimit.server";
|
|
export { runWithHttpContext } from "./services/httpAsyncStorage.server";
|
|
export { tenantContextMiddleware } from "./services/tenantContextResolver.server";
|
|
export { socketIo } from "./v3/handleSocketIo.server";
|
|
export { wss } from "./v3/handleWebsockets.server";
|
|
|
|
if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
|
|
eventLoopMonitor.enable();
|
|
}
|
|
|
|
if (remoteBuildsEnabled()) {
|
|
console.log("🏗️ Remote builds enabled");
|
|
} else {
|
|
console.log("🏗️ Local builds enabled");
|
|
}
|
|
|
|
if (env.RESOURCE_MONITOR_ENABLED === "1") {
|
|
resourceMonitor.startMonitoring(1000);
|
|
}
|