8465ac5ac3
## What Wires the run-ops split into the webapp: database topology, environment flags, split-mode gating, and the control-plane resolver/cache layer that the run-store and run-engine seams from the previous PR plug into. - **DB topology & env** (`apps/webapp/app/db.server.ts`, `env.server.ts`, `entry.server.tsx`): adds the run-ops database clients/topology and the environment variables that configure and gate the split. - **runOpsMigration module** (new `apps/webapp/app/v3/runOpsMigration/`): the webapp-side machinery — `splitMode.server.ts`, `controlPlaneResolver.server.ts` + `controlPlaneCache.server.ts`, `readThrough.server.ts`, `crossSeamGuard.server.ts`, `distinctDbSentinel.server.ts`, id-minting helpers (`mintBatchFriendlyId`, `runOpsMintKind`, `resolveInheritedMintKind`), `runOpsCascadeCleanup.server.ts`, the split read gate, and route/unblock catalogs. - **Store/engine wiring** (`app/v3/runStore.server.ts`, `runEngine.server.ts`, `runEngineHandlers.server.ts` + new `runEngineHandlersShared.server.ts`): points the webapp's store/engine construction at the resolver, and factors shared handler logic out so both seams use one path. - **Read-path touch-ups**: `runtimeEnvironment.server.ts`, `eventRepository/index.server.ts`, `taskRunHeartbeatFailed.server.ts`, `engineVersion.server.ts` route their run/environment lookups read-through the resolver. - `413a94511` — interlocks split mode against the native realtime backend so the two aren't enabled in an incompatible combination (see `.server-changes/run-ops-split-realtime-interlock.md`). - `dc74c57fd` — drops the earlier "known-migrated" read layer; residency is determined by id-shape only. ## Why PR5 of the run-ops split stack. This is the webapp foundation layer: it stands up the DB topology, flags, and resolver/cache the rest of the stack depends on, and repoints webapp read paths through the resolver. Additive when the split is not enabled (existing single-DB behavior preserved behind flags); behavior-changing on the read-through paths and the realtime interlock. ## Tests New vitest coverage across `apps/webapp/test/` and colocated `*.server.test.ts` files: db topology, split mode, split read gate, cross-seam guard, mint cutover / flip latency, control-plane cache, control-plane resolver, distinct-db sentinel, read-through loaders (route loaders, run-detail loaders, `findEnvironmentFromRun`), and the run-engine handlers. Testcontainers-backed; no mocks. `pnpm-lock.yaml` synced for the two new webapp deps. ## Notes Draft, **stacked on #4116** (`runops/pr04-store-engine`). Review that first; this diff is against it. Server-change / changeset note to be added at stack-assembly time. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
301 lines
9.1 KiB
TypeScript
301 lines
9.1 KiB
TypeScript
import { env } from "~/env.server";
|
|
import { eventRepository } from "./eventRepository.server";
|
|
import { type IEventRepository, type TraceEventOptions } from "./eventRepository.types";
|
|
import { prisma } from "~/db.server";
|
|
import { runStore } from "../runStore.server";
|
|
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
|
import { logger } from "~/services/logger.server";
|
|
import { FEATURE_FLAG } from "../featureFlags";
|
|
import { flag } from "../featureFlags.server";
|
|
import { getTaskEventStore } from "../taskEventStore.server";
|
|
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
|
|
|
|
export const EVENT_STORE_TYPES = {
|
|
POSTGRES: "postgres",
|
|
CLICKHOUSE: "clickhouse",
|
|
CLICKHOUSE_V2: "clickhouse_v2",
|
|
} as const;
|
|
|
|
export type EventStoreType = (typeof EVENT_STORE_TYPES)[keyof typeof EVENT_STORE_TYPES];
|
|
/**
|
|
* Async variant of {@link resolveEventRepositoryForStore}. Awaits the factory's
|
|
* registry readiness before returning the ClickHouse event repository; for
|
|
* non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed
|
|
* runs) it returns the Prisma event repository without ever touching the
|
|
* factory — so the factory never needs to know about Postgres.
|
|
*/
|
|
export async function getEventRepositoryForStore(
|
|
store: string,
|
|
organizationId: string
|
|
): Promise<IEventRepository> {
|
|
if (store !== EVENT_STORE_TYPES.CLICKHOUSE && store !== EVENT_STORE_TYPES.CLICKHOUSE_V2) {
|
|
return eventRepository;
|
|
}
|
|
const { repository } = await clickhouseFactory.getEventRepositoryForOrganization(
|
|
store,
|
|
organizationId
|
|
);
|
|
return repository;
|
|
}
|
|
|
|
export async function getConfiguredEventRepository(
|
|
organizationId: string
|
|
): Promise<{ repository: IEventRepository; store: EventStoreType }> {
|
|
const organization = await prisma.organization.findFirst({
|
|
select: {
|
|
id: true,
|
|
featureFlags: true,
|
|
},
|
|
where: {
|
|
id: organizationId,
|
|
},
|
|
});
|
|
|
|
if (!organization) {
|
|
throw new Error("Organization not found when configuring event repository");
|
|
}
|
|
|
|
// resolveTaskEventRepositoryFlag checks:
|
|
// 1. organization.featureFlags (highest priority)
|
|
// 2. global feature flags (via flags() function)
|
|
// 3. env.EVENT_REPOSITORY_DEFAULT_STORE (fallback)
|
|
const taskEventStore = await resolveTaskEventRepositoryFlag(
|
|
(organization.featureFlags as Record<string, unknown> | null) ?? undefined
|
|
);
|
|
|
|
if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE_V2) {
|
|
const { repository: resolvedRepository } =
|
|
await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
|
|
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 };
|
|
}
|
|
|
|
if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE) {
|
|
const { repository: resolvedRepository } =
|
|
await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
|
|
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE };
|
|
}
|
|
|
|
return { repository: eventRepository, store: EVENT_STORE_TYPES.POSTGRES };
|
|
}
|
|
|
|
export async function getEventRepository(
|
|
organizationId: string,
|
|
featureFlags: Record<string, unknown> | undefined,
|
|
parentStore: string | undefined
|
|
): Promise<{ repository: IEventRepository; store: string }> {
|
|
const taskEventStore = parentStore ?? (await resolveTaskEventRepositoryFlag(featureFlags));
|
|
|
|
// Non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed
|
|
// runs, or the legacy "postgres" value) resolve to the Prisma event repo.
|
|
if (
|
|
taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE &&
|
|
taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2
|
|
) {
|
|
return { repository: eventRepository, store: getTaskEventStore() };
|
|
}
|
|
|
|
const { repository: resolvedRepository } =
|
|
await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
|
|
|
|
switch (taskEventStore) {
|
|
case EVENT_STORE_TYPES.CLICKHOUSE_V2: {
|
|
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 };
|
|
}
|
|
case EVENT_STORE_TYPES.CLICKHOUSE: {
|
|
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE };
|
|
}
|
|
default: {
|
|
return { repository: eventRepository, store: getTaskEventStore() };
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function getV3EventRepository(
|
|
organizationId: string,
|
|
parentStore: string | undefined
|
|
): Promise<{ repository: IEventRepository; store: string }> {
|
|
if (typeof parentStore === "string") {
|
|
// Support legacy Postgres store for self-hosters and runs persisted with a
|
|
// non-ClickHouse store — fall back to the Prisma-based event repository.
|
|
if (
|
|
parentStore !== EVENT_STORE_TYPES.CLICKHOUSE &&
|
|
parentStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2
|
|
) {
|
|
return { repository: eventRepository, store: parentStore };
|
|
}
|
|
|
|
const { repository: resolvedRepository } =
|
|
await clickhouseFactory.getEventRepositoryForOrganization(parentStore, organizationId);
|
|
return { repository: resolvedRepository, store: parentStore };
|
|
}
|
|
|
|
if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse_v2") {
|
|
const { repository: resolvedRepository } =
|
|
await clickhouseFactory.getEventRepositoryForOrganization("clickhouse_v2", organizationId);
|
|
return { repository: resolvedRepository, store: "clickhouse_v2" };
|
|
} else if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse") {
|
|
const { repository: resolvedRepository } =
|
|
await clickhouseFactory.getEventRepositoryForOrganization("clickhouse", organizationId);
|
|
return { repository: resolvedRepository, store: "clickhouse" };
|
|
} else {
|
|
return { repository: eventRepository, store: getTaskEventStore() };
|
|
}
|
|
}
|
|
|
|
async function resolveTaskEventRepositoryFlag(
|
|
featureFlags: Record<string, unknown> | undefined
|
|
): Promise<"clickhouse" | "clickhouse_v2" | "postgres"> {
|
|
const flagResult = await flag({
|
|
key: FEATURE_FLAG.taskEventRepository,
|
|
defaultValue: env.EVENT_REPOSITORY_DEFAULT_STORE,
|
|
overrides: featureFlags,
|
|
});
|
|
|
|
if (flagResult === "clickhouse_v2") {
|
|
return "clickhouse_v2";
|
|
}
|
|
|
|
if (flagResult === "clickhouse") {
|
|
return "clickhouse";
|
|
}
|
|
|
|
return flagResult;
|
|
}
|
|
|
|
export async function recordRunDebugLog(
|
|
runId: string,
|
|
message: string,
|
|
options: Omit<TraceEventOptions, "environment" | "taskSlug" | "startTime"> & {
|
|
duration?: number;
|
|
parentId?: string;
|
|
startTime?: Date;
|
|
}
|
|
): Promise<
|
|
| {
|
|
success: true;
|
|
}
|
|
| {
|
|
success: false;
|
|
code: "RUN_NOT_FOUND" | "FAILED_TO_RECORD_EVENT";
|
|
error?: unknown;
|
|
}
|
|
> {
|
|
if (env.EVENT_REPOSITORY_DEBUG_LOGS_DISABLED) {
|
|
// drop debug events silently
|
|
return {
|
|
success: true,
|
|
};
|
|
}
|
|
|
|
return recordRunEvent(runId, message, {
|
|
...options,
|
|
attributes: {
|
|
...options?.attributes,
|
|
isDebug: true,
|
|
},
|
|
});
|
|
}
|
|
|
|
async function recordRunEvent(
|
|
runId: string,
|
|
message: string,
|
|
options: Omit<TraceEventOptions, "environment" | "taskSlug" | "startTime"> & {
|
|
duration?: number;
|
|
parentId?: string;
|
|
startTime?: Date;
|
|
}
|
|
): Promise<
|
|
| {
|
|
success: true;
|
|
}
|
|
| {
|
|
success: false;
|
|
code: "RUN_NOT_FOUND" | "FAILED_TO_RECORD_EVENT";
|
|
error?: unknown;
|
|
}
|
|
> {
|
|
try {
|
|
const foundRun = await findRunForEventCreation(runId);
|
|
|
|
if (!foundRun) {
|
|
logger.error("Failed to find run for event creation", { runId });
|
|
return {
|
|
success: false,
|
|
code: "RUN_NOT_FOUND",
|
|
};
|
|
}
|
|
|
|
const $eventRepository = await getEventRepositoryForStore(
|
|
foundRun.taskEventStore,
|
|
foundRun.runtimeEnvironment.organizationId
|
|
);
|
|
|
|
const { attributes, startTime, ...optionsRest } = options;
|
|
|
|
await $eventRepository.recordEvent(message, {
|
|
environment: foundRun.runtimeEnvironment,
|
|
taskSlug: foundRun.taskIdentifier,
|
|
context: foundRun.traceContext as Record<string, string | undefined>,
|
|
attributes: {
|
|
runId: foundRun.friendlyId,
|
|
...attributes,
|
|
},
|
|
startTime: BigInt((startTime?.getTime() ?? Date.now()) * 1_000_000),
|
|
...optionsRest,
|
|
});
|
|
|
|
return {
|
|
success: true,
|
|
};
|
|
} catch (error) {
|
|
logger.error("Failed to record event for run", {
|
|
error: error instanceof Error ? error.message : error,
|
|
runId,
|
|
});
|
|
|
|
return {
|
|
success: false,
|
|
code: "FAILED_TO_RECORD_EVENT",
|
|
error,
|
|
};
|
|
}
|
|
}
|
|
|
|
async function findRunForEventCreation(runId: string) {
|
|
const foundRun = await runStore.findRun(
|
|
{
|
|
id: runId,
|
|
},
|
|
{
|
|
select: {
|
|
friendlyId: true,
|
|
taskIdentifier: true,
|
|
traceContext: true,
|
|
taskEventStore: true,
|
|
runtimeEnvironmentId: true,
|
|
},
|
|
},
|
|
prisma
|
|
);
|
|
|
|
if (!foundRun) {
|
|
return null;
|
|
}
|
|
|
|
const environment = await controlPlaneResolver.resolveAuthenticatedEnv(
|
|
foundRun.runtimeEnvironmentId
|
|
);
|
|
|
|
if (!environment) {
|
|
// Run exists but its environment could not be resolved (e.g. a lagging replica
|
|
// under split); distinguish this from a genuinely missing run.
|
|
logger.warn("Run found but environment unresolved for event creation", {
|
|
runId,
|
|
runtimeEnvironmentId: foundRun.runtimeEnvironmentId,
|
|
});
|
|
return null;
|
|
}
|
|
|
|
return { ...foundRun, runtimeEnvironment: environment };
|
|
}
|