github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Support for org-scoped ClickHouse (#3333)

Browse Source 
Added `OrganizationDataStore` which allows orgs to have data stored in
specific separate services.

For now this is just used for ClickHouse. When using ClickHouse we get a
client for the factory and pass in the org id.

Particular care has to be made with two hot-insert paths:
1. RunReplicationService
2. OTLPExporter

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Matt Aitken
2026-05-22 14:43:03 +01:00
committed by GitHub
parent 832cf7220b
commit 71d98b4e6b
87 changed files with 2914 additions and 1055 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cursor/mcp.json
+5 -1
View File
@@ -1,3 +1,7 @@
{
"mcpServers": {}
"mcpServers": {
"linear": {
"url": "https://mcp.linear.app/mcp"
}
}
}
.server-changes/organization-scoped-clickhouse.md
+6
View File
@@ -0,0 +1,6 @@
---
area: webapp
type: feature
---
Organization-scoped ClickHouse routing enables customers with HIPAA and other data security requirements to use dedicated database instances
CLAUDE.md
+11
View File
@@ -69,6 +69,17 @@ containerTest("should use both", async ({ prisma, redisOptions }) => {
});
```
## Code Style
### Imports
**Prefer static imports over dynamic imports.** Only use dynamic `import()` when:
- Circular dependencies cannot be resolved otherwise
- Code splitting is genuinely needed for performance
- The module must be loaded conditionally at runtime
Dynamic imports add unnecessary overhead in hot paths and make code harder to analyze. If you find yourself using `await import()`, ask if a regular `import` statement would work instead.
## Changesets and Server Changes
When modifying any public package (`packages/*` or `integrations/*`), add a changeset:
apps/webapp/app/entry.server.tsx
+5 -37
View File
@@ -24,44 +24,12 @@ import {
registerRunEngineEventBusHandlers,
setupBatchQueueCallbacks,
} from "./v3/runEngineHandlers.server";
// Touch the sessions replication singleton at entry so it boots deterministically
// on webapp startup. The singleton's initializer wires start (gated on
// `clickhouseFactory.isReady()`) and SIGTERM/SIGINT shutdown — mirrors
// runsReplicationInstance.
import { sessionsReplicationInstance } from "./services/sessionsReplicationInstance.server";
import { signalsEmitter } from "./services/signals.server";
// Start the sessions replication service (subscribes to the logical replication
// slot, runs leader election, flushes to ClickHouse). Done at entry level so it
// runs deterministically on webapp boot rather than lazily via a singleton
// reference elsewhere in the module graph.
if (sessionsReplicationInstance && env.SESSION_REPLICATION_ENABLED === "1") {
// Capture a non-nullable reference so the shutdown closure below
// doesn't need to re-null-check (TS narrowing doesn't follow through
// an inner function scope).
const replicator = sessionsReplicationInstance;
replicator
.start()
.then(() => {
console.log("🗃️ Sessions replication service started");
})
.catch((error) => {
console.error("🗃️ Sessions replication service failed to start", {
error,
});
});
// Wrap the async shutdown in a sync handler that catches rejections —
// SIGTERM/SIGINT fire during process teardown, and an unhandled
// promise rejection from `_replicationClient.stop()` there would
// bubble up past the process exit. Matches the pattern in
// dynamicFlushScheduler.server.ts.
const shutdownSessionsReplication = () => {
replicator.shutdown().catch((error) => {
console.error("🗃️ Sessions replication service shutdown error", {
error,
});
});
};
signalsEmitter.on("SIGTERM", shutdownSessionsReplication);
signalsEmitter.on("SIGINT", shutdownSessionsReplication);
}
void sessionsReplicationInstance;
const ABORT_DELAY = 30000;
apps/webapp/app/env.server.ts
+14 -2
View File
@@ -459,7 +459,10 @@ const EnvironmentSchema = z
// If specified, you must configure the corresponding provider using OBJECT_STORE_{PROTOCOL}_* env vars.
// Example: OBJECT_STORE_DEFAULT_PROTOCOL=s3 requires OBJECT_STORE_S3_BASE_URL, OBJECT_STORE_S3_ACCESS_KEY_ID, etc.
// Enables zero-downtime migration between providers (old data keeps working, new data uses new provider).
OBJECT_STORE_DEFAULT_PROTOCOL: z.string().regex(/^[a-z0-9]+$/).optional(),
OBJECT_STORE_DEFAULT_PROTOCOL: z
.string()
.regex(/^[a-z0-9]+$/)
.optional(),
ARTIFACTS_OBJECT_STORE_BUCKET: z.string().optional(),
ARTIFACTS_OBJECT_STORE_BASE_URL: z.string().optional(),
@@ -1489,9 +1492,18 @@ const EnvironmentSchema = z
EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(5_000),
EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING: z.coerce.number().int().default(2000),
// Organization data stores registry
ORGANIZATION_DATA_STORES_RELOAD_INTERVAL_MS: z.coerce
.number()
.int()
.default(60 * 1000), // 1 minute
// LLM cost tracking
LLM_COST_TRACKING_ENABLED: BoolEnv.default(true),
LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes
LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce
.number()
.int()
.default(5 * 60 * 1000), // 5 minutes
LLM_PRICING_RELOAD_CHANNEL: z.string().default("llm-registry:reload"),
LLM_PRICING_RELOAD_DEBOUNCE_MS: z.coerce.number().int().default(1000),
// Whether to subscribe this process to the LLM_PRICING_RELOAD_CHANNEL.
apps/webapp/app/presenters/v3/AgentListPresenter.server.ts
+21 -15
View File
@@ -3,10 +3,10 @@ import {
type RuntimeEnvironmentType,
type TaskTriggerSource,
} from "@trigger.dev/database";
import { ClickHouse } from "@internal/clickhouse";
import { type ClickHouse } from "@internal/clickhouse";
import { z } from "zod";
import { $replica } from "~/db.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { singleton } from "~/utils/singleton";
import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server";
@@ -24,10 +24,7 @@ export type AgentActiveState = {
};
export class AgentListPresenter {
constructor(
private readonly clickhouse: ClickHouse,
private readonly _replica: PrismaClientOrTransaction
) {}
constructor(private readonly _replica: PrismaClientOrTransaction) {}
public async call({
organizationId,
@@ -40,6 +37,11 @@ export class AgentListPresenter {
environmentId: string;
environmentType: RuntimeEnvironmentType;
}) {
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
organizationId,
"standard"
);
const currentWorker = await findCurrentWorkerFromEnvironment(
{
id: environmentId,
@@ -89,20 +91,21 @@ export class AgentListPresenter {
}
// All queries are deferred for streaming
const activeStates = this.#getActiveStates(environmentId, slugs);
const conversationSparklines = this.#getConversationSparklines(environmentId, slugs);
const costSparklines = this.#getCostSparklines(environmentId, slugs);
const tokenSparklines = this.#getTokenSparklines(environmentId, slugs);
const activeStates = this.#getActiveStates(clickhouse, environmentId, slugs);
const conversationSparklines = this.#getConversationSparklines(clickhouse, environmentId, slugs);
const costSparklines = this.#getCostSparklines(clickhouse, environmentId, slugs);
const tokenSparklines = this.#getTokenSparklines(clickhouse, environmentId, slugs);
return { agents, activeStates, conversationSparklines, costSparklines, tokenSparklines };
}
/** Count runs currently executing vs suspended per agent */
async #getActiveStates(
clickhouse: ClickHouse,
environmentId: string,
slugs: string[]
): Promise<Record<string, AgentActiveState>> {
const queryFn = this.clickhouse.reader.query({
const queryFn = clickhouse.reader.query({
name: "agentActiveStates",
query: `SELECT
task_identifier,
@@ -140,10 +143,11 @@ export class AgentListPresenter {
/** 24h hourly sparkline of conversation (run) count per agent */
async #getConversationSparklines(
clickhouse: ClickHouse,
environmentId: string,
slugs: string[]
): Promise<Record<string, number[]>> {
const queryFn = this.clickhouse.reader.query({
const queryFn = clickhouse.reader.query({
name: "agentConversationSparklines",
query: `SELECT
task_identifier,
@@ -172,10 +176,11 @@ export class AgentListPresenter {
/** 24h hourly sparkline of LLM cost per agent */
async #getCostSparklines(
clickhouse: ClickHouse,
environmentId: string,
slugs: string[]
): Promise<Record<string, number[]>> {
const queryFn = this.clickhouse.reader.query({
const queryFn = clickhouse.reader.query({
name: "agentCostSparklines",
query: `SELECT
task_identifier,
@@ -203,10 +208,11 @@ export class AgentListPresenter {
/** 24h hourly sparkline of total tokens per agent */
async #getTokenSparklines(
clickhouse: ClickHouse,
environmentId: string,
slugs: string[]
): Promise<Record<string, number[]>> {
const queryFn = this.clickhouse.reader.query({
const queryFn = clickhouse.reader.query({
name: "agentTokenSparklines",
query: `SELECT
task_identifier,
@@ -284,5 +290,5 @@ export class AgentListPresenter {
export const agentListPresenter = singleton("agentListPresenter", setupAgentListPresenter);
function setupAgentListPresenter() {
return new AgentListPresenter(clickhouseClient, $replica);
return new AgentListPresenter($replica);
}
apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts
+3 -2
View File
@@ -9,7 +9,7 @@ import { type Project, type RuntimeEnvironment, type TaskRunStatus } from "@trig
import assertNever from "assert-never";
import { z } from "zod";
import { API_VERSIONS, RunStatusUnspecifiedApiVersion } from "~/api/versions";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { logger } from "~/services/logger.server";
import { CoercedDate } from "~/utils/zod";
import { ServiceValidationError } from "~/v3/services/baseService.server";
@@ -269,7 +269,8 @@ export class ApiRunListPresenter extends BasePresenter {
options.machines = searchParams["filter[machine]"];
}
const presenter = new NextRunListPresenter(this._replica, clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard");
const presenter = new NextRunListPresenter(this._replica, clickhouse);
logger.debug("Calling RunListPresenter", { options });
apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts
+3 -2
View File
@@ -1,6 +1,6 @@
import { type PrismaClient } from "@trigger.dev/database";
import { CreateBulkActionSearchParams } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
import { getRunFiltersFromRequest } from "../RunFilters.server";
import { BasePresenter } from "./basePresenter.server";
@@ -24,8 +24,9 @@ export class CreateBulkActionPresenter extends BasePresenter {
Object.fromEntries(new URL(request.url).searchParams)
);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard");
const runsRepository = new RunsRepository({
clickhouse: clickhouseClient,
clickhouse,
prisma: this._replica as PrismaClient,
});
apps/webapp/app/presenters/v3/RunPresenter.server.ts
+7 -4
View File
@@ -3,11 +3,11 @@ import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/Tr
import { prisma, type PrismaClient } from "~/db.server";
import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
import { getUsername } from "~/utils/username";
import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { SpanSummary } from "~/v3/eventRepository/eventRepository.types";
import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
import { isFinalRunStatus } from "~/v3/taskStatus";
import { env } from "~/env.server";
import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
type Result = Awaited<ReturnType<RunPresenter["call"]>>;
export type Run = Result["run"];
@@ -145,10 +145,13 @@ export class RunPresenter {
};
}
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
const repository = await getEventRepositoryForStore(
run.taskEventStore,
run.runtimeEnvironment.organizationId
);
// get the events
let traceSummary = await eventRepository.getTraceSummary(
let traceSummary = await repository.getTraceSummary(
getTaskEventStoreTableForRun(run),
run.runtimeEnvironment.id,
run.traceId,
@@ -272,7 +275,7 @@ export class RunPresenter {
overridesBySpanId: traceSummary.overridesBySpanId,
linkedRunIdBySpanId,
},
maximumLiveReloadingSetting: eventRepository.maximumLiveReloadingSetting,
maximumLiveReloadingSetting: repository.maximumLiveReloadingSetting,
};
}
}
apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts
+3 -2
View File
@@ -1,6 +1,6 @@
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
import { BasePresenter } from "./basePresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { type PrismaClient } from "@trigger.dev/database";
import { timeFilters } from "~/components/runs/v3/SharedFilters";
@@ -37,8 +37,9 @@ export class RunTagListPresenter extends BasePresenter {
}: TagListOptions) {
const hasFilters = Boolean(name?.trim());
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard");
const runsRepository = new RunsRepository({
clickhouse: clickhouseClient,
clickhouse,
prisma: this._replica as PrismaClient,
});
apps/webapp/app/presenters/v3/SpanPresenter.server.ts
+7 -4
View File
@@ -23,7 +23,6 @@ import { isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus";
import { BasePresenter } from "./basePresenter.server";
import { WaitpointPresenter } from "./WaitpointPresenter.server";
import { engine } from "~/v3/runEngine.server";
import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { IEventRepository, SpanDetail } from "~/v3/eventRepository/eventRepository.types";
import { safeJsonParse } from "~/utils/json";
import {
@@ -32,6 +31,7 @@ import {
extractAIToolCallData,
extractAIEmbedData,
} from "~/components/runs/v3/ai";
import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
export type PromptSpanData = {
slug: string;
@@ -132,14 +132,17 @@ export class SpanPresenter extends BasePresenter {
const { traceId } = parentRun;
const eventRepository = resolveEventRepositoryForStore(parentRun.taskEventStore);
const repository = await getEventRepositoryForStore(
parentRun.taskEventStore,
project.organizationId
);
const eventStore = getTaskEventStoreTableForRun(parentRun);
const run = await this.getRun({
eventStore,
traceId,
eventRepository,
eventRepository: repository,
spanId,
linkedRunId,
createdAt: parentRun.createdAt,
@@ -161,7 +164,7 @@ export class SpanPresenter extends BasePresenter {
projectId: parentRun.projectId,
createdAt: parentRun.createdAt,
completedAt: parentRun.completedAt,
eventRepository,
eventRepository: repository,
});
if (!span) {
apps/webapp/app/presenters/v3/TaskListPresenter.server.ts
+12 -13
View File
@@ -4,7 +4,7 @@ import {
type TaskTriggerSource,
} from "@trigger.dev/database";
import { $replica } from "~/db.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import {
type AverageDurations,
ClickHouseEnvironmentMetricsRepository,
@@ -25,10 +25,7 @@ export type TaskListItem = {
export type TaskActivity = DailyTaskActivity[string];
export class TaskListPresenter {
constructor(
private readonly environmentMetricsRepository: EnvironmentMetricsRepository,
private readonly _replica: PrismaClientOrTransaction
) {}
constructor(private readonly _replica: PrismaClientOrTransaction) {}
public async call({
organizationId,
@@ -77,9 +74,15 @@ export class TaskListPresenter {
const slugs = tasks.map((t) => t.slug);
// Create org-specific environment metrics repository
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard");
const environmentMetricsRepository = new ClickHouseEnvironmentMetricsRepository({
clickhouse,
});
// IMPORTANT: Don't await these, we want to return the promises
// so we can defer the loading of the data
const activity = this.environmentMetricsRepository.getDailyTaskActivity({
const activity = environmentMetricsRepository.getDailyTaskActivity({
organizationId,
projectId,
environmentId,
@@ -87,7 +90,7 @@ export class TaskListPresenter {
tasks: slugs,
});
const runningStats = this.environmentMetricsRepository.getCurrentRunningStats({
const runningStats = environmentMetricsRepository.getCurrentRunningStats({
organizationId,
projectId,
environmentId,
@@ -95,7 +98,7 @@ export class TaskListPresenter {
tasks: slugs,
});
const durations = this.environmentMetricsRepository.getAverageDurations({
const durations = environmentMetricsRepository.getAverageDurations({
organizationId,
projectId,
environmentId,
@@ -110,9 +113,5 @@ export class TaskListPresenter {
export const taskListPresenter = singleton("taskListPresenter", setupTaskListPresenter);
function setupTaskListPresenter() {
const environmentMetricsRepository = new ClickHouseEnvironmentMetricsRepository({
clickhouse: clickhouseClient,
});
return new TaskListPresenter(environmentMetricsRepository, $replica);
return new TaskListPresenter($replica);
}
apps/webapp/app/presenters/v3/UsagePresenter.server.ts
+3 -2
View File
@@ -4,7 +4,7 @@ import { getUsage, getUsageSeries } from "~/services/platform.v3.server";
import { createTimeSeriesData } from "~/utils/graphs";
import { BasePresenter } from "./basePresenter.server";
import { DataPoint, linear } from "regression";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
type Options = {
organizationId: string;
@@ -124,7 +124,8 @@ async function getTaskUsageByOrganization(
endOfMonth: Date,
replica: PrismaClientOrTransaction
) {
const [queryError, tasks] = await clickhouseClient.taskRuns.getTaskUsageByOrganization({
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard");
const [queryError, tasks] = await clickhouse.taskRuns.getTaskUsageByOrganization({
startTime: startOfMonth.getTime(),
endTime: endOfMonth.getTime(),
organizationId,
apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts
+3 -2
View File
@@ -1,7 +1,7 @@
import { ScheduleObject } from "@trigger.dev/core/v3";
import { PrismaClient, prisma } from "~/db.server";
import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { nextScheduledTimestamps } from "~/v3/utils/calculateNextSchedule.server";
import { NextRunListPresenter } from "./NextRunListPresenter.server";
import { scheduleWhereClause } from "~/models/schedules.server";
@@ -75,7 +75,8 @@ export class ViewSchedulePresenter {
? nextScheduledTimestamps(schedule.generatorExpression, schedule.timezone, new Date(), 5)
: [];
const runPresenter = new NextRunListPresenter(this.#prismaClient, clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(schedule.project.organizationId, "standard");
const runPresenter = new NextRunListPresenter(this.#prismaClient, clickhouse);
const { runs } = await runPresenter.call(schedule.project.organizationId, environmentId, {
projectId: schedule.project.id,
scheduleId: schedule.id,
apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts
+3 -2
View File
@@ -1,5 +1,5 @@
import { isWaitpointOutputTimeout, prettyPrintPacket } from "@trigger.dev/core/v3";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { generateHttpCallbackUrl } from "~/services/httpCallback.server";
import { logger } from "~/services/logger.server";
import { BasePresenter } from "./basePresenter.server";
@@ -79,7 +79,8 @@ export class WaitpointPresenter extends BasePresenter {
const connectedRuns: NextRunListItem[] = [];
if (connectedRunIds.length > 0) {
const runPresenter = new NextRunListPresenter(this._prisma, clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(waitpoint.environment.organizationId, "standard");
const runPresenter = new NextRunListPresenter(this._prisma, clickhouse);
const { runs } = await runPresenter.call(
waitpoint.environment.organizationId,
environmentId,
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx
+5 -3
View File
@@ -34,7 +34,7 @@ import {
MetricDashboardPresenter,
} from "~/presenters/v3/MetricDashboardPresenter.server";
import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUser } from "~/services/session.server";
import { cn } from "~/utils/cn";
import { EnvironmentParamSchema } from "~/utils/pathBuilder";
@@ -77,10 +77,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const filters = dashboard.filters ?? ["tasks", "queues"];
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
// Load distinct models from ClickHouse if the dashboard has a models filter
let possibleModels: { model: string; system: string }[] = [];
if (filters.includes("models")) {
const queryFn = clickhouseClient.reader.query({
const queryFn = clickhouse.reader.query({
name: "getDistinctModels",
query: `SELECT response_model, any(gen_ai_system) AS gen_ai_system FROM trigger_dev.llm_metrics_v1 WHERE organization_id = {organizationId: String} AND project_id = {projectId: String} AND environment_id = {environmentId: String} AND response_model != '' GROUP BY response_model ORDER BY response_model`,
params: z.object({
@@ -100,7 +102,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
}
}
const promptPresenter = new PromptPresenter(clickhouseClient);
const promptPresenter = new PromptPresenter(clickhouse);
const [possiblePrompts, possibleOperations, possibleProviders] = await Promise.all([
filters.includes("prompts")
? promptPresenter.getDistinctPromptSlugs(project.organizationId, project.id, environment.id)
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx
+69 -62
View File
@@ -1,56 +1,16 @@
import { type LoaderFunctionArgs, type ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { type MetaFunction, useFetcher, useRevalidator } from "@remix-run/react";
import { parse } from "@conform-to/zod";
import { BellAlertIcon } from "@heroicons/react/20/solid";
import { type MetaFunction, useFetcher, useRevalidator } from "@remix-run/react";
import { type ActionFunctionArgs, json, type LoaderFunctionArgs } from "@remix-run/server-runtime";
import {
IconAlarmSnooze as IconAlarmSnoozeBase,
IconBugFilled,
IconCircleDotted,
} from "@tabler/icons-react";
import { parse } from "@conform-to/zod";
import { z } from "zod";
import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge";
import { ServiceValidationError } from "~/v3/services/baseService.server";
import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
import { requireUser, requireUserId } from "~/services/session.server";
import {
EnvironmentParamSchema,
v3CreateBulkActionPath,
v3ErrorsPath,
v3RunsPath,
} from "~/utils/pathBuilder";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import {
ErrorGroupPresenter,
type ErrorGroupActivity,
type ErrorGroupActivityVersions,
type ErrorGroupOccurrences,
type ErrorGroupSummary,
type ErrorGroupState,
} from "~/presenters/v3/ErrorGroupPresenter.server";
import { type NextRunList } from "~/presenters/v3/NextRunListPresenter.server";
import { $replica } from "~/db.server";
import { logsClickhouseClient, clickhouseClient } from "~/services/clickhouseInstance.server";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { PageBody } from "~/components/layout/AppLayout";
import {
ResizableHandle,
ResizablePanel,
ResizablePanelGroup,
} from "~/components/primitives/Resizable";
import { ErrorId } from "@trigger.dev/core/v3/isomorphic";
import { isPast } from "date-fns";
import { AnimatePresence, motion } from "framer-motion";
import { Suspense, useEffect, useMemo, useRef, useState } from "react";
import { Spinner } from "~/components/primitives/Spinner";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Callout } from "~/components/primitives/Callout";
import { Header2, Header3 } from "~/components/primitives/Headers";
import { formatDistanceToNow, isPast } from "date-fns";
import * as Property from "~/components/primitives/PropertyTable";
import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable";
import { DateTime, RelativeDateTime } from "~/components/primitives/DateTime";
import { ErrorId } from "@trigger.dev/core/v3/isomorphic";
import {
Bar,
BarChart,
@@ -61,31 +21,68 @@ import {
XAxis,
YAxis,
} from "recharts";
import TooltipPortal from "~/components/primitives/TooltipPortal";
import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters";
import { useOptimisticLocation } from "~/hooks/useOptimisticLocation";
import { DirectionSchema, ListPagination } from "~/components/ListPagination";
import { Button, LinkButton } from "~/components/primitives/Buttons";
import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
import { z } from "zod";
import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon";
import { useOrganization } from "~/hooks/useOrganizations";
import { useProject } from "~/hooks/useProject";
import { useEnvironment } from "~/hooks/useEnvironment";
import { RunsIcon } from "~/assets/icons/RunsIcon";
import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
import { useSearchParams } from "~/hooks/useSearchParam";
import { CopyableText } from "~/components/primitives/CopyableText";
import { cn } from "~/utils/cn";
import { LogsVersionFilter } from "~/components/logs/LogsVersionFilter";
import { CodeBlock } from "~/components/code/CodeBlock";
import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
import { ErrorGroupActions } from "~/v3/services/errorGroupActions.server";
import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge";
import {
ErrorStatusMenuItems,
CustomIgnoreDialog,
ErrorStatusMenuItems,
statusActionToastMessage,
} from "~/components/errors/ErrorStatusMenu";
import { PageBody } from "~/components/layout/AppLayout";
import { DirectionSchema, ListPagination } from "~/components/ListPagination";
import { LogsVersionFilter } from "~/components/logs/LogsVersionFilter";
import { LinkButton } from "~/components/primitives/Buttons";
import { Callout } from "~/components/primitives/Callout";
import { CopyableText } from "~/components/primitives/CopyableText";
import { DateTime, RelativeDateTime } from "~/components/primitives/DateTime";
import { Header2, Header3 } from "~/components/primitives/Headers";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
import * as Property from "~/components/primitives/PropertyTable";
import {
ResizableHandle,
ResizablePanel,
ResizablePanelGroup,
} from "~/components/primitives/Resizable";
import { Spinner } from "~/components/primitives/Spinner";
import { useToast } from "~/components/primitives/Toast";
import TooltipPortal from "~/components/primitives/TooltipPortal";
import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters";
import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable";
import { $replica } from "~/db.server";
import { useEnvironment } from "~/hooks/useEnvironment";
import { useOptimisticLocation } from "~/hooks/useOptimisticLocation";
import { useOrganization } from "~/hooks/useOrganizations";
import { useProject } from "~/hooks/useProject";
import { useSearchParams } from "~/hooks/useSearchParam";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import {
type ErrorGroupActivity,
type ErrorGroupActivityVersions,
type ErrorGroupOccurrences,
ErrorGroupPresenter,
type ErrorGroupState,
type ErrorGroupSummary,
} from "~/presenters/v3/ErrorGroupPresenter.server";
import { type NextRunList } from "~/presenters/v3/NextRunListPresenter.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUser, requireUserId } from "~/services/session.server";
import { cn } from "~/utils/cn";
import {
EnvironmentParamSchema,
v3CreateBulkActionPath,
v3ErrorsPath,
v3RunsPath,
} from "~/utils/pathBuilder";
import { ServiceValidationError } from "~/v3/services/baseService.server";
import { ErrorGroupActions } from "~/v3/services/errorGroupActions.server";
export const meta: MetaFunction<typeof loader> = ({ data }) => {
return [
@@ -167,6 +164,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
let occurrenceCountAtIgnoreTime: number | undefined;
if (submission.value.totalOccurrences) {
const clickhouseClient = await clickhouseFactory.getClickhouseForOrganization(
environment.organizationId,
"query"
);
const qb = clickhouseClient.errors.listQueryBuilder();
qb.where("organization_id = {organizationId: String}", {
organizationId: project.organizationId,
@@ -240,6 +242,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const directionRaw = url.searchParams.get("direction") ?? undefined;
const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined;
const [logsClickhouseClient, clickhouseClient] = await Promise.all([
clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "logs"),
clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "standard"),
]);
const presenter = new ErrorGroupPresenter($replica, logsClickhouseClient, clickhouseClient);
const detailPromise = presenter
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx
+5 -1
View File
@@ -71,7 +71,7 @@ import {
type ErrorOccurrences,
type ErrorsList as ErrorsListData,
} from "~/presenters/v3/ErrorsListPresenter.server";
import { logsClickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { getCurrentPlan } from "~/services/platform.v3.server";
import { requireUser } from "~/services/session.server";
import { formatNumberCompact } from "~/utils/numberFormatter";
@@ -124,6 +124,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const plan = await getCurrentPlan(project.organizationId);
const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30;
const logsClickhouseClient = await clickhouseFactory.getClickhouseForOrganization(
project.organizationId,
"logs"
);
const presenter = new ErrorsListPresenter($replica, logsClickhouseClient);
const listPromise = presenter
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx
+3 -2
View File
@@ -16,7 +16,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { LogsListPresenter, LogEntry } from "~/presenters/v3/LogsListPresenter.server";
import type { LogLevel } from "~/utils/logUtils";
import { $replica, prisma } from "~/db.server";
import { logsClickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
import { PageBody, PageContainer } from "~/components/layout/AppLayout";
import { Suspense, useCallback, useEffect, useMemo, useRef, useState, useTransition } from "react";
@@ -137,7 +137,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const plan = await getCurrentPlan(project.organizationId);
const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30;
const presenter = new LogsListPresenter($replica, logsClickhouseClient);
const logsClickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "logs");
const presenter = new LogsListPresenter($replica, logsClickhouse);
const listPromise = presenter
.call(project.organizationId, environment.id, {
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx
+3 -2
View File
@@ -28,7 +28,7 @@ import type { QueryWidgetConfig } from "~/components/metrics/QueryWidget";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { ModelRegistryPresenter } from "~/presenters/v3/ModelRegistryPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUserId } from "~/services/session.server";
import { useOrganization } from "~/hooks/useOrganizations";
import { useProject } from "~/hooks/useProject";
@@ -68,7 +68,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
throw new Response("Environment not found", { status: 404 });
}
const presenter = new ModelRegistryPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const presenter = new ModelRegistryPresenter(clickhouse);
const model = await presenter.getModelDetail(modelId);
if (!model) {
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx
+3 -2
View File
@@ -73,7 +73,7 @@ import {
type PopularModel,
ModelRegistryPresenter,
} from "~/presenters/v3/ModelRegistryPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUserId } from "~/services/session.server";
import { useEnvironment } from "~/hooks/useEnvironment";
import { useOrganization } from "~/hooks/useOrganizations";
@@ -112,7 +112,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
throw new Response("Environment not found", { status: 404 });
}
const presenter = new ModelRegistryPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const presenter = new ModelRegistryPresenter(clickhouse);
const catalog = await presenter.getModelCatalog();
const now = new Date();
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx
+3 -2
View File
@@ -20,7 +20,7 @@ import {
type ModelComparisonItem,
ModelRegistryPresenter,
} from "~/presenters/v3/ModelRegistryPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUserId } from "~/services/session.server";
import { useOrganization } from "~/hooks/useOrganizations";
import { useProject } from "~/hooks/useProject";
@@ -55,7 +55,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
return typedjson({ comparison: [] as ModelComparisonItem[], models: responseModels });
}
const presenter = new ModelRegistryPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const presenter = new ModelRegistryPresenter(clickhouse);
const now = new Date();
const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx
+4 -3
View File
@@ -70,7 +70,7 @@ import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { type GenerationRow, PromptPresenter } from "~/presenters/v3/PromptPresenter.server";
import { SpanView } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { getResizableSnapshot } from "~/services/resizablePanel.server";
import { requireUserId } from "~/services/session.server";
import { PromptService } from "~/v3/services/promptService.server";
@@ -242,7 +242,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const startTime = fromTime ? new Date(fromTime) : new Date(Date.now() - periodMs);
const endTime = toTime ? new Date(toTime) : new Date();
const presenter = new PromptPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const presenter = new PromptPresenter(clickhouse);
let generations: Awaited<ReturnType<typeof presenter.listGenerations>>["generations"] = [];
let generationsPagination: { next?: string } = {};
try {
@@ -273,7 +274,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
// Load distinct filter values and resizable snapshots in parallel
const distinctQuery = (col: string, name: string) =>
clickhouseClient.reader.query({
clickhouse.reader.query({
name,
query: `SELECT DISTINCT ${col} AS val FROM trigger_dev.llm_metrics_v1 WHERE environment_id = {environmentId: String} AND prompt_slug = {promptSlug: String} AND ${col} != '' ORDER BY val`,
params: z.object({ environmentId: z.string(), promptSlug: z.string() }),
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx
+3 -2
View File
@@ -22,7 +22,7 @@ import { useProject } from "~/hooks/useProject";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUserId } from "~/services/session.server";
import { docsPath, EnvironmentParamSchema, v3PromptsPath } from "~/utils/pathBuilder";
import { LinkButton } from "~/components/primitives/Buttons";
@@ -46,7 +46,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
throw new Response("Environment not found", { status: 404 });
}
const presenter = new PromptPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const presenter = new PromptPresenter(clickhouse);
const prompts = await presenter.listPrompts(project.id, environment.id);
const sparklines = await presenter.getUsageSparklines(
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx
+3 -2
View File
@@ -92,7 +92,7 @@ import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server";
import { RunEnvironmentMismatchError, RunPresenter } from "~/presenters/v3/RunPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { getImpersonationId } from "~/services/impersonation.server";
import { logger } from "~/services/logger.server";
import { getResizableSnapshot } from "~/services/resizablePanel.server";
@@ -182,7 +182,8 @@ async function getRunsListFromTableState({
return null;
}
const runsListPresenter = new NextRunListPresenter($replica, clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const runsListPresenter = new NextRunListPresenter($replica, clickhouse);
const currentPageResult = await runsListPresenter.call(project.organizationId, environment.id, {
userId,
projectId: project.id,
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx
+3 -2
View File
@@ -45,7 +45,7 @@ import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server";
import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import {
setRootOnlyFilterPreference,
uiPreferencesStorage,
@@ -89,7 +89,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const filters = await getRunFiltersFromRequest(request);
const presenter = new NextRunListPresenter($replica, clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const presenter = new NextRunListPresenter($replica, clickhouse);
const list = presenter.call(project.organizationId, environment.id, {
userId,
projectId: project.id,
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.sessions._index/route.tsx
+6 -2
View File
@@ -16,7 +16,7 @@ import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { getSessionFiltersFromRequest } from "~/presenters/SessionFilters.server";
import { SessionListPresenter } from "~/presenters/v3/SessionListPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUserId } from "~/services/session.server";
import { docsPath, EnvironmentParamSchema } from "~/utils/pathBuilder";
import { throwNotFound } from "~/utils/httpErrors";
@@ -45,7 +45,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const filters = getSessionFiltersFromRequest(request);
const presenter = new SessionListPresenter($replica, clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
project.organizationId,
"standard"
);
const presenter = new SessionListPresenter($replica, clickhouse);
const list = await presenter.call(project.organizationId, environment.id, {
userId,
projectId: project.id,
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx
+7 -2
View File
@@ -74,7 +74,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components
import { DialogClose, DialogDescription } from "@radix-ui/react-dialog";
import { FormButtons } from "~/components/primitives/FormButtons";
import { $replica } from "~/db.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { RegionsPresenter, type Region } from "~/presenters/v3/RegionsPresenter.server";
import { TestSidebarTabs } from "./TestSidebarTabs";
import { AIPayloadTabContent } from "./AIPayloadTabContent";
@@ -102,8 +102,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
});
}
const presenter = new TestTaskPresenter($replica, clickhouseClient);
try {
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
project.organizationId,
"standard"
);
const presenter = new TestTaskPresenter($replica, clickhouse);
const [result, regionsResult] = await Promise.all([
presenter.call({
userId: user.id,
apps/webapp/app/routes/admin.api.v1.runs-replication.create.ts
+26 -25
View File
@@ -1,8 +1,8 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { z } from "zod";
import { ClickHouse } from "@internal/clickhouse";
import { env } from "~/env.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { RunsReplicationService } from "~/services/runsReplicationService.server";
import {
getRunsReplicationGlobal,
@@ -42,6 +42,8 @@ export async function action({ request }: ActionFunctionArgs) {
const params = CreateRunReplicationServiceParams.parse(await request.json());
await clickhouseFactory.isReady();
const service = createRunReplicationService(params);
setRunsReplicationGlobal(service);
@@ -57,24 +59,23 @@ export async function action({ request }: ActionFunctionArgs) {
}
function createRunReplicationService(params: CreateRunReplicationServiceParams) {
const clickhouse = new ClickHouse({
url: env.RUN_REPLICATION_CLICKHOUSE_URL,
name: params.name,
keepAlive: {
enabled: params.keepAliveEnabled,
idleSocketTtl: params.keepAliveIdleSocketTtl,
},
logLevel: "debug",
compression: {
request: true,
},
maxOpenConnections: params.maxOpenConnections,
});
const {
name,
maxFlushConcurrency,
flushIntervalMs,
flushBatchSize,
leaderLockTimeoutMs,
leaderLockExtendIntervalMs,
leaderLockAcquireAdditionalTimeMs,
leaderLockRetryIntervalMs,
ackIntervalSeconds,
waitForAsyncInsert,
} = params;
const service = new RunsReplicationService({
clickhouse: clickhouse,
clickhouseFactory,
pgConnectionUrl: env.DATABASE_URL,
serviceName: params.name,
serviceName: name,
slotName: env.RUN_REPLICATION_SLOT_NAME,
publicationName: env.RUN_REPLICATION_PUBLICATION_NAME,
redisOptions: {
@@ -86,16 +87,16 @@ function createRunReplicationService(params: CreateRunReplicationServiceParams)
enableAutoPipelining: true,
...(env.RUN_REPLICATION_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
},
maxFlushConcurrency: params.maxFlushConcurrency,
flushIntervalMs: params.flushIntervalMs,
flushBatchSize: params.flushBatchSize,
leaderLockTimeoutMs: params.leaderLockTimeoutMs,
leaderLockExtendIntervalMs: params.leaderLockExtendIntervalMs,
leaderLockAcquireAdditionalTimeMs: params.leaderLockAcquireAdditionalTimeMs,
leaderLockRetryIntervalMs: params.leaderLockRetryIntervalMs,
ackIntervalSeconds: params.ackIntervalSeconds,
maxFlushConcurrency,
flushIntervalMs,
flushBatchSize,
leaderLockTimeoutMs,
leaderLockExtendIntervalMs,
leaderLockAcquireAdditionalTimeMs,
leaderLockRetryIntervalMs,
ackIntervalSeconds,
logLevel: "debug",
waitForAsyncInsert: params.waitForAsyncInsert,
waitForAsyncInsert,
});
return service;
apps/webapp/app/routes/admin.api.v1.runs-replication.start.ts
+3
View File
@@ -1,5 +1,6 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { getRunsReplicationGlobal } from "~/services/runsReplicationGlobal.server";
import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
@@ -9,6 +10,8 @@ export async function action({ request }: ActionFunctionArgs) {
try {
const globalService = getRunsReplicationGlobal();
await clickhouseFactory.isReady();
if (globalService) {
await globalService.start();
} else {
apps/webapp/app/routes/admin.data-stores.tsx
+481
View File
@@ -0,0 +1,481 @@
import { useState } from "react";
import { useFetcher } from "@remix-run/react";
import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
import { redirect } from "@remix-run/server-runtime";
import { typedjson, useTypedLoaderData } from "remix-typedjson";
import { z } from "zod";
import { Button } from "~/components/primitives/Buttons";
import {
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
} from "~/components/primitives/Dialog";
import { Input } from "~/components/primitives/Input";
import { Paragraph } from "~/components/primitives/Paragraph";
import { Popover, PopoverContent, PopoverTrigger } from "~/components/primitives/Popover";
import {
Table,
TableBlankRow,
TableBody,
TableCell,
TableHeader,
TableHeaderCell,
TableRow,
} from "~/components/primitives/Table";
import { prisma } from "~/db.server";
import { requireUser } from "~/services/session.server";
import { ClickhouseConnectionSchema } from "~/services/clickhouse/clickhouseSecretSchemas.server";
import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server";
import { tryCatch } from "@trigger.dev/core/utils";
// ---------------------------------------------------------------------------
// Loader
// ---------------------------------------------------------------------------
export const loader = async ({ request }: LoaderFunctionArgs) => {
const user = await requireUser(request);
if (!user.admin) throw redirect("/");
const dataStores = await prisma.organizationDataStore.findMany({
orderBy: { createdAt: "desc" },
});
return typedjson({ dataStores });
};
// ---------------------------------------------------------------------------
// Action
// ---------------------------------------------------------------------------
const AddSchema = z.object({
_action: z.literal("add"),
key: z.string().min(1),
organizationIds: z.string().min(1),
connectionUrl: z.string().url(),
});
const UpdateSchema = z.object({
_action: z.literal("update"),
key: z.string().min(1),
organizationIds: z.string().min(1),
connectionUrl: z.string().url().optional(),
});
const DeleteSchema = z.object({
_action: z.literal("delete"),
key: z.string().min(1),
});
const FormSchema = z.discriminatedUnion("_action", [AddSchema, UpdateSchema, DeleteSchema]);
export async function action({ request }: ActionFunctionArgs) {
const user = await requireUser(request);
if (!user.admin) throw redirect("/");
const formData = await request.formData();
const result = FormSchema.safeParse(Object.fromEntries(formData));
if (!result.success) {
return typedjson(
{ error: result.error.issues.map((i) => i.message).join(", ") },
{ status: 400 }
);
}
switch (result.data._action) {
case "add": {
const { key, organizationIds: rawOrgIds, connectionUrl } = result.data;
const organizationIds = rawOrgIds
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const parsedConfig = ClickhouseConnectionSchema.safeParse({ url: connectionUrl });
if (!parsedConfig.success) {
return typedjson(
{ error: parsedConfig.error.issues.map((i) => i.message).join(", ") },
{ status: 400 }
);
}
const [error, _] = await tryCatch(
organizationDataStoresRegistry.addDataStore({
key,
kind: "CLICKHOUSE",
organizationIds,
config: parsedConfig.data,
})
);
if (error) {
return typedjson({ error: error.message }, { status: 400 });
}
return typedjson({ success: true });
}
case "update": {
const { key, organizationIds: rawOrgIds, connectionUrl } = result.data;
const organizationIds = rawOrgIds
.split(",")
.map((s) => s.trim())
.filter(Boolean);
let config: ReturnType<typeof ClickhouseConnectionSchema.parse> | undefined;
if (connectionUrl) {
const parsedConfig = ClickhouseConnectionSchema.safeParse({ url: connectionUrl });
if (!parsedConfig.success) {
return typedjson(
{ error: parsedConfig.error.issues.map((i) => i.message).join(", ") },
{ status: 400 }
);
}
config = parsedConfig.data;
}
const [error, _] = await tryCatch(
organizationDataStoresRegistry.updateDataStore({
key,
kind: "CLICKHOUSE",
organizationIds,
config,
})
);
if (error) {
return typedjson({ error: error.message }, { status: 400 });
}
return typedjson({ success: true });
}
case "delete": {
const { key } = result.data;
const [error, _] = await tryCatch(
organizationDataStoresRegistry.deleteDataStore({
key,
kind: "CLICKHOUSE",
})
);
if (error) {
return typedjson({ error: error.message }, { status: 400 });
}
return typedjson({ success: true });
}
default: {
return typedjson({ error: "Unknown action" }, { status: 400 });
}
}
}
// ---------------------------------------------------------------------------
// Component
// ---------------------------------------------------------------------------
export default function AdminDataStoresRoute() {
const { dataStores } = useTypedLoaderData<typeof loader>();
const [addOpen, setAddOpen] = useState(false);
return (
<main className="flex h-full min-w-0 flex-1 flex-col overflow-y-auto px-4 pb-4">
<div className="space-y-4">
<div className="flex items-center justify-between">
<Paragraph variant="small" className="text-text-dimmed">
{dataStores.length} data store{dataStores.length !== 1 ? "s" : ""}
</Paragraph>
<Button variant="primary/small" onClick={() => setAddOpen(true)}>
Add data store
</Button>
</div>
<Table>
<TableHeader>
<TableRow>
<TableHeaderCell>Key</TableHeaderCell>
<TableHeaderCell>Kind</TableHeaderCell>
<TableHeaderCell>Organizations</TableHeaderCell>
<TableHeaderCell>Created</TableHeaderCell>
<TableHeaderCell>Updated</TableHeaderCell>
<TableHeaderCell>
<span className="sr-only">Actions</span>
</TableHeaderCell>
</TableRow>
</TableHeader>
<TableBody>
{dataStores.length === 0 ? (
<TableBlankRow colSpan={6}>
<Paragraph>No data stores configured</Paragraph>
</TableBlankRow>
) : (
dataStores.map((ds) => (
<TableRow key={ds.id}>
<TableCell>
<span className="font-mono text-xs text-text-bright">{ds.key}</span>
</TableCell>
<TableCell>
<span className="inline-flex rounded-sm bg-indigo-500/20 px-1.5 py-0.5 text-[11px] font-medium text-indigo-400">
{ds.kind}
</span>
</TableCell>
<TableCell>
<span className="text-xs text-text-dimmed">
{ds.organizationIds.length} org{ds.organizationIds.length !== 1 ? "s" : ""}
</span>
{ds.organizationIds.length > 0 && (
<span
className="ml-1 text-xs text-text-dimmed"
title={ds.organizationIds.join(", ")}
>
({ds.organizationIds.slice(0, 2).join(", ")}
{ds.organizationIds.length > 2
? ` +${ds.organizationIds.length - 2} more`
: ""}
)
</span>
)}
</TableCell>
<TableCell>
<span className="text-xs text-text-dimmed">
{new Date(ds.createdAt).toLocaleString()}
</span>
</TableCell>
<TableCell>
<span className="text-xs text-text-dimmed">
{new Date(ds.updatedAt).toLocaleString()}
</span>
</TableCell>
<TableCell isSticky>
<div className="flex items-center gap-1">
<EditButton name={ds.key} organizationIds={ds.organizationIds} />
<DeleteButton name={ds.key} />
</div>
</TableCell>
</TableRow>
))
)}
</TableBody>
</Table>
</div>
<AddDataStoreDialog open={addOpen} onOpenChange={setAddOpen} />
</main>
);
}
// ---------------------------------------------------------------------------
// Delete button with popover confirmation
// ---------------------------------------------------------------------------
function DeleteButton({ name }: { name: string }) {
const [open, setOpen] = useState(false);
const fetcher = useFetcher<{ success?: boolean; error?: string }>();
const isDeleting = fetcher.state !== "idle";
return (
<Popover open={open} onOpenChange={setOpen}>
<PopoverTrigger asChild>
<Button variant="danger/small" disabled={isDeleting}>
{isDeleting ? "Deleting…" : "Delete"}
</Button>
</PopoverTrigger>
<PopoverContent align="end" className="w-72 space-y-3">
<Paragraph variant="small" className="text-text-bright">
Delete <span className="font-mono font-medium">{name}</span>?
</Paragraph>
<Paragraph variant="extra-small" className="text-text-dimmed">
This will remove the data store and its secret. Organizations using it will fall back to
the default ClickHouse instance.
</Paragraph>
<div className="flex items-center justify-end gap-2">
<Button variant="tertiary/small" onClick={() => setOpen(false)}>
Cancel
</Button>
<fetcher.Form method="post" onSubmit={() => setOpen(false)}>
<input type="hidden" name="_action" value="delete" />
<input type="hidden" name="key" value={name} />
<Button type="submit" variant="danger/small">
Confirm delete
</Button>
</fetcher.Form>
</div>
</PopoverContent>
</Popover>
);
}
// ---------------------------------------------------------------------------
// Edit button with dialog
// ---------------------------------------------------------------------------
function EditButton({ name, organizationIds }: { name: string; organizationIds: string[] }) {
const [open, setOpen] = useState(false);
const fetcher = useFetcher<{ success?: boolean; error?: string }>();
const isSubmitting = fetcher.state !== "idle";
if (fetcher.data?.success && open) {
setOpen(false);
}
return (
<>
<Button variant="secondary/small" onClick={() => setOpen(true)}>
Edit
</Button>
<Dialog open={open} onOpenChange={setOpen}>
<DialogContent className="sm:max-w-lg">
<DialogHeader>
<DialogTitle>Edit data store</DialogTitle>
</DialogHeader>
<fetcher.Form method="post" className="space-y-4 pt-2">
<input type="hidden" name="_action" value="update" />
<input type="hidden" name="key" value={name} />
<div className="space-y-1.5">
<label className="text-xs font-medium text-text-dimmed">Key</label>
<Input name="_key_display" value={name} readOnly variant="medium" className="font-mono opacity-60" />
</div>
<div className="space-y-1.5">
<label className="text-xs font-medium text-text-dimmed">
Organization IDs <span className="text-rose-400">*</span>
</label>
<Input
name="organizationIds"
defaultValue={organizationIds.join(", ")}
placeholder="clxxxxx, clyyyyy, clzzzzz"
variant="medium"
required
/>
<p className="text-[11px] text-text-dimmed">Comma-separated organization IDs.</p>
</div>
{fetcher.data?.error && <p className="text-xs text-rose-400">{fetcher.data.error}</p>}
<DialogFooter>
<Button variant="tertiary/small" type="button" onClick={() => setOpen(false)} disabled={isSubmitting}>
Cancel
</Button>
<Button type="submit" variant="primary/small" disabled={isSubmitting}>
{isSubmitting ? "Saving…" : "Save"}
</Button>
</DialogFooter>
</fetcher.Form>
</DialogContent>
</Dialog>
</>
);
}
// ---------------------------------------------------------------------------
// Add data store dialog
// ---------------------------------------------------------------------------
function AddDataStoreDialog({
open,
onOpenChange,
}: {
open: boolean;
onOpenChange: (open: boolean) => void;
}) {
const fetcher = useFetcher<{ success?: boolean; error?: string }>();
const isSubmitting = fetcher.state !== "idle";
// Close dialog on success
if (fetcher.data?.success && open) {
onOpenChange(false);
}
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-lg">
<DialogHeader>
<DialogTitle>Add data store</DialogTitle>
</DialogHeader>
<fetcher.Form method="post" className="space-y-4 pt-2">
<input type="hidden" name="_action" value="add" />
<div className="space-y-1.5">
<label className="text-xs font-medium text-text-dimmed">
Key <span className="text-rose-400">*</span>
</label>
<Input
name="key"
placeholder="e.g. hipaa-clickhouse-us-east"
variant="medium"
required
className="font-mono"
/>
<p className="text-[11px] text-text-dimmed">
Unique identifier for this data store. Used as the secret key prefix.
</p>
</div>
<div className="space-y-1.5">
<label className="text-xs font-medium text-text-dimmed">
Kind <span className="text-rose-400">*</span>
</label>
<Input
name="kind"
value="CLICKHOUSE"
readOnly
variant="medium"
className="opacity-60"
/>
</div>
<div className="space-y-1.5">
<label className="text-xs font-medium text-text-dimmed">
Organization IDs <span className="text-rose-400">*</span>
</label>
<Input
name="organizationIds"
placeholder="clxxxxx, clyyyyy, clzzzzz"
variant="medium"
required
/>
<p className="text-[11px] text-text-dimmed">Comma-separated organization IDs.</p>
</div>
<div className="space-y-1.5">
<label className="text-xs font-medium text-text-dimmed">
ClickHouse connection URL <span className="text-rose-400">*</span>
</label>
<Input
name="connectionUrl"
type="password"
placeholder="https://user:password@host:8443"
variant="medium"
required
className="font-mono"
/>
<p className="text-[11px] text-text-dimmed">
Stored encrypted in SecretStore. Never logged or displayed again.
</p>
</div>
{fetcher.data?.error && <p className="text-xs text-rose-400">{fetcher.data.error}</p>}
<DialogFooter>
<Button
variant="tertiary/small"
type="button"
onClick={() => onOpenChange(false)}
disabled={isSubmitting}
>
Cancel
</Button>
<Button type="submit" variant="primary/small" disabled={isSubmitting}>
{isSubmitting ? "Adding…" : "Add data store"}
</Button>
</DialogFooter>
</fetcher.Form>
</DialogContent>
</Dialog>
);
}
apps/webapp/app/routes/admin.tsx
+4
View File
@@ -48,6 +48,10 @@ export default function Page() {
to: "/admin/back-office",
end: false,
},
{
label: "Data Stores",
to: "/admin/data-stores",
},
]}
layoutId={"admin"}
/>
apps/webapp/app/routes/api.v1.prompts.$slug.ts
+12 -3
View File
@@ -2,7 +2,7 @@ import { json } from "@remix-run/server-runtime";
import { z } from "zod";
import { prisma } from "~/db.server";
import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import {
createActionApiRoute,
createLoaderApiRoute,
@@ -33,6 +33,13 @@ export const loader = createLoaderApiRoute(
slug: params.slug,
},
},
include: {
project: {
select: {
organizationId: true,
},
},
},
});
},
authorization: {
@@ -45,7 +52,8 @@ export const loader = createLoaderApiRoute(
return json({ error: "Prompt not found" }, { status: 404 });
}
const presenter = new PromptPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(prompt.project.organizationId, "standard");
const presenter = new PromptPresenter(clickhouse);
const version = await presenter.resolveVersion(prompt.id, {
version: searchParams.version,
label: searchParams.label,
@@ -115,7 +123,8 @@ const { action } = createActionApiRoute(
return json({ error: "Prompt not found" }, { status: 404 });
}
const presenter = new PromptPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(authentication.environment.organizationId, "standard");
const presenter = new PromptPresenter(clickhouse);
const version = await presenter.resolveVersion(prompt.id, {
version: body.version,
label: body.label,
apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts
+10 -2
View File
@@ -2,7 +2,7 @@ import { json } from "@remix-run/server-runtime";
import { z } from "zod";
import { prisma } from "~/db.server";
import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
const ParamsSchema = z.object({
@@ -23,6 +23,13 @@ export const loader = createLoaderApiRoute(
slug: params.slug,
},
},
include: {
project: {
select: {
organizationId: true,
},
},
},
});
},
authorization: {
@@ -35,7 +42,8 @@ export const loader = createLoaderApiRoute(
return json({ error: "Prompt not found" }, { status: 404 });
}
const presenter = new PromptPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(prompt.project.organizationId, "standard");
const presenter = new PromptPresenter(clickhouse);
const versions = await presenter.listVersions(prompt.id);
return json({
apps/webapp/app/routes/api.v1.prompts._index.ts
+3 -2
View File
@@ -1,6 +1,6 @@
import { json } from "@remix-run/server-runtime";
import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
export const loader = createLoaderApiRoute(
@@ -14,7 +14,8 @@ export const loader = createLoaderApiRoute(
},
},
async ({ authentication }) => {
const presenter = new PromptPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(authentication.environment.organizationId, "standard");
const presenter = new PromptPresenter(clickhouse);
const prompts = await presenter.listPrompts(
authentication.environment.projectId,
authentication.environment.id
apps/webapp/app/routes/api.v1.runs.$runId.events.ts
+5 -2
View File
@@ -6,7 +6,7 @@ import {
createLoaderApiRoute,
} from "~/services/routeBuilders/apiBuilder.server";
import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server";
import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
const ParamsSchema = z.object({
runId: z.string(), // This is the run friendly ID
@@ -38,7 +38,10 @@ export const loader = createLoaderApiRoute(
},
},
async ({ resource: run, authentication }) => {
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
authentication.environment.organization.id
);
const runEvents = await eventRepository.getRunEvents(
getTaskEventStoreTableForRun(run),
apps/webapp/app/routes/api.v1.runs.$runId.spans.$spanId.ts
+5 -2
View File
@@ -7,7 +7,7 @@ import {
anyResource,
createLoaderApiRoute,
} from "~/services/routeBuilders/apiBuilder.server";
import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
const ParamsSchema = z.object({
@@ -45,7 +45,10 @@ export const loader = createLoaderApiRoute(
},
},
async ({ params, resource: run, authentication }) => {
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
authentication.environment.organization.id
);
const eventStore = getTaskEventStoreTableForRun(run);
const span = await eventRepository.getSpan(
apps/webapp/app/routes/api.v1.runs.$runId.trace.ts
+5 -2
View File
@@ -6,7 +6,7 @@ import {
anyResource,
createLoaderApiRoute,
} from "~/services/routeBuilders/apiBuilder.server";
import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
const ParamsSchema = z.object({
@@ -43,7 +43,10 @@ export const loader = createLoaderApiRoute(
},
},
async ({ resource: run, authentication }) => {
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
authentication.environment.organization.id
);
const traceSummary = await eventRepository.getTraceDetailedSummary(
getTaskEventStoreTableForRun(run),
apps/webapp/app/routes/api.v1.sessions.ts
+6 -2
View File
@@ -10,7 +10,7 @@ import {
import { SessionId } from "@trigger.dev/core/v3/isomorphic";
import type { Prisma, Session } from "@trigger.dev/database";
import { $replica, prisma, type PrismaClient } from "~/db.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { logger } from "~/services/logger.server";
import { mintSessionToken } from "~/services/realtime/mintSessionToken.server";
import {
@@ -58,8 +58,12 @@ export const loader = createLoaderApiRoute(
findResource: async () => 1,
},
async ({ searchParams, authentication }) => {
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(
authentication.environment.organizationId,
"standard"
);
const repository = new SessionsRepository({
clickhouse: clickhouseClient,
clickhouse,
prisma: $replica as PrismaClient,
});
apps/webapp/app/routes/otel.v1.logs.ts
+3 -2
View File
@@ -4,12 +4,13 @@ import { otlpExporter } from "~/v3/otlpExporter.server";
export async function action({ request }: ActionFunctionArgs) {
try {
const exporter = await otlpExporter;
const contentType = request.headers.get("content-type")?.toLowerCase() ?? "";
if (contentType.startsWith("application/json")) {
const body = await request.json();
const exportResponse = await otlpExporter.exportLogs(body as ExportLogsServiceRequest);
const exportResponse = await exporter.exportLogs(body as ExportLogsServiceRequest);
return json(exportResponse, { status: 200 });
} else if (contentType.startsWith("application/x-protobuf")) {
@@ -17,7 +18,7 @@ export async function action({ request }: ActionFunctionArgs) {
const exportRequest = ExportLogsServiceRequest.decode(new Uint8Array(buffer));
const exportResponse = await otlpExporter.exportLogs(exportRequest);
const exportResponse = await exporter.exportLogs(exportRequest);
return new Response(ExportLogsServiceResponse.encode(exportResponse).finish(), {
status: 200,
apps/webapp/app/routes/otel.v1.metrics.ts
+4 -2
View File
@@ -10,19 +10,21 @@ export async function action({ request }: ActionFunctionArgs) {
const contentType = request.headers.get("content-type")?.toLowerCase() ?? "";
if (contentType.startsWith("application/json")) {
const exporter = await otlpExporter;
const body = await request.json();
const exportResponse = await otlpExporter.exportMetrics(
const exportResponse = await exporter.exportMetrics(
body as ExportMetricsServiceRequest
);
return json(exportResponse, { status: 200 });
} else if (contentType.startsWith("application/x-protobuf")) {
const exporter = await otlpExporter;
const buffer = await request.arrayBuffer();
const exportRequest = ExportMetricsServiceRequest.decode(new Uint8Array(buffer));
const exportResponse = await otlpExporter.exportMetrics(exportRequest);
const exportResponse = await exporter.exportMetrics(exportRequest);
return new Response(ExportMetricsServiceResponse.encode(exportResponse).finish(), {
status: 200,
apps/webapp/app/routes/otel.v1.traces.ts
+3 -2
View File
@@ -4,12 +4,13 @@ import { otlpExporter } from "~/v3/otlpExporter.server";
export async function action({ request }: ActionFunctionArgs) {
try {
const exporter = await otlpExporter;
const contentType = request.headers.get("content-type")?.toLowerCase() ?? "";
if (contentType.startsWith("application/json")) {
const body = await request.json();
const exportResponse = await otlpExporter.exportTraces(body as ExportTraceServiceRequest);
const exportResponse = await exporter.exportTraces(body as ExportTraceServiceRequest);
return json(exportResponse, { status: 200 });
} else if (contentType.startsWith("application/x-protobuf")) {
@@ -17,7 +18,7 @@ export async function action({ request }: ActionFunctionArgs) {
const exportRequest = ExportTraceServiceRequest.decode(new Uint8Array(buffer));
const exportResponse = await otlpExporter.exportTraces(exportRequest);
const exportResponse = await exporter.exportTraces(exportRequest);
return new Response(ExportTraceServiceResponse.encode(exportResponse).finish(), {
status: 200,
apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx
+3 -2
View File
@@ -1,7 +1,7 @@
import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
import { typedjson } from "remix-typedjson";
import { z } from "zod";
import { logsClickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { requireUserId } from "~/services/session.server";
import { LogDetailPresenter } from "~/presenters/v3/LogDetailPresenter.server";
import { findProjectBySlug } from "~/models/project.server";
@@ -43,7 +43,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const [traceId, spanId, , startTime] = parts;
const presenter = new LogDetailPresenter($replica, logsClickhouseClient);
const logsClickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "logs");
const presenter = new LogDetailPresenter($replica, logsClickhouse);
let result;
try {
apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts
+3 -2
View File
@@ -6,7 +6,7 @@ import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server";
import { $replica } from "~/db.server";
import { logsClickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { getCurrentPlan } from "~/services/platform.v3.server";
// Valid log levels for filtering
@@ -69,7 +69,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
retentionLimitDays,
}) as any; // Validated by LogsListOptionsSchema at runtime
const presenter = new LogsListPresenter($replica, logsClickhouseClient);
const logsClickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "logs");
const presenter = new LogsListPresenter($replica, logsClickhouse);
const result = await presenter.call(project.organizationId, environment.id, options);
return json({
apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts
+3 -2
View File
@@ -6,7 +6,7 @@ import { EnvironmentParamSchema } from "~/utils/pathBuilder";
import { parsePeriodToMs } from "~/utils/periods";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import {
PromptPresenter,
type GenerationRow,
@@ -59,7 +59,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const operations = url.searchParams.getAll("operations").filter(Boolean);
const providers = url.searchParams.getAll("providers").filter(Boolean);
const presenter = new PromptPresenter(clickhouseClient);
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard");
const presenter = new PromptPresenter(clickhouse);
const result = await presenter.listGenerations({
environmentId: environment.id,
promptSlug,
apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts
+6 -3
View File
@@ -8,7 +8,7 @@ import { Readable } from "stream";
import { formatDurationMilliseconds } from "@trigger.dev/core/v3/utils/durations";
import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
import { TaskEventKind } from "@trigger.dev/database";
import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
export async function loader({ params, request }: LoaderFunctionArgs) {
const user = await requireUser(request);
@@ -29,11 +29,14 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
},
});
if (!run) {
if (!run || !run.organizationId) {
return new Response("Not found", { status: 404 });
}
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
run.organizationId
);
const runEvents = await eventRepository.getRunEvents(
getTaskEventStoreTableForRun(run),
apps/webapp/app/runEngine/concerns/traceEvents.server.ts
+10 -12
View File
@@ -10,6 +10,7 @@ export class DefaultTraceEventsConcern implements TraceEventConcern {
parentStore: string | undefined
): Promise<{ repository: IEventRepository; store: string }> {
return await getEventRepository(
request.environment.organization.id,
request.environment.organization.featureFlags as Record<string, unknown>,
parentStore
);
@@ -162,18 +163,15 @@ export class DefaultTraceEventsConcern implements TraceEventConcern {
},
async (event, traceContext, traceparent) => {
// Log a message about the debounced trigger
await repository.recordEvent(
`Debounced: using existing run with key "${debounceKey}"`,
{
taskSlug: request.taskId,
environment: request.environment,
attributes: {
runId: existingRun.friendlyId,
},
context: request.options?.traceContext,
parentId: event.spanId,
}
);
await repository.recordEvent(`Debounced: using existing run with key "${debounceKey}"`, {
taskSlug: request.taskId,
environment: request.environment,
attributes: {
runId: existingRun.friendlyId,
},
context: request.options?.traceContext,
parentId: event.spanId,
});
return await callback(
{
apps/webapp/app/runEngine/services/triggerFailedTask.server.ts
+19 -20
View File
@@ -74,6 +74,7 @@ export class TriggerFailedTaskService {
try {
const { repository, store } = await getEventRepository(
request.environment.organization.id,
request.environment.organization.featureFlags as Record<string, unknown>,
undefined
);
@@ -81,11 +82,11 @@ export class TriggerFailedTaskService {
// Resolve parent run for rootTaskRunId and depth (same as triggerTask.server.ts)
const parentRun = request.parentRunId
? await this.prisma.taskRun.findFirst({
where: {
id: RunId.fromFriendlyId(request.parentRunId),
runtimeEnvironmentId: request.environment.id,
},
})
where: {
id: RunId.fromFriendlyId(request.parentRunId),
runtimeEnvironmentId: request.environment.id,
},
})
: undefined;
const depth = parentRun ? parentRun.depth + 1 : 0;
@@ -116,18 +117,18 @@ export class TriggerFailedTaskService {
// resolveQueueProperties requires the worker to be passed when lockToVersion is present.
const lockedToBackgroundWorker = bodyOptions?.lockToVersion
? await this.prisma.backgroundWorker.findFirst({
where: {
projectId: request.environment.projectId,
runtimeEnvironmentId: request.environment.id,
version: bodyOptions.lockToVersion,
},
select: {
id: true,
version: true,
sdkVersion: true,
cliVersion: true,
},
})
where: {
projectId: request.environment.projectId,
runtimeEnvironmentId: request.environment.id,
version: bodyOptions.lockToVersion,
},
select: {
id: true,
version: true,
sdkVersion: true,
cliVersion: true,
},
})
: undefined;
const resolved = await queueConcern.resolveQueueProperties(
@@ -273,9 +274,7 @@ export class TriggerFailedTaskService {
},
taskIdentifier: opts.taskId,
payload:
typeof opts.payload === "string"
? opts.payload
: JSON.stringify(opts.payload ?? ""),
typeof opts.payload === "string" ? opts.payload : JSON.stringify(opts.payload ?? ""),
payloadType: opts.payloadType ?? "application/json",
error: {
type: "INTERNAL_ERROR" as const,
apps/webapp/app/services/admin/missingLlmModels.server.ts
+7 -3
View File
@@ -1,4 +1,4 @@
import { adminClickhouseClient } from "~/services/clickhouseInstance.server";
import { getAdminClickhouse } from "~/services/clickhouse/clickhouseFactory.server";
import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server";
export type MissingLlmModel = {
@@ -13,8 +13,10 @@ export async function getMissingLlmModels(opts: {
const lookbackHours = opts.lookbackHours ?? 24;
const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000);
const adminClickhouse = getAdminClickhouse();
// queryBuilderFast returns a factory function — call it to get the builder
const createBuilder = adminClickhouseClient.reader.queryBuilderFast<{
const createBuilder = adminClickhouse.reader.queryBuilderFast<{
model: string;
system: string;
cnt: string;
@@ -93,7 +95,9 @@ export async function getMissingModelSamples(opts: {
const limit = opts.limit ?? 10;
const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000);
const createBuilder = adminClickhouseClient.reader.queryBuilderFast<MissingModelSample>({
const adminClickhouse = getAdminClickhouse();
const createBuilder = adminClickhouse.reader.queryBuilderFast<MissingModelSample>({
name: "missingModelSamples",
table: "trigger_dev.task_events_v2",
columns: [
apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts
+489
View File
@@ -0,0 +1,489 @@
import { ClickHouse } from "@internal/clickhouse";
import { createHash } from "crypto";
import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server";
import { env } from "~/env.server";
import { singleton } from "~/utils/singleton";
import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
import { type IEventRepository } from "~/v3/eventRepository/eventRepository.types";
// ---------------------------------------------------------------------------
// Default clients (singleton per process)
// ---------------------------------------------------------------------------
const defaultClickhouseClient = singleton("clickhouseClient", initializeClickhouseClient);
function initializeClickhouseClient() {
const url = new URL(env.CLICKHOUSE_URL);
url.searchParams.delete("secure");
console.log(`🗃️ Clickhouse service enabled to host ${url.host}`);
return new ClickHouse({
url: url.toString(),
name: "clickhouse-instance",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
const defaultLogsClickhouseClient = singleton(
"logsClickhouseClient",
initializeLogsClickhouseClient
);
function getLogsListClickhouseSettings() {
return {
max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
max_bytes_before_external_sort:
env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
}),
...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
}),
};
}
function initializeLogsClickhouseClient() {
if (!env.LOGS_CLICKHOUSE_URL) {
throw new Error("LOGS_CLICKHOUSE_URL is not set");
}
const url = new URL(env.LOGS_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "logs-clickhouse",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
clickhouseSettings: getLogsListClickhouseSettings(),
});
}
const defaultAdminClickhouseClient = singleton(
"adminClickhouseClient",
initializeAdminClickhouseClient
);
function initializeAdminClickhouseClient() {
if (!env.ADMIN_CLICKHOUSE_URL) {
throw new Error("ADMIN_CLICKHOUSE_URL is not set");
}
const url = new URL(env.ADMIN_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "admin-clickhouse",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
const defaultQueryClickhouseClient = singleton(
"queryClickhouseClient",
initializeQueryClickhouseClient
);
function initializeQueryClickhouseClient() {
if (!env.QUERY_CLICKHOUSE_URL) {
throw new Error("QUERY_CLICKHOUSE_URL is not set");
}
const url = new URL(env.QUERY_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "query-clickhouse",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
/** TaskRun replication to ClickHouse (`RUN_REPLICATION_CLICKHOUSE_URL`); not exported. */
const defaultRunsReplicationClickhouseClient = singleton(
"runsReplicationClickhouseClient",
initializeRunsReplicationClickhouseClient
);
function initializeRunsReplicationClickhouseClient(): ClickHouse {
if (!env.RUN_REPLICATION_CLICKHOUSE_URL) {
// Runs replication worker gates on this URL; factory may still resolve "replication" for tests.
return defaultClickhouseClient;
}
const url = new URL(env.RUN_REPLICATION_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "runs-replication",
keepAlive: {
enabled: env.RUN_REPLICATION_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.RUN_REPLICATION_CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.RUN_REPLICATION_MAX_OPEN_CONNECTIONS,
});
}
/** Session replication to ClickHouse (`SESSION_REPLICATION_CLICKHOUSE_URL`); not exported. */
const defaultSessionsReplicationClickhouseClient = singleton(
"sessionsReplicationClickhouseClient",
initializeSessionsReplicationClickhouseClient
);
function initializeSessionsReplicationClickhouseClient(): ClickHouse {
if (!env.SESSION_REPLICATION_CLICKHOUSE_URL) {
// Sessions replication worker gates on this URL; factory may still resolve "sessions_replication" for tests.
return defaultClickhouseClient;
}
const url = new URL(env.SESSION_REPLICATION_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "sessions-replication",
keepAlive: {
enabled: env.SESSION_REPLICATION_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.SESSION_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.SESSION_REPLICATION_CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.SESSION_REPLICATION_MAX_OPEN_CONNECTIONS,
});
}
/** Task events (`EVENTS_CLICKHOUSE_URL`); not exported — accessed via factory. */
const defaultEventsClickhouseClient = singleton(
"eventsClickhouseClient",
initializeEventsClickhouseClient
);
function initializeEventsClickhouseClient(): ClickHouse {
if (!env.EVENTS_CLICKHOUSE_URL) {
throw new Error("EVENTS_CLICKHOUSE_URL is not set");
}
const url = new URL(env.EVENTS_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "task-events",
keepAlive: {
enabled: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.EVENTS_CLICKHOUSE_LOG_LEVEL,
compression: {
request: env.EVENTS_CLICKHOUSE_COMPRESSION_REQUEST === "1",
},
maxOpenConnections: env.EVENTS_CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function hashHostname(url: string): string {
const parsed = new URL(url);
return createHash("sha256").update(parsed.hostname).digest("hex");
}
export type ClientType =
| "standard"
| "events"
| "replication"
| "sessions_replication"
| "logs"
| "query"
| "admin";
function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHouse {
const parsed = new URL(url);
parsed.searchParams.delete("secure");
const name = `org-clickhouse-${clientType}`;
switch (clientType) {
case "events":
return new ClickHouse({
url: parsed.toString(),
name,
keepAlive: {
enabled: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.EVENTS_CLICKHOUSE_LOG_LEVEL,
compression: {
request: env.EVENTS_CLICKHOUSE_COMPRESSION_REQUEST === "1",
},
maxOpenConnections: env.EVENTS_CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
case "replication":
return new ClickHouse({
url: parsed.toString(),
name,
keepAlive: {
enabled: env.RUN_REPLICATION_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.RUN_REPLICATION_CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.RUN_REPLICATION_MAX_OPEN_CONNECTIONS,
});
case "sessions_replication":
return new ClickHouse({
url: parsed.toString(),
name,
keepAlive: {
enabled: env.SESSION_REPLICATION_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.SESSION_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.SESSION_REPLICATION_CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.SESSION_REPLICATION_MAX_OPEN_CONNECTIONS,
});
case "logs":
return new ClickHouse({
url: parsed.toString(),
name,
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
clickhouseSettings: getLogsListClickhouseSettings(),
});
case "standard":
case "query":
case "admin":
return new ClickHouse({
url: parsed.toString(),
name,
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: { request: true },
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
}
// ---------------------------------------------------------------------------
// Factory class (injectable for testing)
// ---------------------------------------------------------------------------
export class ClickhouseFactory {
/** ClickHouse clients keyed by hostname hash + clientType. */
private readonly _clientCache = new Map<string, ClickHouse>();
/** Event repositories keyed by hostname hash (stateful, must be reused). */
private readonly _eventRepositoryCache = new Map<string, ClickhouseEventRepository>();
constructor(private readonly _registry: OrganizationDataStoresRegistry) {}
async isReady(): Promise<boolean> {
if (!this._registry.isLoaded) {
await this._registry.isReady;
}
return true;
}
async getClickhouseForOrganization(
organizationId: string,
clientType: ClientType
): Promise<ClickHouse> {
if (!this._registry.isLoaded) {
await this._registry.isReady;
}
return this.getClickhouseForOrganizationSync(organizationId, clientType);
}
getClickhouseForOrganizationSync(organizationId: string, clientType: ClientType): ClickHouse {
const dataStore = this._registry.get(organizationId, "CLICKHOUSE");
if (!dataStore) {
switch (clientType) {
case "standard":
return defaultClickhouseClient;
case "events":
return defaultEventsClickhouseClient;
case "replication":
return defaultRunsReplicationClickhouseClient;
case "sessions_replication":
return defaultSessionsReplicationClickhouseClient;
case "logs":
return defaultLogsClickhouseClient;
case "query":
return defaultQueryClickhouseClient;
case "admin":
return defaultAdminClickhouseClient;
}
}
const hostnameHash = hashHostname(dataStore.url);
const cacheKey = `${hostnameHash}:${clientType}`;
let client = this._clientCache.get(cacheKey);
if (!client) {
client = buildOrgClickhouseClient(dataStore.url, clientType);
this._clientCache.set(cacheKey, client);
}
return client;
}
async getEventRepositoryForOrganization(
store: string,
organizationId: string
): Promise<{ key: string; repository: IEventRepository }> {
if (!this._registry.isLoaded) {
await this._registry.isReady;
}
return this.getEventRepositoryForOrganizationSync(store, organizationId);
}
getEventRepositoryForOrganizationSync(
store: string,
organizationId: string
): { key: string; repository: IEventRepository } {
const dataStore = this._registry.get(organizationId, "CLICKHOUSE");
if (!dataStore) {
const defaultKey = `default:events:${store}`;
let defaultRepo = this._eventRepositoryCache.get(defaultKey);
if (!defaultRepo) {
const eventsClickhouse = getEventsClickhouseClient();
defaultRepo = buildEventRepository(store, eventsClickhouse);
this._eventRepositoryCache.set(defaultKey, defaultRepo);
}
return { key: defaultKey, repository: defaultRepo };
}
const hostnameHash = hashHostname(dataStore.url);
const cacheKey = `${hostnameHash}:events:${store}`;
let repository = this._eventRepositoryCache.get(cacheKey);
if (!repository) {
const client = this.getClickhouseForOrganizationSync(organizationId, "events");
repository = buildEventRepository(store, client);
this._eventRepositoryCache.set(cacheKey, repository);
}
return { key: cacheKey, repository: repository };
}
}
/**
* Get admin ClickHouse client for cross-organization queries.
* Only use for admin tools and analytics that need to query across all orgs.
*/
export function getAdminClickhouse(): ClickHouse {
return defaultAdminClickhouseClient;
}
export function getDefaultClickhouseClient(): ClickHouse {
return defaultClickhouseClient;
}
export function getDefaultLogsClickhouseClient(): ClickHouse {
return defaultLogsClickhouseClient;
}
// ---------------------------------------------------------------------------
// Private helpers
// ---------------------------------------------------------------------------
function getEventsClickhouseClient(): ClickHouse {
return defaultEventsClickhouseClient;
}
function buildEventRepository(store: string, clickhouse: ClickHouse): ClickhouseEventRepository {
switch (store) {
case "clickhouse": {
return new ClickhouseEventRepository({
clickhouse,
batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
maximumTraceDetailedSummaryViewCount:
env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",
asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE,
asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS,
startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS,
llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE,
llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS,
llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE,
llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY,
otlpMetricsBatchSize: env.METRICS_CLICKHOUSE_BATCH_SIZE,
otlpMetricsFlushInterval: env.METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS,
otlpMetricsMaxConcurrency: env.METRICS_CLICKHOUSE_MAX_CONCURRENCY,
version: "v1",
});
}
case "clickhouse_v2": {
return new ClickhouseEventRepository({
clickhouse: clickhouse,
batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
maximumTraceDetailedSummaryViewCount:
env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",
asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE,
asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS,
startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS,
llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE,
llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS,
llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE,
llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY,
otlpMetricsBatchSize: env.METRICS_CLICKHOUSE_BATCH_SIZE,
otlpMetricsFlushInterval: env.METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS,
otlpMetricsMaxConcurrency: env.METRICS_CLICKHOUSE_MAX_CONCURRENCY,
version: "v2",
});
}
default: {
throw new Error(`Unknown ClickHouse event repository store: ${store}`);
}
}
}
apps/webapp/app/services/clickhouse/clickhouseFactoryInstance.server.ts
+13
View File
@@ -0,0 +1,13 @@
import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server";
import { singleton } from "~/utils/singleton";
import { ClickhouseFactory } from "./clickhouseFactory.server";
/**
* Production singleton wired to the global organization data-stores registry.
* Import this only from app/runtime code — not from tests that construct a
* {@link ClickhouseFactory} with a stub registry (see `clickhouseFactory.server.ts`).
*/
export const clickhouseFactory = singleton(
"clickhouseFactory",
() => new ClickhouseFactory(organizationDataStoresRegistry)
);
apps/webapp/app/services/clickhouse/clickhouseSecretSchemas.server.ts
+11
View File
@@ -0,0 +1,11 @@
import { z } from "zod";
export const ClickhouseConnectionSchema = z.object({
url: z.string().url(),
});
export type ClickhouseConnection = z.infer<typeof ClickhouseConnectionSchema>;
export function getClickhouseSecretKey(orgId: string, clientType: string): string {
return `org:${orgId}:clickhouse:${clientType}`;
}
apps/webapp/app/services/clickhouseInstance.server.ts
-130
View File
@@ -1,130 +0,0 @@
import { ClickHouse } from "@internal/clickhouse";
import { env } from "~/env.server";
import { singleton } from "~/utils/singleton";
export const clickhouseClient = singleton("clickhouseClient", initializeClickhouseClient);
function initializeClickhouseClient() {
const url = new URL(env.CLICKHOUSE_URL);
// Remove secure param
url.searchParams.delete("secure");
console.log(`🗃️ Clickhouse service enabled to host ${url.host}`);
const clickhouse = new ClickHouse({
url: url.toString(),
name: "clickhouse-instance",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: {
request: true,
},
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
return clickhouse;
}
export const logsClickhouseClient = singleton(
"logsClickhouseClient",
initializeLogsClickhouseClient
);
function initializeLogsClickhouseClient() {
if (!env.LOGS_CLICKHOUSE_URL) {
throw new Error("LOGS_CLICKHOUSE_URL is not set");
}
const url = new URL(env.LOGS_CLICKHOUSE_URL);
// Remove secure param
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "logs-clickhouse",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: {
request: true,
},
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
clickhouseSettings: {
max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
max_bytes_before_external_sort:
env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
}),
...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
}),
},
});
}
export const adminClickhouseClient = singleton(
"adminClickhouseClient",
initializeAdminClickhouseClient
);
function initializeAdminClickhouseClient() {
if (!env.ADMIN_CLICKHOUSE_URL) {
throw new Error("ADMIN_CLICKHOUSE_URL is not set");
}
const url = new URL(env.ADMIN_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "admin-clickhouse",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: {
request: true,
},
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
export const queryClickhouseClient = singleton(
"queryClickhouseClient",
initializeQueryClickhouseClient
);
function initializeQueryClickhouseClient() {
if (!env.QUERY_CLICKHOUSE_URL) {
throw new Error("QUERY_CLICKHOUSE_URL is not set");
}
const url = new URL(env.QUERY_CLICKHOUSE_URL);
// Remove secure param
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "query-clickhouse",
keepAlive: {
enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.CLICKHOUSE_LOG_LEVEL,
compression: {
request: true,
},
maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
apps/webapp/app/services/dataStores/organizationDataStoreConfigSchemas.server.ts
+39
View File
@@ -0,0 +1,39 @@
import { z } from "zod";
// ---------------------------------------------------------------------------
// ClickHouse config (kind = CLICKHOUSE)
// ---------------------------------------------------------------------------
/** V1: single secret-store key that supplies the ClickHouse connection URL. */
export const ClickhouseDataStoreConfigV1 = z.object({
version: z.literal(1),
data: z.object({
/** Key into the SecretStore that resolves to a ClickhouseConnection ({url}). */
secretKey: z.string(),
}),
});
export type ClickhouseDataStoreConfigV1 = z.infer<typeof ClickhouseDataStoreConfigV1>;
/** Discriminated union over version — extend by adding new literals here. */
export const ClickhouseDataStoreConfig = z.discriminatedUnion("version", [
ClickhouseDataStoreConfigV1,
]);
export type ClickhouseDataStoreConfig = z.infer<typeof ClickhouseDataStoreConfig>;
// ---------------------------------------------------------------------------
// Top-level per-kind union
// ---------------------------------------------------------------------------
/**
* Secrets are resolved to URLs at registry load time so the factory never
* needs to touch the secret store on the hot path.
*/
export type ParsedClickhouseDataStore = {
kind: "CLICKHOUSE";
url: string;
};
/** Union of all parsed data store types. Extend as new DataStoreKind values are added. */
export type ParsedDataStore = ParsedClickhouseDataStore;
apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts
+189
View File
@@ -0,0 +1,189 @@
import type { DataStoreKind, PrismaClient, PrismaReplicaClient } from "@trigger.dev/database";
import {
ClickhouseDataStoreConfig,
type ParsedDataStore,
} from "./organizationDataStoreConfigSchemas.server";
import { getSecretStore } from "../secrets/secretStore.server";
import { ClickhouseConnectionSchema } from "../clickhouse/clickhouseSecretSchemas.server";
export class OrganizationDataStoresRegistry {
private _prisma: PrismaClient | PrismaReplicaClient;
/** Keyed by `${organizationId}:${kind}` */
private _lookup: Map<string, ParsedDataStore> = new Map();
private _loaded = false;
private _readyResolve!: () => void;
/**
* Resolves once the initial `loadFromDatabase()` completes successfully.
* At process startup the singleton loads the registry with unbounded retries
* (exponential backoff, capped delay) until Postgres is reachable; until then
* this promise stays pending and callers that await readiness will block.
*/
readonly isReady: Promise<void>;
constructor(prisma: PrismaClient | PrismaReplicaClient) {
this._prisma = prisma;
this.isReady = new Promise<void>((resolve) => {
this._readyResolve = resolve;
});
}
get isLoaded(): boolean {
return this._loaded;
}
async loadFromDatabase(): Promise<void> {
// Sort by `key` (unique, immutable) to ensure a deterministic winner when the
// same `${orgId}:${kind}` appears in multiple rows. The registry must never
// throw on overlap — failing the load would break every customer, not just the
// misconfigured orgs — so we keep the first entry and log an error instead.
const rows = await this._prisma.organizationDataStore.findMany({
orderBy: { key: "asc" },
});
const secretStore = getSecretStore("DATABASE", { prismaClient: this._prisma });
const lookup = new Map<string, ParsedDataStore>();
/** Tracks which row's `key` already owns each `${orgId}:${kind}` so we can log conflicts. */
const winnerByLookupKey = new Map<string, string>();
for (const row of rows) {
let parsed: ParsedDataStore | null = null;
switch (row.kind) {
case "CLICKHOUSE": {
const result = ClickhouseDataStoreConfig.safeParse(row.config);
if (!result.success) {
console.warn(
`[OrganizationDataStoresRegistry] Invalid config for OrganizationDataStore "${row.key}" (kind=CLICKHOUSE): ${result.error.message}`
);
continue;
}
const connection = await secretStore.getSecret(
ClickhouseConnectionSchema,
result.data.data.secretKey
);
if (!connection) {
console.warn(
`[OrganizationDataStoresRegistry] Secret "${result.data.data.secretKey}" not found for OrganizationDataStore "${row.key}" — skipping`
);
continue;
}
parsed = { kind: "CLICKHOUSE", url: connection.url };
break;
}
default: {
console.warn(
`[OrganizationDataStoresRegistry] Unknown kind "${row.kind}" for OrganizationDataStore "${row.key}" — skipping`
);
continue;
}
}
for (const orgId of row.organizationIds) {
const lookupKey = `${orgId}:${row.kind}`;
const existingWinner = winnerByLookupKey.get(lookupKey);
if (existingWinner) {
console.error(
`[OrganizationDataStoresRegistry] Overlapping OrganizationDataStore assignment for orgId="${orgId}" kind=${row.kind}: already routed to "${existingWinner}", ignoring "${row.key}". Pick one store per (org, kind) to resolve.`
);
continue;
}
winnerByLookupKey.set(lookupKey, row.key);
lookup.set(lookupKey, parsed);
}
}
this._lookup = lookup;
if (!this._loaded) {
this._loaded = true;
this._readyResolve();
}
}
async reload(): Promise<void> {
await this.loadFromDatabase();
}
#secretKey(key: string, kind: DataStoreKind) {
return `data-store:${key}:${kind.toLocaleLowerCase()}`;
}
async addDataStore({
key,
kind,
organizationIds,
config,
}: {
key: string;
kind: DataStoreKind;
organizationIds: string[];
config: any;
}) {
const secretKey = this.#secretKey(key, kind);
const secretStore = getSecretStore("DATABASE", { prismaClient: this._prisma });
await secretStore.setSecret(secretKey, config);
return this._prisma.organizationDataStore.create({
data: {
key,
organizationIds,
kind,
config: { version: 1, data: { secretKey } },
},
});
}
async updateDataStore({
key,
kind,
organizationIds,
config,
}: {
key: string;
kind: DataStoreKind;
organizationIds: string[];
config?: any;
}) {
const secretKey = this.#secretKey(key, kind);
if (config) {
const secretStore = getSecretStore("DATABASE", { prismaClient: this._prisma });
await secretStore.setSecret(secretKey, config);
}
return this._prisma.organizationDataStore.update({
where: {
key,
},
data: {
organizationIds,
kind: "CLICKHOUSE",
},
});
}
async deleteDataStore({ key, kind }: { key: string; kind: DataStoreKind }) {
const secretKey = this.#secretKey(key, kind);
const secretStore = getSecretStore("DATABASE", { prismaClient: this._prisma });
await secretStore.deleteSecret(secretKey).catch(() => {
// Secret may not exist — proceed with deletion
});
await this._prisma.organizationDataStore.delete({ where: { key } });
}
/**
* Returns the parsed data store config for the given organization and kind,
* or `null` if no override is configured (caller should use the default).
*/
get(organizationId: string, kind: DataStoreKind): ParsedDataStore | null {
if (!this._loaded) return null;
return this._lookup.get(`${organizationId}:${kind}`) ?? null;
}
}
apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts
+42
View File
@@ -0,0 +1,42 @@
import pRetry from "p-retry";
import { $replica } from "~/db.server";
import { env } from "~/env.server";
import { logger } from "~/services/logger.server";
import { signalsEmitter } from "~/services/signals.server";
import { singleton } from "~/utils/singleton";
import { OrganizationDataStoresRegistry } from "./organizationDataStoresRegistry.server";
export const organizationDataStoresRegistry = singleton("organizationDataStoresRegistry", () => {
const registry = new OrganizationDataStoresRegistry($replica);
// Runs as soon as this singleton is created (first import of this module). The
// registry’s `isReady` promise resolves when this eventually succeeds.
const startupLoadPromise = pRetry(() => registry.loadFromDatabase(), {
forever: true,
retries: 10,
minTimeout: 1_000,
maxTimeout: 60_000,
factor: 2,
onFailedAttempt: (error) => {
logger.warn("[OrganizationDataStoresRegistry] Startup load failed, retrying", {
attemptNumber: error.attemptNumber,
retriesLeft: error.retriesLeft,
error: error.message,
});
},
});
startupLoadPromise.catch((err) => {
console.error("[OrganizationDataStoresRegistry] Unexpected startup load failure", err);
});
const interval = setInterval(() => {
registry.reload().catch((err) => {
console.error("[OrganizationDataStoresRegistry] Failed to reload", err);
});
}, env.ORGANIZATION_DATA_STORES_RELOAD_INTERVAL_MS);
signalsEmitter.on("SIGTERM", () => clearInterval(interval));
signalsEmitter.on("SIGINT", () => clearInterval(interval));
return registry;
});
apps/webapp/app/services/queryService.server.ts
+3 -2
View File
@@ -11,7 +11,7 @@ import type { TableSchema, WhereClauseCondition } from "@internal/tsql";
import { z } from "zod";
import { prisma } from "~/db.server";
import { env } from "~/env.server";
import { queryClickhouseClient } from "./clickhouseInstance.server";
import { clickhouseFactory } from "./clickhouse/clickhouseFactoryInstance.server";
import {
queryConcurrencyLimiter,
DEFAULT_ORG_CONCURRENCY_LIMIT,
@@ -275,7 +275,8 @@ export async function executeQuery<TOut extends z.ZodSchema>(
environment: Object.fromEntries(environments.map((e) => [e.id, e.slug])),
};
const result = await executeTSQL(queryClickhouseClient.reader, {
const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "query");
const result = await executeTSQL(queryClickhouse.reader, {
...baseOptions,
schema: z.record(z.any()),
tableSchema: querySchemas,
apps/webapp/app/services/runsReplicationInstance.server.ts
+5 -18
View File
@@ -1,6 +1,6 @@
import { ClickHouse } from "@internal/clickhouse";
import invariant from "tiny-invariant";
import { env } from "~/env.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { singleton } from "~/utils/singleton";
import { meter, provider } from "~/v3/tracer.server";
import { RunsReplicationService } from "./runsReplicationService.server";
@@ -22,22 +22,8 @@ function initializeRunsReplicationInstance() {
console.log("🗃️ Runs replication service enabled");
const clickhouse = new ClickHouse({
url: env.RUN_REPLICATION_CLICKHOUSE_URL,
name: "runs-replication",
keepAlive: {
enabled: env.RUN_REPLICATION_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.RUN_REPLICATION_CLICKHOUSE_LOG_LEVEL,
compression: {
request: true,
},
maxOpenConnections: env.RUN_REPLICATION_MAX_OPEN_CONNECTIONS,
});
const service = new RunsReplicationService({
clickhouse: clickhouse,
clickhouseFactory,
pgConnectionUrl: DATABASE_URL,
serviceName: "runs-replication",
slotName: env.RUN_REPLICATION_SLOT_NAME,
@@ -72,8 +58,9 @@ function initializeRunsReplicationInstance() {
});
if (env.RUN_REPLICATION_ENABLED === "1") {
service
.start()
clickhouseFactory
.isReady()
.then(() => service.start())
.then(() => {
console.log("🗃️ Runs replication service started");
})
apps/webapp/app/services/runsReplicationService.server.ts
+129 -57
View File
@@ -1,5 +1,11 @@
import type { ClickHouse, TaskRunInsertArray, PayloadInsertArray } from "@internal/clickhouse";
import { getTaskRunField, getPayloadField } from "@internal/clickhouse";
import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server";
import {
type ClickHouse,
type PayloadInsertArray,
type TaskRunInsertArray,
getPayloadField,
getTaskRunField,
} from "@internal/clickhouse";
import { type RedisOptions } from "@internal/redis";
import {
LogicalReplicationClient,
@@ -21,7 +27,10 @@ import {
import { Logger, type LogLevel } from "@trigger.dev/core/logger";
import { tryCatch } from "@trigger.dev/core/utils";
import { parsePacketAsJson } from "@trigger.dev/core/v3/utils/ioSerialization";
import { unsafeExtractIdempotencyKeyScope, unsafeExtractIdempotencyKeyUser } from "@trigger.dev/core/v3/serverOnly";
import {
unsafeExtractIdempotencyKeyScope,
unsafeExtractIdempotencyKeyUser,
} from "@trigger.dev/core/v3/serverOnly";
import { RunAnnotations } from "@trigger.dev/core/v3";
import { type TaskRun } from "@trigger.dev/database";
import { nanoid } from "nanoid";
@@ -46,7 +55,7 @@ interface Transaction<T = any> {
}
export type RunsReplicationServiceOptions = {
clickhouse: ClickHouse;
clickhouseFactory: ClickhouseFactory;
pgConnectionUrl: string;
serviceName: string;
slotName: string;
@@ -560,16 +569,50 @@ export class RunsReplicationService {
const flushStartTime = performance.now();
await startSpan(this._tracer, "flushBatch", async (span) => {
const preparedInserts = await startSpan(this._tracer, "prepare_inserts", async (span) => {
const preparedInserts = await startSpan(this._tracer, "prepare_inserts", async () => {
return await Promise.all(batch.map(this.#prepareRunInserts.bind(this)));
});
const taskRunInserts = preparedInserts
.map(({ taskRunInsert }) => taskRunInsert)
.filter((x): x is TaskRunInsertArray => Boolean(x))
// batch inserts in clickhouse are more performant if the items
// are pre-sorted by the primary key
.sort((a, b) => {
const routeCache = new Map<string, ClickHouse>();
const groups = new Map<
ClickHouse,
{ taskRunInserts: TaskRunInsertArray[]; payloadInserts: PayloadInsertArray[] }
>();
for (let i = 0; i < batch.length; i++) {
const batchedRun = batch[i]!;
const prep = preparedInserts[i]!;
const { run } = batchedRun;
if (!run.organizationId || !run.environmentType) {
continue;
}
let client = routeCache.get(run.organizationId);
if (!client) {
client = this.options.clickhouseFactory.getClickhouseForOrganizationSync(
run.organizationId,
"replication"
);
routeCache.set(run.organizationId, client);
}
let group = groups.get(client);
if (!group) {
group = { taskRunInserts: [], payloadInserts: [] };
groups.set(client, group);
}
if (prep.taskRunInsert) {
group.taskRunInserts.push(prep.taskRunInsert);
}
if (prep.payloadInsert) {
group.payloadInserts.push(prep.payloadInsert);
}
}
const sortTaskRunInserts = (rows: TaskRunInsertArray[]) =>
rows.sort((a, b) => {
const aOrgId = getTaskRunField(a, "organization_id");
const bOrgId = getTaskRunField(b, "organization_id");
if (aOrgId !== bOrgId) {
@@ -596,41 +639,61 @@ export class RunsReplicationService {
return aRunId < bRunId ? -1 : 1;
});
const payloadInserts = preparedInserts
.map(({ payloadInsert }) => payloadInsert)
.filter((x): x is PayloadInsertArray => Boolean(x))
// batch inserts in clickhouse are more performant if the items
// are pre-sorted by the primary key
.sort((a, b) => {
const sortPayloadInserts = (rows: PayloadInsertArray[]) =>
rows.sort((a, b) => {
const aRunId = getPayloadField(a, "run_id");
const bRunId = getPayloadField(b, "run_id");
if (aRunId === bRunId) return 0;
return aRunId < bRunId ? -1 : 1;
});
span.setAttribute("task_run_inserts", taskRunInserts.length);
span.setAttribute("payload_inserts", payloadInserts.length);
const combinedTaskRunInserts: TaskRunInsertArray[] = [];
const combinedPayloadInserts: PayloadInsertArray[] = [];
let taskRunError: Error | null = null;
let payloadError: Error | null = null;
for (const [clickhouse, group] of groups) {
sortTaskRunInserts(group.taskRunInserts);
sortPayloadInserts(group.payloadInserts);
combinedTaskRunInserts.push(...group.taskRunInserts);
combinedPayloadInserts.push(...group.payloadInserts);
const [trErr] = await this.#insertWithRetry(
(attempt) => this.#insertTaskRunInserts(clickhouse, group.taskRunInserts, attempt),
"task run inserts",
flushId
);
if (trErr && !taskRunError) {
taskRunError = trErr;
}
const [plErr] = await this.#insertWithRetry(
(attempt) => this.#insertPayloadInserts(clickhouse, group.payloadInserts, attempt),
"payload inserts",
flushId
);
if (plErr && !payloadError) {
payloadError = plErr;
}
if (!trErr) {
this._taskRunsInsertedCounter.add(group.taskRunInserts.length);
}
if (!plErr) {
this._payloadsInsertedCounter.add(group.payloadInserts.length);
}
}
span.setAttribute("task_run_inserts", combinedTaskRunInserts.length);
span.setAttribute("payload_inserts", combinedPayloadInserts.length);
this.logger.debug("Flushing inserts", {
flushId,
taskRunInserts: taskRunInserts.length,
payloadInserts: payloadInserts.length,
taskRunInserts: combinedTaskRunInserts.length,
payloadInserts: combinedPayloadInserts.length,
clickhouseGroups: groups.size,
});
// Insert task runs and payloads with retry logic for connection errors
const [taskRunError, taskRunResult] = await this.#insertWithRetry(
(attempt) => this.#insertTaskRunInserts(taskRunInserts, attempt),
"task run inserts",
flushId
);
const [payloadError, payloadResult] = await this.#insertWithRetry(
(attempt) => this.#insertPayloadInserts(payloadInserts, attempt),
"payload inserts",
flushId
);
// Log any errors that occurred
if (taskRunError) {
this.logger.error("Error inserting task run inserts", {
error: taskRunError,
@@ -649,27 +712,22 @@ export class RunsReplicationService {
this.logger.debug("Flushed inserts", {
flushId,
taskRunInserts: taskRunInserts.length,
payloadInserts: payloadInserts.length,
taskRunInserts: combinedTaskRunInserts.length,
payloadInserts: combinedPayloadInserts.length,
});
this.events.emit("batchFlushed", { flushId, taskRunInserts, payloadInserts });
this.events.emit("batchFlushed", {
flushId,
taskRunInserts: combinedTaskRunInserts,
payloadInserts: combinedPayloadInserts,
});
// Record metrics
const flushDurationMs = performance.now() - flushStartTime;
const hasErrors = taskRunError !== null || payloadError !== null;
this._batchSizeHistogram.record(batch.length);
this._flushDurationHistogram.record(flushDurationMs);
this._batchesFlushedCounter.add(1, { success: !hasErrors });
if (!taskRunError) {
this._taskRunsInsertedCounter.add(taskRunInserts.length);
}
if (!payloadError) {
this._payloadsInsertedCounter.add(payloadInserts.length);
}
});
}
@@ -770,10 +828,17 @@ export class RunsReplicationService {
};
}
async #insertTaskRunInserts(taskRunInserts: TaskRunInsertArray[], attempt: number) {
async #insertTaskRunInserts(
clickhouse: ClickHouse,
taskRunInserts: TaskRunInsertArray[],
attempt: number
) {
if (taskRunInserts.length === 0) {
return;
}
return await startSpan(this._tracer, "insertTaskRunsInserts", async (span) => {
const [insertError, insertResult] =
await this.options.clickhouse.taskRuns.insertCompactArrays(taskRunInserts, {
await clickhouse.taskRuns.insertCompactArrays(taskRunInserts, {
params: {
clickhouse_settings: this.#getClickhouseInsertSettings(),
},
@@ -793,10 +858,17 @@ export class RunsReplicationService {
});
}
async #insertPayloadInserts(payloadInserts: PayloadInsertArray[], attempt: number) {
async #insertPayloadInserts(
clickhouse: ClickHouse,
payloadInserts: PayloadInsertArray[],
attempt: number
) {
if (payloadInserts.length === 0) {
return;
}
return await startSpan(this._tracer, "insertPayloadInserts", async (span) => {
const [insertError, insertResult] =
await this.options.clickhouse.taskRuns.insertPayloadsCompactArrays(payloadInserts, {
await clickhouse.taskRuns.insertPayloadsCompactArrays(payloadInserts, {
params: {
clickhouse_settings: this.#getClickhouseInsertSettings(),
},
@@ -860,12 +932,13 @@ export class RunsReplicationService {
const errorData = { data: run.error };
// Calculate error fingerprint for failed runs
const errorFingerprint = (
const errorFingerprint =
!this._disableErrorFingerprinting &&
['SYSTEM_FAILURE', 'CRASHED', 'INTERRUPTED', 'COMPLETED_WITH_ERRORS', 'TIMED_OUT'].includes(run.status)
)
? calculateErrorFingerprint(run.error)
: '';
["SYSTEM_FAILURE", "CRASHED", "INTERRUPTED", "COMPLETED_WITH_ERRORS", "TIMED_OUT"].includes(
run.status
)
? calculateErrorFingerprint(run.error)
: "";
const annotations = this.#parseAnnotations(run.annotations);
@@ -979,7 +1052,6 @@ export class RunsReplicationService {
return { data: parsedData };
}
}
export type ConcurrentFlushSchedulerConfig<T> = {
apps/webapp/app/services/sessionsReplicationInstance.server.ts
+32 -16
View File
@@ -1,9 +1,10 @@
import { ClickHouse } from "@internal/clickhouse";
import invariant from "tiny-invariant";
import { env } from "~/env.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { singleton } from "~/utils/singleton";
import { meter, provider } from "~/v3/tracer.server";
import { SessionsReplicationService } from "./sessionsReplicationService.server";
import { signalsEmitter } from "./signals.server";
export const sessionsReplicationInstance = singleton(
"sessionsReplicationInstance",
@@ -21,22 +22,8 @@ function initializeSessionsReplicationInstance() {
console.log("🗃️ Sessions replication service enabled");
const clickhouse = new ClickHouse({
url: env.SESSION_REPLICATION_CLICKHOUSE_URL,
name: "sessions-replication",
keepAlive: {
enabled: env.SESSION_REPLICATION_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.SESSION_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.SESSION_REPLICATION_CLICKHOUSE_LOG_LEVEL,
compression: {
request: true,
},
maxOpenConnections: env.SESSION_REPLICATION_MAX_OPEN_CONNECTIONS,
});
const service = new SessionsReplicationService({
clickhouse: clickhouse,
clickhouseFactory,
pgConnectionUrl: DATABASE_URL,
serviceName: "sessions-replication",
slotName: env.SESSION_REPLICATION_SLOT_NAME,
@@ -68,5 +55,34 @@ function initializeSessionsReplicationInstance() {
insertStrategy: env.SESSION_REPLICATION_INSERT_STRATEGY,
});
if (env.SESSION_REPLICATION_ENABLED === "1") {
// Gate start() on the org data-stores registry being loaded. Starting earlier would
// race the registry load — sync factory lookups would return `null` and route org-scoped
// sessions to the default ClickHouse, writing them to the wrong cluster.
clickhouseFactory
.isReady()
.then(() => service.start())
.then(() => {
console.log("🗃️ Sessions replication service started");
})
.catch((error) => {
console.error("🗃️ Sessions replication service failed to start", {
error,
});
});
// SIGTERM/SIGINT fire during process teardown; wrap the async shutdown so an
// unhandled rejection doesn't bubble past process exit.
const shutdownSessionsReplication = () => {
service.shutdown().catch((error) => {
console.error("🗃️ Sessions replication service shutdown error", {
error,
});
});
};
signalsEmitter.on("SIGTERM", shutdownSessionsReplication);
signalsEmitter.on("SIGINT", shutdownSessionsReplication);
}
return service;
}
apps/webapp/app/services/sessionsReplicationService.server.ts
+75 -25
View File
@@ -22,6 +22,7 @@ import { Logger, type LogLevel } from "@trigger.dev/core/logger";
import { tryCatch } from "@trigger.dev/core/utils";
import { type Session } from "@trigger.dev/database";
import EventEmitter from "node:events";
import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server";
import { ConcurrentFlushScheduler } from "./runsReplicationService.server";
interface TransactionEvent<T = any> {
@@ -40,7 +41,7 @@ interface Transaction<T = any> {
}
export type SessionsReplicationServiceOptions = {
clickhouse: ClickHouse;
clickhouseFactory: ClickhouseFactory;
pgConnectionUrl: string;
serviceName: string;
slotName: string;
@@ -537,11 +538,38 @@ export class SessionsReplicationService {
const flushStartTime = performance.now();
await startSpan(this._tracer, "flushBatch", async (span) => {
const sessionInserts = batch
.map((item) => toSessionInsertArray(item.session, item._version, item.event === "delete"))
// batch inserts in clickhouse are more performant if the items
// are pre-sorted by the primary key
.sort((a, b) => {
const routeCache = new Map<string, ClickHouse>();
const groups = new Map<ClickHouse, { sessionInserts: SessionInsertArray[] }>();
for (const item of batch) {
if (!item.session.organizationId) {
continue;
}
let client = routeCache.get(item.session.organizationId);
if (!client) {
client = this.options.clickhouseFactory.getClickhouseForOrganizationSync(
item.session.organizationId,
"sessions_replication"
);
routeCache.set(item.session.organizationId, client);
}
let group = groups.get(client);
if (!group) {
group = { sessionInserts: [] };
groups.set(client, group);
}
group.sessionInserts.push(
toSessionInsertArray(item.session, item._version, item.event === "delete")
);
}
// batch inserts in clickhouse are more performant if the items
// are pre-sorted by the primary key
const sortSessionInserts = (rows: SessionInsertArray[]) =>
rows.sort((a, b) => {
const aOrgId = getSessionField(a, "organization_id");
const bOrgId = getSessionField(b, "organization_id");
if (aOrgId !== bOrgId) {
@@ -568,19 +596,37 @@ export class SessionsReplicationService {
return aSessionId < bSessionId ? -1 : 1;
});
span.setAttribute("session_inserts", sessionInserts.length);
const combinedSessionInserts: SessionInsertArray[] = [];
let sessionError: Error | null = null;
// Sequential per-group flush — matches runsReplicationService for the same reason
// (parallel writes have hit Linux net.ipv4.tcp_wmem buffer pressure at high throughput).
for (const [clickhouse, group] of groups) {
sortSessionInserts(group.sessionInserts);
combinedSessionInserts.push(...group.sessionInserts);
const [insErr] = await this.#insertWithRetry(
(attempt) => this.#insertSessionInserts(clickhouse, group.sessionInserts, attempt),
"session inserts",
flushId
);
if (insErr && !sessionError) {
sessionError = insErr;
}
if (!insErr) {
this._sessionsInsertedCounter.add(group.sessionInserts.length);
}
}
span.setAttribute("session_inserts", combinedSessionInserts.length);
this.logger.debug("Flushing inserts", {
flushId,
sessionInserts: sessionInserts.length,
sessionInserts: combinedSessionInserts.length,
clickhouseGroups: groups.size,
});
const [sessionError, sessionResult] = await this.#insertWithRetry(
(attempt) => this.#insertSessionInserts(sessionInserts, attempt),
"session inserts",
flushId
);
if (sessionError) {
this.logger.error("Error inserting session inserts", {
error: sessionError,
@@ -591,22 +637,17 @@ export class SessionsReplicationService {
this.logger.debug("Flushed inserts", {
flushId,
sessionInserts: sessionInserts.length,
sessionInserts: combinedSessionInserts.length,
});
this.events.emit("batchFlushed", { flushId, sessionInserts });
this.events.emit("batchFlushed", { flushId, sessionInserts: combinedSessionInserts });
// Record metrics
const flushDurationMs = performance.now() - flushStartTime;
const hasErrors = sessionError !== null;
this._batchSizeHistogram.record(batch.length);
this._flushDurationHistogram.record(flushDurationMs);
this._batchesFlushedCounter.add(1, { success: !hasErrors });
if (!sessionError) {
this._sessionsInsertedCounter.add(sessionInserts.length);
}
});
}
@@ -706,14 +747,23 @@ export class SessionsReplicationService {
};
}
async #insertSessionInserts(sessionInserts: SessionInsertArray[], attempt: number) {
async #insertSessionInserts(
clickhouse: ClickHouse,
sessionInserts: SessionInsertArray[],
attempt: number
) {
if (sessionInserts.length === 0) {
return;
}
return await startSpan(this._tracer, "insertSessionInserts", async (span) => {
const [insertError, insertResult] =
await this.options.clickhouse.sessions.insertCompactArrays(sessionInserts, {
const [insertError, insertResult] = await clickhouse.sessions.insertCompactArrays(
sessionInserts,
{
params: {
clickhouse_settings: this.#getClickhouseInsertSettings(),
},
});
}
);
if (insertError) {
this.logger.error("Error inserting session inserts attempt", {
apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts
+51 -13
View File
@@ -1,6 +1,7 @@
import type {
ClickHouse,
LlmMetricsV1Input,
MetricsV1Input,
TaskEventDetailedSummaryV1Result,
TaskEventDetailsV1Result,
TaskEventSummaryV1Result,
@@ -96,6 +97,10 @@ export type ClickhouseEventRepositoryConfig = {
llmMetricsFlushInterval?: number;
llmMetricsMaxBatchSize?: number;
llmMetricsMaxConcurrency?: number;
/** OTLP / task metrics_v1 flush scheduler config */
otlpMetricsBatchSize?: number;
otlpMetricsFlushInterval?: number;
otlpMetricsMaxConcurrency?: number;
};
/**
@@ -107,6 +112,7 @@ export class ClickhouseEventRepository implements IEventRepository {
private _config: ClickhouseEventRepositoryConfig;
private readonly _flushScheduler: DynamicFlushScheduler<TaskEventV1Input | TaskEventV2Input>;
private readonly _llmMetricsFlushScheduler: DynamicFlushScheduler<LlmMetricsV1Input>;
private readonly _otlpMetricsFlushScheduler: DynamicFlushScheduler<MetricsV1Input>;
private _tracer: Tracer;
private _version: "v1" | "v2";
/**
@@ -149,6 +155,15 @@ export class ClickhouseEventRepository implements IEventRepository {
memoryPressureThreshold: config.llmMetricsMaxBatchSize ?? 10000,
loadSheddingEnabled: false,
});
this._otlpMetricsFlushScheduler = new DynamicFlushScheduler({
batchSize: config.otlpMetricsBatchSize ?? 10000,
flushInterval: config.otlpMetricsFlushInterval ?? 1000,
callback: this.#flushOtelMetricsBatch.bind(this),
minConcurrency: 1,
maxConcurrency: config.otlpMetricsMaxConcurrency ?? 3,
loadSheddingEnabled: false,
});
}
get version() {
@@ -376,24 +391,42 @@ export class ClickhouseEventRepository implements IEventRepository {
typeof retryError === "object" && retryError !== null && "message" in retryError
? String((retryError as { message?: unknown }).message ?? "")
: String(retryError);
logger.error(
"Dropped batch after sanitize-retry still hit ClickHouse JSON parse error",
{
flushId,
contextLabel,
batchSize: rows.length,
permanentlyDroppedBatches: this._permanentlyDroppedBatches,
sampleRow: JSON.stringify(rows[0] ?? null).slice(0, 1024),
firstError: firstMessage.split("\n")[0],
retryError: retryMessage.split("\n")[0],
}
);
logger.error("Dropped batch after sanitize-retry still hit ClickHouse JSON parse error", {
flushId,
contextLabel,
batchSize: rows.length,
permanentlyDroppedBatches: this._permanentlyDroppedBatches,
sampleRow: JSON.stringify(rows[0] ?? null).slice(0, 1024),
firstError: firstMessage.split("\n")[0],
retryError: retryMessage.split("\n")[0],
});
return { kind: "dropped" };
}
}
}
async #flushOtelMetricsBatch(flushId: string, rows: MetricsV1Input[]) {
await startSpan(this._tracer, "flushOtelMetricsBatch", async (span) => {
span.setAttribute("flush_id", flushId);
span.setAttribute("row_count", rows.length);
const [insertError] = await this._clickhouse.metrics.insert(rows, {
params: {
clickhouse_settings: this.#getClickhouseInsertSettings(),
},
});
if (insertError) {
throw insertError;
}
logger.info("ClickhouseEventRepository.flushOtelMetricsBatch Inserted OTLP metrics batch", {
rows: rows.length,
});
});
}
#createLlmMetricsInput(event: CreateEventInput): LlmMetricsV1Input {
const llmMetrics = event._llmMetrics!;
@@ -452,7 +485,7 @@ export class ClickhouseEventRepository implements IEventRepository {
await tracePubSub.publish(events.map((e) => e.trace_id));
}
async insertMany(events: CreateEventInput[]): Promise<void> {
insertMany(events: CreateEventInput[]): void {
this.addToBatch(events.flatMap((event) => this.createEventToTaskEventV1Input(event)));
// Dual-write LLM metrics records for spans with cost enrichment
@@ -469,6 +502,11 @@ export class ClickhouseEventRepository implements IEventRepository {
this.insertMany(events);
}
insertManyMetrics(rows: MetricsV1Input[]): void {
if (rows.length === 0) return;
this._otlpMetricsFlushScheduler.addToBatch(rows);
}
private createEventToTaskEventV1Input(event: CreateEventInput): TaskEventV1Input[] {
return [
{
apps/webapp/app/v3/eventRepository/clickhouseEventRepositoryInstance.server.ts
-112
View File
@@ -1,112 +0,0 @@
import { ClickHouse } from "@internal/clickhouse";
import { env } from "~/env.server";
import { singleton } from "~/utils/singleton";
import { ClickhouseEventRepository } from "./clickhouseEventRepository.server";
export const clickhouseEventRepository = singleton(
"clickhouseEventRepository",
initializeClickhouseRepository
);
export const clickhouseEventRepositoryV2 = singleton(
"clickhouseEventRepositoryV2",
initializeClickhouseRepositoryV2
);
function getClickhouseClient() {
if (!env.EVENTS_CLICKHOUSE_URL) {
throw new Error("EVENTS_CLICKHOUSE_URL is not set");
}
const url = new URL(env.EVENTS_CLICKHOUSE_URL);
url.searchParams.delete("secure");
return new ClickHouse({
url: url.toString(),
name: "task-events",
keepAlive: {
enabled: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
idleSocketTtl: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
},
logLevel: env.EVENTS_CLICKHOUSE_LOG_LEVEL,
compression: {
request: env.EVENTS_CLICKHOUSE_COMPRESSION_REQUEST === "1",
},
maxOpenConnections: env.EVENTS_CLICKHOUSE_MAX_OPEN_CONNECTIONS,
});
}
function initializeClickhouseRepository() {
if (!env.EVENTS_CLICKHOUSE_URL) {
throw new Error("EVENTS_CLICKHOUSE_URL is not set");
}
const url = new URL(env.EVENTS_CLICKHOUSE_URL);
url.searchParams.delete("secure");
const safeUrl = new URL(url.toString());
safeUrl.password = "redacted";
console.log("🗃️ Initializing Clickhouse event repository (v1)", { url: safeUrl.toString() });
const clickhouse = getClickhouseClient();
const repository = new ClickhouseEventRepository({
clickhouse: clickhouse,
batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
maximumTraceDetailedSummaryViewCount:
env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",
asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE,
asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS,
startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS,
llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE,
llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS,
llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE,
llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY,
version: "v1",
});
return repository;
}
function initializeClickhouseRepositoryV2() {
if (!env.EVENTS_CLICKHOUSE_URL) {
throw new Error("EVENTS_CLICKHOUSE_URL is not set");
}
const url = new URL(env.EVENTS_CLICKHOUSE_URL);
url.searchParams.delete("secure");
const safeUrl = new URL(url.toString());
safeUrl.password = "redacted";
console.log("🗃️ Initializing Clickhouse event repository (v2)", { url: safeUrl.toString() });
const clickhouse = getClickhouseClient();
const repository = new ClickhouseEventRepository({
clickhouse: clickhouse,
batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
maximumTraceDetailedSummaryViewCount:
env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",
asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE,
asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS,
llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE,
llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS,
llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE,
llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY,
version: "v2",
});
return repository;
}
apps/webapp/app/v3/eventRepository/eventRepository.server.ts
+4 -1
View File
@@ -18,6 +18,7 @@ import {
unflattenAttributes,
} from "@trigger.dev/core/v3";
import { serializeTraceparent } from "@trigger.dev/core/v3/isomorphic";
import type { MetricsV1Input } from "@internal/clickhouse";
import { Prisma, TaskEvent, TaskEventKind } from "@trigger.dev/database";
import { nanoid } from "nanoid";
import { Gauge } from "prom-client";
@@ -151,7 +152,7 @@ export class EventRepository implements IEventRepository {
await this.#flushBatch(nanoid(), [this.#createableEventToPrismaEvent(event)]);
}
async insertMany(events: CreateEventInput[]) {
insertMany(events: CreateEventInput[]) {
this._flushScheduler.addToBatch(events.map(this.#createableEventToPrismaEvent));
}
@@ -159,6 +160,8 @@ export class EventRepository implements IEventRepository {
await this.#flushBatchWithReturn(nanoid(), events.map(this.#createableEventToPrismaEvent));
}
insertManyMetrics(_rows: MetricsV1Input[]): void {}
async completeSuccessfulRunEvent({ run, endTime }: { run: CompleteableTaskRun; endTime?: Date }) {
const startTime = convertDateToNanoseconds(run.createdAt);
apps/webapp/app/v3/eventRepository/eventRepository.types.ts
+3 -1
View File
@@ -14,6 +14,7 @@ import type {
TaskEventStatus,
TaskRun,
} from "@trigger.dev/database";
import type { MetricsV1Input } from "@internal/clickhouse";
import type { DetailedTraceEvent, TaskEventStoreTable } from "../taskEventStore.server";
export type { ExceptionEventProperties };
@@ -345,8 +346,9 @@ export type TraceDetailedSummary = {
export interface IEventRepository {
maximumLiveReloadingSetting: number;
// Event insertion methods
insertMany(events: CreateEventInput[]): Promise<void>;
insertMany(events: CreateEventInput[]): void;
insertManyImmediate(events: CreateEventInput[]): Promise<void>;
insertManyMetrics(rows: MetricsV1Input[]): void;
// Run event completion methods
completeSuccessfulRunEvent(params: { run: CompleteableTaskRun; endTime?: Date }): Promise<void>;
apps/webapp/app/v3/eventRepository/index.server.ts
+96 -49
View File
@@ -1,29 +1,12 @@
import { env } from "~/env.server";
import { eventRepository } from "./eventRepository.server";
import {
clickhouseEventRepository,
clickhouseEventRepositoryV2,
} from "./clickhouseEventRepositoryInstance.server";
import { IEventRepository, TraceEventOptions } from "./eventRepository.types";
import { type IEventRepository, type TraceEventOptions } from "./eventRepository.types";
import { prisma } from "~/db.server";
import { logger } from "~/services/logger.server";
import { FEATURE_FLAG } from "../featureFlags";
import { flag } from "../featureFlags.server";
import { getTaskEventStore } from "../taskEventStore.server";
export function resolveEventRepositoryForStore(store: string | undefined): IEventRepository {
const taskEventStore = store ?? env.EVENT_REPOSITORY_DEFAULT_STORE;
if (taskEventStore === "clickhouse_v2") {
return clickhouseEventRepositoryV2;
}
if (taskEventStore === "clickhouse") {
return clickhouseEventRepository;
}
return eventRepository;
}
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
export const EVENT_STORE_TYPES = {
POSTGRES: "postgres",
@@ -33,6 +16,50 @@ export const EVENT_STORE_TYPES = {
export type EventStoreType = (typeof EVENT_STORE_TYPES)[keyof typeof EVENT_STORE_TYPES];
/**
* Resolve the event repository for a run's persisted `taskEventStore` value and org.
* Postgres-backed runs use the Prisma `eventRepository`; ClickHouse-backed runs use
* `clickhouseFactory.getEventRepositoryForOrganizationSync`.
*
* Intentionally NOT exported. Sync resolution can race the org data-stores
* registry load and silently route writes to the default ClickHouse instead of
* the org's configured override. Hot paths that genuinely cannot afford to await
* (OTEL exporter, replication services) call `clickhouseFactory.getEvent…Sync`
* directly and gate startup on `clickhouseFactory.isReady()`. Everything else
* should use {@link getEventRepositoryForStore}, the async variant below.
*/
function resolveEventRepositoryForStore(
store: string,
organizationId: string
): IEventRepository {
if (store === EVENT_STORE_TYPES.CLICKHOUSE || store === EVENT_STORE_TYPES.CLICKHOUSE_V2) {
return clickhouseFactory.getEventRepositoryForOrganizationSync(store, organizationId)
.repository;
}
return eventRepository;
}
/**
* Async variant of {@link resolveEventRepositoryForStore}. Awaits the factory's
* registry readiness before returning the ClickHouse event repository; for
* non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed
* runs) it returns the Prisma event repository without ever touching the
* factory — so the factory never needs to know about Postgres.
*/
export async function getEventRepositoryForStore(
store: string,
organizationId: string
): Promise<IEventRepository> {
if (store !== EVENT_STORE_TYPES.CLICKHOUSE && store !== EVENT_STORE_TYPES.CLICKHOUSE_V2) {
return eventRepository;
}
const { repository } = await clickhouseFactory.getEventRepositoryForOrganization(
store,
organizationId
);
return repository;
}
export async function getConfiguredEventRepository(
organizationId: string
): Promise<{ repository: IEventRepository; store: EventStoreType }> {
@@ -59,62 +86,79 @@ export async function getConfiguredEventRepository(
);
if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE_V2) {
return { repository: clickhouseEventRepositoryV2, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 };
const { repository: resolvedRepository } =
await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 };
}
if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE) {
return { repository: clickhouseEventRepository, store: EVENT_STORE_TYPES.CLICKHOUSE };
const { repository: resolvedRepository } =
await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE };
}
return { repository: eventRepository, store: EVENT_STORE_TYPES.POSTGRES };
}
export async function getEventRepository(
organizationId: string,
featureFlags: Record<string, unknown> | undefined,
parentStore: string | undefined
): Promise<{ repository: IEventRepository; store: string }> {
if (typeof parentStore === "string") {
if (parentStore === "clickhouse_v2") {
return { repository: clickhouseEventRepositoryV2, store: "clickhouse_v2" };
const taskEventStore = parentStore ?? (await resolveTaskEventRepositoryFlag(featureFlags));
// Non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed
// runs, or the legacy "postgres" value) resolve to the Prisma event repo.
if (
taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE &&
taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2
) {
return { repository: eventRepository, store: getTaskEventStore() };
}
const { repository: resolvedRepository } =
await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
switch (taskEventStore) {
case EVENT_STORE_TYPES.CLICKHOUSE_V2: {
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 };
}
if (parentStore === "clickhouse") {
return { repository: clickhouseEventRepository, store: "clickhouse" };
} else {
case EVENT_STORE_TYPES.CLICKHOUSE: {
return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE };
}
default: {
return { repository: eventRepository, store: getTaskEventStore() };
}
}
const taskEventRepository = await resolveTaskEventRepositoryFlag(featureFlags);
if (taskEventRepository === "clickhouse_v2") {
return { repository: clickhouseEventRepositoryV2, store: "clickhouse_v2" };
}
if (taskEventRepository === "clickhouse") {
return { repository: clickhouseEventRepository, store: "clickhouse" };
}
return { repository: eventRepository, store: getTaskEventStore() };
}
export async function getV3EventRepository(
organizationId: string,
parentStore: string | undefined
): Promise<{ repository: IEventRepository; store: string }> {
if (typeof parentStore === "string") {
if (parentStore === "clickhouse_v2") {
return { repository: clickhouseEventRepositoryV2, store: "clickhouse_v2" };
}
if (parentStore === "clickhouse") {
return { repository: clickhouseEventRepository, store: "clickhouse" };
} else {
return { repository: eventRepository, store: getTaskEventStore() };
// Support legacy Postgres store for self-hosters and runs persisted with a
// non-ClickHouse store — fall back to the Prisma-based event repository.
if (
parentStore !== EVENT_STORE_TYPES.CLICKHOUSE &&
parentStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2
) {
return { repository: eventRepository, store: parentStore };
}
const { repository: resolvedRepository } =
await clickhouseFactory.getEventRepositoryForOrganization(parentStore, organizationId);
return { repository: resolvedRepository, store: parentStore };
}
if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse_v2") {
return { repository: clickhouseEventRepositoryV2, store: "clickhouse_v2" };
const { repository: resolvedRepository } =
await clickhouseFactory.getEventRepositoryForOrganization("clickhouse_v2", organizationId);
return { repository: resolvedRepository, store: "clickhouse_v2" };
} else if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse") {
return { repository: clickhouseEventRepository, store: "clickhouse" };
const { repository: resolvedRepository } =
await clickhouseFactory.getEventRepositoryForOrganization("clickhouse", organizationId);
return { repository: resolvedRepository, store: "clickhouse" };
} else {
return { repository: eventRepository, store: getTaskEventStore() };
}
@@ -203,7 +247,10 @@ async function recordRunEvent(
};
}
const $eventRepository = resolveEventRepositoryForStore(foundRun.taskEventStore);
const $eventRepository = await getEventRepositoryForStore(
foundRun.taskEventStore,
foundRun.runtimeEnvironment.organizationId
);
const { attributes, startTime, ...optionsRest } = options;
apps/webapp/app/v3/otlpExporter.server.ts
+89 -71
View File
@@ -20,15 +20,10 @@ import {
} from "@trigger.dev/otlp-importer";
import type { MetricsV1Input } from "@internal/clickhouse";
import { logger } from "~/services/logger.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
import { ClickhouseEventRepository } from "./eventRepository/clickhouseEventRepository.server";
import {
clickhouseEventRepository,
clickhouseEventRepositoryV2,
} from "./eventRepository/clickhouseEventRepositoryInstance.server";
import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { generateSpanId } from "./eventRepository/common.server";
import { EventRepository, eventRepository } from "./eventRepository/eventRepository.server";
import type {
CreatableEventKind,
CreatableEventStatus,
@@ -41,18 +36,23 @@ import { waitForLlmPricingReady } from "./llmPricingRegistry.server";
import { env } from "~/env.server";
import { singleton } from "~/utils/singleton";
type OTLPExporterConfig = {
clickhouseFactory: ClickhouseFactory;
verbose: boolean;
spanAttributeValueLengthLimit: number;
};
class OTLPExporter {
private _tracer: Tracer;
private readonly _clickhouseFactory: ClickhouseFactory;
private readonly _verbose: boolean;
private readonly _spanAttributeValueLengthLimit: number;
constructor(
private readonly _eventRepository: EventRepository,
private readonly _clickhouseEventRepository: ClickhouseEventRepository,
private readonly _clickhouseEventRepositoryV2: ClickhouseEventRepository,
private readonly _metricsFlushScheduler: DynamicFlushScheduler<MetricsV1Input>,
private readonly _verbose: boolean,
private readonly _spanAttributeValueLengthLimit: number
) {
constructor(config: OTLPExporterConfig) {
this._tracer = trace.getTracer("otlp-exporter");
this._clickhouseFactory = config.clickhouseFactory;
this._verbose = config.verbose;
this._spanAttributeValueLengthLimit = config.spanAttributeValueLengthLimit;
}
async exportTraces(request: ExportTraceServiceRequest): Promise<ExportTraceServiceResponse> {
@@ -73,23 +73,16 @@ class OTLPExporter {
});
}
async exportMetrics(
request: ExportMetricsServiceRequest
): Promise<ExportMetricsServiceResponse> {
async exportMetrics(request: ExportMetricsServiceRequest): Promise<ExportMetricsServiceResponse> {
return await startSpan(this._tracer, "exportMetrics", async (span) => {
const rows = this.#filterResourceMetrics(request.resourceMetrics).flatMap(
(resourceMetrics) => {
return convertMetricsToClickhouseRows(
resourceMetrics,
this._spanAttributeValueLengthLimit
);
}
const rows = this.#filterResourceMetrics(request.resourceMetrics).flatMap((resourceMetrics) =>
convertMetricsToClickhouseRows(resourceMetrics, this._spanAttributeValueLengthLimit)
);
span.setAttribute("metric_row_count", rows.length);
if (rows.length > 0) {
this._metricsFlushScheduler.addToBatch(rows);
await this.#exportMetricRows(rows);
}
return ExportMetricsServiceResponse.create();
@@ -117,40 +110,73 @@ class OTLPExporter {
async #exportEvents(
eventsWithStores: { events: Array<CreateEventInput>; taskEventStore: string }[]
) {
const eventsGroupedByStore = eventsWithStores.reduce((acc, { events, taskEventStore }) => {
acc[taskEventStore] = acc[taskEventStore] || [];
acc[taskEventStore].push(...events);
return acc;
}, {} as Record<string, Array<CreateEventInput>>);
await waitForLlmPricingReady();
// Group by unique event repositories
const routeCache = new Map<string, { key: string; repository: IEventRepository }>();
const groups = new Map<string, { repository: IEventRepository; events: CreateEventInput[] }>();
for (const { events, taskEventStore } of eventsWithStores) {
for (const event of events) {
const routeKey = `${event.organizationId}\0${taskEventStore}`;
let resolved = routeCache.get(routeKey);
if (!resolved) {
resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
taskEventStore,
event.organizationId
);
routeCache.set(routeKey, resolved);
}
let group = groups.get(resolved.key);
if (!group) {
group = { repository: resolved.repository, events: [] };
groups.set(resolved.key, group);
}
group.events.push(event);
}
}
let eventCount = 0;
for (const [store, events] of Object.entries(eventsGroupedByStore)) {
const eventRepository = this.#getEventRepositoryForStore(store);
await waitForLlmPricingReady();
for (const [repoKey, { repository, events }] of groups) {
const enrichedEvents = enrichCreatableEvents(events);
this.#logEventsVerbose(enrichedEvents, `exportEvents ${store}`);
this.#logEventsVerbose(enrichedEvents, `exportEvents ${repoKey}`);
eventCount += enrichedEvents.length;
await eventRepository.insertMany(enrichedEvents);
repository.insertMany(enrichedEvents);
}
return eventCount;
}
#getEventRepositoryForStore(store: string): IEventRepository {
if (store === "clickhouse") {
return this._clickhouseEventRepository;
async #exportMetricRows(rows: MetricsV1Input[]): Promise<void> {
const routeCache = new Map<string, { key: string; repository: IEventRepository }>();
const groups = new Map<string, { repository: IEventRepository; rows: MetricsV1Input[] }>();
for (const row of rows) {
const routeKey = row.organization_id;
let resolved = routeCache.get(routeKey);
if (!resolved) {
resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
"clickhouse_v2",
row.organization_id
);
routeCache.set(routeKey, resolved);
}
let group = groups.get(resolved.key);
if (!group) {
group = { repository: resolved.repository, rows: [] };
groups.set(resolved.key, group);
}
group.rows.push(row);
}
if (store === "clickhouse_v2") {
return this._clickhouseEventRepositoryV2;
for (const [, { repository, rows: groupedRows }] of groups) {
repository.insertManyMetrics(groupedRows);
}
return this._eventRepository;
}
#logEventsVerbose(events: CreateEventInput[], prefix: string) {
@@ -392,7 +418,10 @@ function convertSpansToCreateableEvents(
SemanticInternalAttributes.METADATA
);
const runTags = extractArrayAttribute(span.attributes ?? [], SemanticInternalAttributes.RUN_TAGS);
const runTags = extractArrayAttribute(
span.attributes ?? [],
SemanticInternalAttributes.RUN_TAGS
);
const properties =
truncateAttributes(
@@ -463,7 +492,10 @@ function floorToTenSecondBucket(timeUnixNano: bigint | number): string {
const flooredMs = Math.floor(epochMs / 10_000) * 10_000;
const date = new Date(flooredMs);
// Format as ClickHouse DateTime: YYYY-MM-DD HH:MM:SS
return date.toISOString().replace("T", " ").replace(/\.\d{3}Z$/, "");
return date
.toISOString()
.replace("T", " ")
.replace(/\.\d{3}Z$/, "");
}
function convertMetricsToClickhouseRows(
@@ -583,8 +615,7 @@ function resolveDataPointContext(
attributes: Record<string, unknown>;
} {
const runId =
resourceCtx.runId ??
extractStringAttribute(dpAttributes, SemanticInternalAttributes.RUN_ID);
resourceCtx.runId ?? extractStringAttribute(dpAttributes, SemanticInternalAttributes.RUN_ID);
const taskSlug =
resourceCtx.taskSlug ??
extractStringAttribute(dpAttributes, SemanticInternalAttributes.TASK_SLUG);
@@ -1171,26 +1202,13 @@ function hasUnpairedSurrogateAtEnd(str: string): boolean {
export const otlpExporter = singleton("otlpExporter", initializeOTLPExporter);
function initializeOTLPExporter() {
const metricsFlushScheduler = new DynamicFlushScheduler<MetricsV1Input>({
batchSize: env.METRICS_CLICKHOUSE_BATCH_SIZE,
flushInterval: env.METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS,
callback: async (_flushId, batch) => {
await clickhouseClient.metrics.insert(batch);
},
minConcurrency: 1,
maxConcurrency: env.METRICS_CLICKHOUSE_MAX_CONCURRENCY,
loadSheddingEnabled: false,
});
return new OTLPExporter(
eventRepository,
clickhouseEventRepository,
clickhouseEventRepositoryV2,
metricsFlushScheduler,
process.env.OTLP_EXPORTER_VERBOSE === "1",
process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT
async function initializeOTLPExporter() {
await clickhouseFactory.isReady();
return new OTLPExporter({
clickhouseFactory,
verbose: process.env.OTLP_EXPORTER_VERBOSE === "1",
spanAttributeValueLengthLimit: process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT
? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10)
: 8192
);
}
: 8192,
});
}
apps/webapp/app/v3/runEngineHandlers.server.ts
+282 -244
View File
@@ -17,17 +17,14 @@ import { QueueSizeLimitExceededError } from "~/v3/services/common.server";
import { TriggerTaskService } from "~/v3/services/triggerTask.server";
import { tracer } from "~/v3/tracer.server";
import { createExceptionPropertiesFromError } from "./eventRepository/common.server";
import {
recordRunDebugLog,
resolveEventRepositoryForStore,
} from "./eventRepository/index.server";
import { getEventRepositoryForStore, recordRunDebugLog } from "./eventRepository/index.server";
import { roomFromFriendlyRunId, socketIo } from "./handleSocketIo.server";
import { engine } from "./runEngine.server";
import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
import { TaskRunErrorCodes } from "@trigger.dev/core/v3";
export function registerRunEngineEventBusHandlers() {
engine.eventBus.on("runSucceeded", async ({ time, run }) => {
engine.eventBus.on("runSucceeded", async ({ time, run, organization }) => {
const [taskRunError, taskRun] = await tryCatch(
$replica.taskRun.findFirstOrThrow({
where: {
@@ -60,7 +57,10 @@ export function registerRunEngineEventBusHandlers() {
return;
}
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
taskRun.organizationId ?? organization.id
);
const [completeSuccessfulRunEventError] = await tryCatch(
eventRepository.completeSuccessfulRunEvent({
@@ -91,7 +91,7 @@ export function registerRunEngineEventBusHandlers() {
});
// Handle events
engine.eventBus.on("runFailed", async ({ time, run }) => {
engine.eventBus.on("runFailed", async ({ time, run, organization }) => {
const sanitizedError = sanitizeError(run.error);
const exception = createExceptionPropertiesFromError(sanitizedError);
@@ -127,7 +127,10 @@ export function registerRunEngineEventBusHandlers() {
return;
}
const eventRepository = resolveEventRepositoryForStore(taskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
taskRun.organizationId ?? organization.id
);
const [completeFailedRunEventError] = await tryCatch(
eventRepository.completeFailedRunEvent({
@@ -181,7 +184,17 @@ export function registerRunEngineEventBusHandlers() {
return;
}
const eventRepository = resolveEventRepositoryForStore(taskRun.taskEventStore);
if (!taskRun.organizationId) {
logger.error("[runAttemptFailed] Task run has no organization id", {
runId: run.id,
});
return;
}
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
taskRun.organizationId
);
const [createAttemptFailedRunEventError] = await tryCatch(
eventRepository.createAttemptFailedRunEvent({
@@ -282,7 +295,17 @@ export function registerRunEngineEventBusHandlers() {
return;
}
const eventRepository = resolveEventRepositoryForStore(blockedRun.taskEventStore);
if (!blockedRun.organizationId) {
logger.error("[cachedRunCompleted] Blocked run has no organization id", {
blockedRunId,
});
return;
}
const eventRepository = await getEventRepositoryForStore(
blockedRun.taskEventStore,
blockedRun.organizationId
);
const [completeCachedRunEventError] = await tryCatch(
eventRepository.completeCachedRunEvent({
@@ -305,7 +328,7 @@ export function registerRunEngineEventBusHandlers() {
}
);
engine.eventBus.on("runExpired", async ({ time, run }) => {
engine.eventBus.on("runExpired", async ({ time, run, organization }) => {
if (!run.ttl) {
return;
}
@@ -342,7 +365,10 @@ export function registerRunEngineEventBusHandlers() {
return;
}
const eventRepository = resolveEventRepositoryForStore(taskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
taskRun.taskEventStore,
taskRun.organizationId ?? organization.id
);
const [completeExpiredRunEventError] = await tryCatch(
eventRepository.completeExpiredRunEvent({
@@ -360,7 +386,7 @@ export function registerRunEngineEventBusHandlers() {
}
});
engine.eventBus.on("runCancelled", async ({ time, run }) => {
engine.eventBus.on("runCancelled", async ({ time, run, organization }) => {
const [taskRunError, taskRun] = await tryCatch(
$replica.taskRun.findFirstOrThrow({
where: {
@@ -393,7 +419,10 @@ export function registerRunEngineEventBusHandlers() {
return;
}
const eventRepository = resolveEventRepositoryForStore(taskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
taskRun.taskEventStore,
taskRun.organizationId ?? organization.id
);
const error = createJsonErrorObject(run.error);
@@ -413,46 +442,53 @@ export function registerRunEngineEventBusHandlers() {
}
});
engine.eventBus.on("runRetryScheduled", async ({ time, run, environment, retryAt }) => {
try {
if (retryAt && time && time >= retryAt) {
return;
}
engine.eventBus.on(
"runRetryScheduled",
async ({ time, run, environment, retryAt, organization }) => {
try {
if (retryAt && time && time >= retryAt) {
return;
}
let retryMessage = `Retry ${typeof run.attemptNumber === "number" ? `#${run.attemptNumber - 1}` : ""
let retryMessage = `Retry ${
typeof run.attemptNumber === "number" ? `#${run.attemptNumber - 1}` : ""
} delay`;
if (run.nextMachineAfterOOM) {
retryMessage += ` after OOM`;
if (run.nextMachineAfterOOM) {
retryMessage += ` after OOM`;
}
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore ?? "taskEvent",
organization.id
);
await eventRepository.recordEvent(retryMessage, {
startTime: BigInt(time.getTime() * 1000000),
taskSlug: run.taskIdentifier,
environment,
attributes: {
properties: {
retryAt: retryAt.toISOString(),
nextMachine: run.nextMachineAfterOOM,
},
runId: run.friendlyId,
style: {
icon: "schedule-attempt",
},
},
context: run.traceContext as Record<string, string | undefined>,
endTime: retryAt,
});
} catch (error) {
logger.error("[runRetryScheduled] Failed to record retry event", {
error: error instanceof Error ? error.message : error,
runId: run.id,
spanId: run.spanId,
});
}
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
await eventRepository.recordEvent(retryMessage, {
startTime: BigInt(time.getTime() * 1000000),
taskSlug: run.taskIdentifier,
environment,
attributes: {
properties: {
retryAt: retryAt.toISOString(),
nextMachine: run.nextMachineAfterOOM,
},
runId: run.friendlyId,
style: {
icon: "schedule-attempt",
},
},
context: run.traceContext as Record<string, string | undefined>,
endTime: retryAt,
});
} catch (error) {
logger.error("[runRetryScheduled] Failed to record retry event", {
error: error instanceof Error ? error.message : error,
runId: run.id,
spanId: run.spanId,
});
}
});
);
engine.eventBus.on("runAttemptStarted", async ({ time, run, organization }) => {
try {
@@ -485,10 +521,10 @@ export function registerRunEngineEventBusHandlers() {
error:
e instanceof Error
? {
name: e.name,
message: e.message,
stack: e.stack,
}
name: e.name,
message: e.message,
stack: e.stack,
}
: e,
});
} else {
@@ -497,10 +533,10 @@ export function registerRunEngineEventBusHandlers() {
error:
e instanceof Error
? {
name: e.name,
message: e.message,
stack: e.stack,
}
name: e.name,
message: e.message,
stack: e.stack,
}
: e,
});
}
@@ -658,121 +694,200 @@ const QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE = "QUEUE_SIZE_LIMIT_EXCEEDED";
*/
export function setupBatchQueueCallbacks() {
// Item processing callback - creates a run for each batch item
engine.setBatchProcessItemCallback(async ({ batchId, friendlyId, itemIndex, item, meta, attempt, isFinalAttempt }) => {
return tracer.startActiveSpan(
"batch.processItem",
{
kind: SpanKind.INTERNAL,
attributes: {
"batch.id": friendlyId,
"batch.item_index": itemIndex,
"batch.task": item.task,
"batch.environment_id": meta.environmentId,
"batch.parent_run_id": meta.parentRunId ?? "",
"batch.attempt": attempt,
"batch.is_final_attempt": isFinalAttempt,
engine.setBatchProcessItemCallback(
async ({ batchId, friendlyId, itemIndex, item, meta, attempt, isFinalAttempt }) => {
return tracer.startActiveSpan(
"batch.processItem",
{
kind: SpanKind.INTERNAL,
attributes: {
"batch.id": friendlyId,
"batch.item_index": itemIndex,
"batch.task": item.task,
"batch.environment_id": meta.environmentId,
"batch.parent_run_id": meta.parentRunId ?? "",
"batch.attempt": attempt,
"batch.is_final_attempt": isFinalAttempt,
},
},
},
async (span) => {
const triggerFailedTaskService = new TriggerFailedTaskService({
prisma,
engine,
replicaPrisma: $replica,
});
async (span) => {
const triggerFailedTaskService = new TriggerFailedTaskService({
prisma,
engine,
replicaPrisma: $replica,
});
// Check for pre-marked error items (e.g. oversized payloads)
const itemError = item.options?.__error as string | undefined;
if (itemError) {
const errorCode = (item.options?.__errorCode as string) ?? "ITEM_ERROR";
// Check for pre-marked error items (e.g. oversized payloads)
const itemError = item.options?.__error as string | undefined;
if (itemError) {
const errorCode = (item.options?.__errorCode as string) ?? "ITEM_ERROR";
let environment: AuthenticatedEnvironment | undefined;
try {
environment = (await findEnvironmentById(meta.environmentId)) ?? undefined;
} catch {
// Best-effort environment lookup
}
if (environment) {
const failedRunId = await triggerFailedTaskService.call({
taskId: item.task,
environment,
payload: item.payload ?? "{}",
payloadType: item.payloadType as string,
errorMessage: itemError,
errorCode: errorCode as TaskRunErrorCodes,
parentRunId: meta.parentRunId,
resumeParentOnCompletion: meta.resumeParentOnCompletion,
batch: { id: batchId, index: itemIndex },
traceContext: meta.traceContext as Record<string, unknown> | undefined,
spanParentAsLink: meta.spanParentAsLink,
});
if (failedRunId) {
span.setAttribute("batch.result.pre_failed", true);
span.setAttribute("batch.result.run_id", failedRunId);
span.end();
return { success: true as const, runId: failedRunId };
}
}
// Fallback if TriggerFailedTaskService or environment lookup fails
span.end();
return { success: false as const, error: itemError, errorCode };
}
let environment: AuthenticatedEnvironment | undefined;
try {
environment = (await findEnvironmentById(meta.environmentId)) ?? undefined;
} catch {
// Best-effort environment lookup
}
if (environment) {
const failedRunId = await triggerFailedTaskService.call({
taskId: item.task,
environment,
payload: item.payload ?? "{}",
payloadType: item.payloadType as string,
errorMessage: itemError,
errorCode: errorCode as TaskRunErrorCodes,
parentRunId: meta.parentRunId,
resumeParentOnCompletion: meta.resumeParentOnCompletion,
batch: { id: batchId, index: itemIndex },
traceContext: meta.traceContext as Record<string, unknown> | undefined,
spanParentAsLink: meta.spanParentAsLink,
});
if (failedRunId) {
span.setAttribute("batch.result.pre_failed", true);
span.setAttribute("batch.result.run_id", failedRunId);
if (!environment) {
span.setAttribute("batch.result.error", "Environment not found");
span.end();
return { success: true as const, runId: failedRunId };
return {
success: false as const,
error: "Environment not found",
errorCode: "ENVIRONMENT_NOT_FOUND",
};
}
}
// Fallback if TriggerFailedTaskService or environment lookup fails
span.end();
return { success: false as const, error: itemError, errorCode };
}
const triggerTaskService = new TriggerTaskService();
let environment: AuthenticatedEnvironment | undefined;
try {
environment = (await findEnvironmentById(meta.environmentId)) ?? undefined;
// Normalize payload - for application/store (R2 paths), this passes through as-is
const payload = normalizePayload(item.payload, item.payloadType);
if (!environment) {
span.setAttribute("batch.result.error", "Environment not found");
span.end();
return {
success: false as const,
error: "Environment not found",
errorCode: "ENVIRONMENT_NOT_FOUND",
};
}
const triggerTaskService = new TriggerTaskService();
// Normalize payload - for application/store (R2 paths), this passes through as-is
const payload = normalizePayload(item.payload, item.payloadType);
const result = await triggerTaskService.call(
item.task,
environment,
{
payload,
options: {
...(item.options as Record<string, unknown>),
payloadType: item.payloadType,
parentRunId: meta.parentRunId,
resumeParentOnCompletion: meta.resumeParentOnCompletion,
parentBatch: batchId,
const result = await triggerTaskService.call(
item.task,
environment,
{
payload,
options: {
...(item.options as Record<string, unknown>),
payloadType: item.payloadType,
parentRunId: meta.parentRunId,
resumeParentOnCompletion: meta.resumeParentOnCompletion,
parentBatch: batchId,
},
},
},
{
triggerVersion: meta.triggerVersion,
traceContext: meta.traceContext as Record<string, unknown> | undefined,
spanParentAsLink: meta.spanParentAsLink,
batchId,
batchIndex: itemIndex,
realtimeStreamsVersion: meta.realtimeStreamsVersion,
planType: meta.planType,
triggerSource: meta.parentRunId ? "sdk" : meta.triggerSource ?? "api",
triggerAction: "trigger",
},
"V2"
);
{
triggerVersion: meta.triggerVersion,
traceContext: meta.traceContext as Record<string, unknown> | undefined,
spanParentAsLink: meta.spanParentAsLink,
batchId,
batchIndex: itemIndex,
realtimeStreamsVersion: meta.realtimeStreamsVersion,
planType: meta.planType,
triggerSource: meta.parentRunId ? "sdk" : meta.triggerSource ?? "api",
triggerAction: "trigger",
},
"V2"
);
if (result) {
span.setAttribute("batch.result.run_id", result.run.friendlyId);
span.end();
return { success: true as const, runId: result.run.friendlyId };
} else {
logger.error("[BatchQueue] TriggerTaskService returned undefined", {
if (result) {
span.setAttribute("batch.result.run_id", result.run.friendlyId);
span.end();
return { success: true as const, runId: result.run.friendlyId };
} else {
logger.error("[BatchQueue] TriggerTaskService returned undefined", {
batchId,
friendlyId,
itemIndex,
task: item.task,
environmentId: meta.environmentId,
attempt,
isFinalAttempt,
});
span.setAttribute("batch.result.error", "TriggerTaskService returned undefined");
// Only create a pre-failed run on the final attempt; otherwise let the retry mechanism handle it
if (isFinalAttempt) {
const failedRunId = await triggerFailedTaskService.call({
taskId: item.task,
environment,
payload: item.payload,
payloadType: item.payloadType as string,
errorMessage: "TriggerTaskService returned undefined",
parentRunId: meta.parentRunId,
resumeParentOnCompletion: meta.resumeParentOnCompletion,
batch: { id: batchId, index: itemIndex },
options: item.options as Record<string, unknown>,
traceContext: meta.traceContext as Record<string, unknown> | undefined,
spanParentAsLink: meta.spanParentAsLink,
errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER,
});
span.end();
if (failedRunId) {
return { success: true as const, runId: failedRunId };
}
} else {
span.end();
}
return {
success: false as const,
error: "TriggerTaskService returned undefined",
errorCode: "TRIGGER_FAILED",
};
}
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
// Queue-size-limit rejections are a customer-overload scenario (the
// env's queue is at its configured max). Retrying is pointless — the
// same item will fail again — and creating pre-failed TaskRuns for
// every item of every retried batch is exactly what chews through
// DB capacity when a noisy tenant fills their queue. Signal the
// BatchQueue to skip retries and skip pre-failed run creation, and
// let the completion callback collapse the per-item errors into a
// single summary row.
if (error instanceof QueueSizeLimitExceededError) {
logger.warn("[BatchQueue] Batch item rejected: queue size limit reached", {
batchId,
friendlyId,
itemIndex,
task: item.task,
environmentId: meta.environmentId,
maximumSize: error.maximumSize,
});
span.setAttribute("batch.result.error", errorMessage);
span.setAttribute("batch.result.errorCode", QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE);
span.setAttribute("batch.result.skipRetries", true);
span.end();
return {
success: false as const,
error: errorMessage,
errorCode: QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE,
skipRetries: true,
};
}
logger.error("[BatchQueue] Failed to trigger batch item", {
batchId,
friendlyId,
itemIndex,
@@ -780,18 +895,20 @@ export function setupBatchQueueCallbacks() {
environmentId: meta.environmentId,
attempt,
isFinalAttempt,
error,
});
span.setAttribute("batch.result.error", "TriggerTaskService returned undefined");
span.setAttribute("batch.result.error", errorMessage);
span.recordException(error instanceof Error ? error : new Error(String(error)));
// Only create a pre-failed run on the final attempt; otherwise let the retry mechanism handle it
if (isFinalAttempt) {
if (isFinalAttempt && environment) {
const failedRunId = await triggerFailedTaskService.call({
taskId: item.task,
environment,
payload: item.payload,
payloadType: item.payloadType as string,
errorMessage: "TriggerTaskService returned undefined",
errorMessage,
parentRunId: meta.parentRunId,
resumeParentOnCompletion: meta.resumeParentOnCompletion,
batch: { id: batchId, index: itemIndex },
@@ -810,95 +927,16 @@ export function setupBatchQueueCallbacks() {
span.end();
}
return {
success: false as const,
error: "TriggerTaskService returned undefined",
errorCode: "TRIGGER_FAILED",
};
}
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
// Queue-size-limit rejections are a customer-overload scenario (the
// env's queue is at its configured max). Retrying is pointless — the
// same item will fail again — and creating pre-failed TaskRuns for
// every item of every retried batch is exactly what chews through
// DB capacity when a noisy tenant fills their queue. Signal the
// BatchQueue to skip retries and skip pre-failed run creation, and
// let the completion callback collapse the per-item errors into a
// single summary row.
if (error instanceof QueueSizeLimitExceededError) {
logger.warn("[BatchQueue] Batch item rejected: queue size limit reached", {
batchId,
friendlyId,
itemIndex,
task: item.task,
environmentId: meta.environmentId,
maximumSize: error.maximumSize,
});
span.setAttribute("batch.result.error", errorMessage);
span.setAttribute("batch.result.errorCode", QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE);
span.setAttribute("batch.result.skipRetries", true);
span.end();
return {
success: false as const,
error: errorMessage,
errorCode: QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE,
skipRetries: true,
errorCode: "TRIGGER_ERROR",
};
}
logger.error("[BatchQueue] Failed to trigger batch item", {
batchId,
friendlyId,
itemIndex,
task: item.task,
environmentId: meta.environmentId,
attempt,
isFinalAttempt,
error,
});
span.setAttribute("batch.result.error", errorMessage);
span.recordException(error instanceof Error ? error : new Error(String(error)));
// Only create a pre-failed run on the final attempt; otherwise let the retry mechanism handle it
if (isFinalAttempt && environment) {
const failedRunId = await triggerFailedTaskService.call({
taskId: item.task,
environment,
payload: item.payload,
payloadType: item.payloadType as string,
errorMessage,
parentRunId: meta.parentRunId,
resumeParentOnCompletion: meta.resumeParentOnCompletion,
batch: { id: batchId, index: itemIndex },
options: item.options as Record<string, unknown>,
traceContext: meta.traceContext as Record<string, unknown> | undefined,
spanParentAsLink: meta.spanParentAsLink,
errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER,
});
span.end();
if (failedRunId) {
return { success: true as const, runId: failedRunId };
}
} else {
span.end();
}
return {
success: false as const,
error: errorMessage,
errorCode: "TRIGGER_ERROR",
};
}
}
);
});
);
}
);
// Batch completion callback - updates Postgres with results
engine.setBatchCompletionCallback(async (result: CompleteBatchResult) => {
apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts
+13 -9
View File
@@ -7,7 +7,7 @@ import {
} from "@trigger.dev/database";
import { $replica, prisma } from "~/db.server";
import { ErrorAlertConfig } from "~/models/projectAlert.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import { logger } from "~/services/logger.server";
import { alertsWorker } from "~/v3/alertsWorker.server";
@@ -45,8 +45,7 @@ const DEFAULT_INTERVAL_MS = 300_000;
export class ErrorAlertEvaluator {
constructor(
protected readonly _prisma: PrismaClientOrTransaction = prisma,
protected readonly _replica: PrismaClientOrTransaction = $replica,
protected readonly _clickhouse: ClickHouse = clickhouseClient
protected readonly _replica: PrismaClientOrTransaction = $replica
) {}
async evaluate(projectId: string, scheduledAt: number): Promise<void> {
@@ -245,10 +244,7 @@ export class ErrorAlertEvaluator {
}
}
if (
state.ignoredUntilTotalOccurrences != null &&
state.ignoredAtOccurrenceCount != null
) {
if (state.ignoredUntilTotalOccurrences != null && state.ignoredAtOccurrenceCount != null) {
const occurrencesSinceIgnored =
context.totalOccurrenceCount - Number(state.ignoredAtOccurrenceCount);
if (occurrencesSinceIgnored >= state.ignoredUntilTotalOccurrences) {
@@ -335,7 +331,11 @@ export class ErrorAlertEvaluator {
envIds: string[],
scheduledAt: number
): Promise<ActiveErrorsSinceQueryResult[]> {
const qb = this._clickhouse.errors.activeErrorsSinceQueryBuilder();
const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
organizationId,
"query"
);
const qb = queryClickhouse.errors.activeErrorsSinceQueryBuilder();
qb.where("organization_id = {organizationId: String}", { organizationId });
qb.where("project_id = {projectId: String}", { projectId });
qb.where("environment_id IN {envIds: Array(String)}", { envIds });
@@ -389,7 +389,11 @@ export class ErrorAlertEvaluator {
occurrences_since: number;
}>
> {
const qb = this._clickhouse.errors.occurrenceCountsSinceQueryBuilder();
const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization(
organizationId,
"query"
);
const qb = queryClickhouse.errors.occurrenceCountsSinceQueryBuilder();
qb.where("organization_id = {organizationId: String}", { organizationId });
qb.where("project_id = {projectId: String}", { projectId });
qb.where("environment_id IN {envIds: Array(String)}", { envIds });
apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts
+5 -3
View File
@@ -7,7 +7,7 @@ import {
} from "@trigger.dev/database";
import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server";
import { type CreateBulkActionPayload } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
import {
parseRunListInputOptions,
type RunListInputFilters,
@@ -38,8 +38,9 @@ export class BulkActionService extends BaseService {
const filters = await getFilters(payload, request);
// Count the runs that will be affected by the bulk action
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard");
const runsRepository = new RunsRepository({
clickhouse: clickhouseClient,
clickhouse,
prisma: this._replica as PrismaClient,
});
const count = await runsRepository.countRuns({
@@ -147,8 +148,9 @@ export class BulkActionService extends BaseService {
...rawParams,
});
const clickhouse = await clickhouseFactory.getClickhouseForOrganization(group.project.organizationId, "standard");
const runsRepository = new RunsRepository({
clickhouse: clickhouseClient,
clickhouse,
prisma: this._replica as PrismaClient,
});
apps/webapp/app/v3/services/cancelTaskRunV1.server.ts
+5 -2
View File
@@ -10,7 +10,7 @@ import { CancelTaskAttemptDependenciesService } from "./cancelTaskAttemptDepende
import { CancelableTaskRun } from "./cancelTaskRun.server";
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
import { tryCatch } from "@trigger.dev/core/utils";
import { resolveEventRepositoryForStore } from "../eventRepository/index.server";
import { getEventRepositoryForStore } from "../eventRepository/index.server";
type ExtendedTaskRun = Prisma.TaskRunGetPayload<{
include: {
@@ -101,7 +101,10 @@ export class CancelTaskRunServiceV1 extends BaseService {
},
});
const eventRepository = resolveEventRepositoryForStore(cancelledTaskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
cancelledTaskRun.taskEventStore,
cancelledTaskRun.runtimeEnvironment.organizationId
);
const [cancelRunEventError] = await tryCatch(
eventRepository.cancelRunEvent({
apps/webapp/app/v3/services/completeAttempt.server.ts
+13 -4
View File
@@ -31,7 +31,7 @@ import { CancelAttemptService } from "./cancelAttempt.server";
import { CreateCheckpointService } from "./createCheckpoint.server";
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
import { RetryAttemptService } from "./retryAttempt.server";
import { resolveEventRepositoryForStore } from "../eventRepository/index.server";
import { getEventRepositoryForStore } from "../eventRepository/index.server";
type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
@@ -163,7 +163,10 @@ export class CompleteAttemptService extends BaseService {
env,
});
const eventRepository = resolveEventRepositoryForStore(taskRunAttempt.taskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
taskRunAttempt.taskRun.taskEventStore,
taskRunAttempt.taskRun.organizationId ?? ""
);
const [completeSuccessfulRunEventError] = await tryCatch(
eventRepository.completeSuccessfulRunEvent({
@@ -316,7 +319,10 @@ export class CompleteAttemptService extends BaseService {
exitRun(taskRunAttempt.taskRunId);
}
const eventRepository = resolveEventRepositoryForStore(taskRunAttempt.taskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
taskRunAttempt.taskRun.taskEventStore,
taskRunAttempt.taskRun.organizationId ?? ""
);
const [completeFailedRunEventError] = await tryCatch(
eventRepository.completeFailedRunEvent({
@@ -538,7 +544,10 @@ export class CompleteAttemptService extends BaseService {
}) {
const retryAt = new Date(executionRetry.timestamp);
const eventRepository = resolveEventRepositoryForStore(taskRunAttempt.taskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
taskRunAttempt.taskRun.taskEventStore,
taskRunAttempt.taskRun.organizationId ?? ""
);
// Retry the task run
await eventRepository.recordEvent(
apps/webapp/app/v3/services/crashTaskRun.server.ts
+5 -2
View File
@@ -7,7 +7,7 @@ import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
import { BaseService } from "./baseService.server";
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
import { resolveEventRepositoryForStore } from "../eventRepository/index.server";
import { getEventRepositoryForStore } from "../eventRepository/index.server";
export type CrashTaskRunServiceOptions = {
reason?: string;
@@ -120,7 +120,10 @@ export class CrashTaskRunService extends BaseService {
},
});
const eventRepository = resolveEventRepositoryForStore(crashedTaskRun.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
crashedTaskRun.taskEventStore,
crashedTaskRun.runtimeEnvironment.organizationId
);
const [createAttemptFailedEventError] = await tryCatch(
eventRepository.completeFailedRunEvent({
apps/webapp/app/v3/services/expireEnqueuedRun.server.ts
+5 -2
View File
@@ -4,7 +4,7 @@ import { commonWorker } from "../commonWorker.server";
import { BaseService } from "./baseService.server";
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
import { tryCatch } from "@trigger.dev/core/utils";
import { resolveEventRepositoryForStore } from "../eventRepository/index.server";
import { getEventRepositoryForStore } from "../eventRepository/index.server";
export class ExpireEnqueuedRunService extends BaseService {
public static async ack(runId: string, tx?: PrismaClientOrTransaction) {
@@ -78,7 +78,10 @@ export class ExpireEnqueuedRunService extends BaseService {
},
});
const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
const eventRepository = await getEventRepositoryForStore(
run.taskEventStore,
run.runtimeEnvironment.organization.id
);
if (run.ttl) {
const [completeExpiredRunEventError] = await tryCatch(
apps/webapp/app/v3/services/triggerTaskV1.server.ts
+1
View File
@@ -294,6 +294,7 @@ export class TriggerTaskServiceV1 extends BaseService {
: undefined;
const { repository, store } = await getV3EventRepository(
environment.organization.id,
dependentAttempt?.taskRun.taskEventStore ??
parentAttempt?.taskRun.taskEventStore ??
dependentBatchRun?.dependentTaskAttempt?.taskRun.taskEventStore
apps/webapp/test/clickhouseFactory.test.ts
+130
View File
@@ -0,0 +1,130 @@
import { describe, expect } from "vitest";
vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} }));
import { postgresTest } from "@internal/testcontainers";
import { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
import { ClickhouseConnectionSchema } from "~/services/clickhouse/clickhouseSecretSchemas.server";
import { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server";
vi.setConfig({ testTimeout: 60_000 });
const TEST_URL = "https://default:password@ch-org.example.com:8443";
const TEST_URL_2 = "https://default:password@ch-other.example.com:8443";
describe("ClickHouse Factory", () => {
postgresTest(
"returns default client when org has no data store",
async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.loadFromDatabase();
const factory = new ClickhouseFactory(registry);
const client = await factory.getClickhouseForOrganization("org-no-store", "standard");
expect(client).toBeDefined();
}
);
postgresTest(
"returns org-specific client when a data store is configured",
async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "factory-store",
kind: "CLICKHOUSE",
organizationIds: ["org-custom"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.loadFromDatabase();
const factory = new ClickhouseFactory(registry);
const client = await factory.getClickhouseForOrganization("org-custom", "standard");
expect(client).toBeDefined();
// Default client is a different instance from the org-specific one
const defaultClient = await factory.getClickhouseForOrganization("org-no-store", "standard");
expect(client).not.toBe(defaultClient);
}
);
postgresTest(
"two orgs sharing the same data store get the same cached client",
async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "shared-factory-store",
kind: "CLICKHOUSE",
organizationIds: ["org-shared-1", "org-shared-2"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.loadFromDatabase();
const factory = new ClickhouseFactory(registry);
const client1 = await factory.getClickhouseForOrganization("org-shared-1", "standard");
const client2 = await factory.getClickhouseForOrganization("org-shared-2", "standard");
// Same hostname → same cached client instance
expect(client1).toBe(client2);
}
);
postgresTest(
"two data stores with different URLs produce different clients",
async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "store-a",
kind: "CLICKHOUSE",
organizationIds: ["org-a"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.addDataStore({
key: "store-b",
kind: "CLICKHOUSE",
organizationIds: ["org-b"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL_2 }),
});
await registry.loadFromDatabase();
const factory = new ClickhouseFactory(registry);
const clientA = await factory.getClickhouseForOrganization("org-a", "standard");
const clientB = await factory.getClickhouseForOrganization("org-b", "standard");
expect(clientA).not.toBe(clientB);
}
);
postgresTest(
"after reload with a deleted store, org falls back to default",
async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "removable-store",
kind: "CLICKHOUSE",
organizationIds: ["org-removable"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.loadFromDatabase();
const factory = new ClickhouseFactory(registry);
const before = await factory.getClickhouseForOrganization("org-removable", "standard");
const defaultClient = await factory.getClickhouseForOrganization("org-no-store", "standard");
expect(before).not.toBe(defaultClient);
await registry.deleteDataStore({ key: "removable-store", kind: "CLICKHOUSE" });
await registry.reload();
const after = await factory.getClickhouseForOrganization("org-removable", "standard");
expect(after).toBe(defaultClient);
}
);
});
apps/webapp/test/organizationDataStoresRegistry.test.ts
+230
View File
@@ -0,0 +1,230 @@
import { describe, expect } from "vitest";
vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} }));
import { postgresTest } from "@internal/testcontainers";
import { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
import { ClickhouseConnectionSchema } from "~/services/clickhouse/clickhouseSecretSchemas.server";
vi.setConfig({ testTimeout: 60_000 });
const TEST_URL = "https://default:password@clickhouse.example.com:8443";
const TEST_URL_2 = "https://default:password@clickhouse2.example.com:8443";
describe("OrganizationDataStoresRegistry", () => {
postgresTest("isLoaded is false before loadFromDatabase", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
expect(registry.isLoaded).toBe(false);
expect(registry.get("any-org", "CLICKHOUSE")).toBeNull();
});
postgresTest("isLoaded is true after loadFromDatabase", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.loadFromDatabase();
expect(registry.isLoaded).toBe(true);
});
postgresTest("isReady resolves after loadFromDatabase", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
let resolved = false;
registry.isReady.then(() => {
resolved = true;
});
await registry.loadFromDatabase();
await registry.isReady;
expect(resolved).toBe(true);
});
postgresTest("get returns null when no data stores exist", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.loadFromDatabase();
expect(registry.get("org-1", "CLICKHOUSE")).toBeNull();
});
postgresTest("addDataStore creates a row and stores the secret", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "test-store",
kind: "CLICKHOUSE",
organizationIds: ["org-1", "org-2"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
const row = await prisma.organizationDataStore.findFirst({ where: { key: "test-store" } });
expect(row).not.toBeNull();
expect(row?.organizationIds).toEqual(["org-1", "org-2"]);
expect(row?.kind).toBe("CLICKHOUSE");
const secret = await prisma.secretStore.findFirst({
where: { key: "data-store:test-store:clickhouse" },
});
expect(secret).not.toBeNull();
});
postgresTest("loadFromDatabase resolves secrets and makes orgs available via get", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "hipaa-store",
kind: "CLICKHOUSE",
organizationIds: ["org-hipaa"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.loadFromDatabase();
const result = registry.get("org-hipaa", "CLICKHOUSE");
expect(result).not.toBeNull();
expect(result?.kind).toBe("CLICKHOUSE");
expect(result?.url).toBe(TEST_URL);
});
postgresTest("get returns null for orgs not in any data store", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "partial-store",
kind: "CLICKHOUSE",
organizationIds: ["org-a"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.loadFromDatabase();
expect(registry.get("org-a", "CLICKHOUSE")).not.toBeNull();
expect(registry.get("org-b", "CLICKHOUSE")).toBeNull();
});
postgresTest("multiple orgs can share the same data store", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "shared-store",
kind: "CLICKHOUSE",
organizationIds: ["org-x", "org-y", "org-z"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.loadFromDatabase();
const x = registry.get("org-x", "CLICKHOUSE");
const y = registry.get("org-y", "CLICKHOUSE");
const z = registry.get("org-z", "CLICKHOUSE");
expect(x?.url).toBe(TEST_URL);
expect(y?.url).toBe(TEST_URL);
expect(z?.url).toBe(TEST_URL);
});
postgresTest(
"when an org appears in multiple data stores, first row by id asc wins",
async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
const sharedOrg = "org-dup-overlap";
await registry.addDataStore({
key: "dup-overlap-first",
kind: "CLICKHOUSE",
organizationIds: [sharedOrg],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.addDataStore({
key: "dup-overlap-second",
kind: "CLICKHOUSE",
organizationIds: [sharedOrg],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL_2 }),
});
const [winner] = await prisma.organizationDataStore.findMany({
where: { key: { in: ["dup-overlap-first", "dup-overlap-second"] } },
orderBy: { id: "asc" },
});
expect(winner).toBeDefined();
await registry.loadFromDatabase();
const expectedUrl =
winner!.key === "dup-overlap-first" ? TEST_URL : TEST_URL_2;
expect(registry.get(sharedOrg, "CLICKHOUSE")?.url).toBe(expectedUrl);
}
);
postgresTest("updateDataStore updates organizationIds and rotates the secret", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "update-store",
kind: "CLICKHOUSE",
organizationIds: ["org-old"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.updateDataStore({
key: "update-store",
kind: "CLICKHOUSE",
organizationIds: ["org-new-1", "org-new-2"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL_2 }),
});
const row = await prisma.organizationDataStore.findFirst({ where: { key: "update-store" } });
expect(row?.organizationIds).toEqual(["org-new-1", "org-new-2"]);
await registry.loadFromDatabase();
expect(registry.get("org-new-1", "CLICKHOUSE")?.url).toBe(TEST_URL_2);
expect(registry.get("org-old", "CLICKHOUSE")).toBeNull();
});
postgresTest("reload picks up changes made after initial load", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.loadFromDatabase();
expect(registry.get("org-reload", "CLICKHOUSE")).toBeNull();
await registry.addDataStore({
key: "reload-store",
kind: "CLICKHOUSE",
organizationIds: ["org-reload"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
expect(registry.get("org-reload", "CLICKHOUSE")).toBeNull();
await registry.reload();
expect(registry.get("org-reload", "CLICKHOUSE")?.url).toBe(TEST_URL);
});
postgresTest("deleteDataStore removes the row and its secret", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "delete-store",
kind: "CLICKHOUSE",
organizationIds: ["org-delete"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.deleteDataStore({ key: "delete-store", kind: "CLICKHOUSE" });
expect(await prisma.organizationDataStore.findFirst({ where: { key: "delete-store" } })).toBeNull();
expect(await prisma.secretStore.findFirst({ where: { key: "data-store:delete-store:clickhouse" } })).toBeNull();
});
postgresTest("after delete and reload, org no longer has a data store", async ({ prisma }) => {
const registry = new OrganizationDataStoresRegistry(prisma);
await registry.addDataStore({
key: "ephemeral-store",
kind: "CLICKHOUSE",
organizationIds: ["org-ephemeral"],
config: ClickhouseConnectionSchema.parse({ url: TEST_URL }),
});
await registry.loadFromDatabase();
expect(registry.get("org-ephemeral", "CLICKHOUSE")?.url).toBe(TEST_URL);
await registry.deleteDataStore({ key: "ephemeral-store", kind: "CLICKHOUSE" });
await registry.reload();
expect(registry.get("org-ephemeral", "CLICKHOUSE")).toBeNull();
});
});
apps/webapp/test/runsBackfiller.test.ts
+2 -1
View File
@@ -12,6 +12,7 @@ import { z } from "zod";
import { RunsBackfillerService } from "~/services/runsBackfiller.server";
import { RunsReplicationService } from "~/services/runsReplicationService.server";
import { createInMemoryTracing } from "./utils/tracing";
import { TestReplicationClickhouseFactory } from "./utils/testReplicationClickhouseFactory";
vi.setConfig({ testTimeout: 60_000 });
@@ -30,7 +31,7 @@ describe("RunsBackfillerService", () => {
const { tracer, exporter } = createInMemoryTracing();
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication",
slotName: "task_runs_to_clickhouse_v1",
apps/webapp/test/runsReplicationBenchmark.test.ts
+2 -1
View File
@@ -7,6 +7,7 @@ import path from "node:path";
import { z } from "zod";
import { RunsReplicationService } from "~/services/runsReplicationService.server";
import { createInMemoryTracing, createInMemoryMetrics } from "./utils/tracing";
import { TestReplicationClickhouseFactory } from "./utils/testReplicationClickhouseFactory";
// Extend test timeout for benchmarks
vi.setConfig({ testTimeout: 300_000 }); // 5 minutes
@@ -320,7 +321,7 @@ async function runBenchmark(
// Create and start replication service
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: `benchmark-${name}`,
slotName: `benchmark_${name.replace(/-/g, "_")}`,
apps/webapp/test/runsReplicationService.part1.test.ts
+14 -13
View File
@@ -5,6 +5,7 @@ import { z } from "zod";
import { TaskRunStatus } from "~/database-types";
import { RunsReplicationService } from "~/services/runsReplicationService.server";
import { createInMemoryTracing, createInMemoryMetrics } from "./utils/tracing";
import { TestReplicationClickhouseFactory } from "./utils/testReplicationClickhouseFactory";
import superjson from "superjson";
vi.setConfig({ testTimeout: 60_000 });
@@ -27,7 +28,7 @@ describe("RunsReplicationService (part 1/2)", () => {
const { tracer, exporter } = createInMemoryTracing();
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication",
slotName: "task_runs_to_clickhouse_v1",
@@ -151,7 +152,7 @@ describe("RunsReplicationService (part 1/2)", () => {
const { tracer, exporter } = createInMemoryTracing();
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication",
slotName: "task_runs_to_clickhouse_v1",
@@ -289,7 +290,7 @@ describe("RunsReplicationService (part 1/2)", () => {
const { tracer, exporter } = createInMemoryTracing();
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication",
slotName: "task_runs_to_clickhouse_v1",
@@ -359,7 +360,7 @@ describe("RunsReplicationService (part 1/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-batching",
slotName: "task_runs_to_clickhouse_v1",
@@ -463,7 +464,7 @@ describe("RunsReplicationService (part 1/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-payload",
slotName: "task_runs_to_clickhouse_v1",
@@ -564,7 +565,7 @@ describe("RunsReplicationService (part 1/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-payload",
slotName: "task_runs_to_clickhouse_v1",
@@ -670,7 +671,7 @@ describe("RunsReplicationService (part 1/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-update",
slotName: "task_runs_to_clickhouse_v1",
@@ -777,7 +778,7 @@ describe("RunsReplicationService (part 1/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-delete",
slotName: "task_runs_to_clickhouse_v1",
@@ -878,7 +879,7 @@ describe("RunsReplicationService (part 1/2)", () => {
// Service A
const runsReplicationServiceA = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-shutdown-handover",
slotName: "task_runs_to_clickhouse_v1",
@@ -982,7 +983,7 @@ describe("RunsReplicationService (part 1/2)", () => {
// Service B
const runsReplicationServiceB = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-shutdown-handover",
slotName: "task_runs_to_clickhouse_v1",
@@ -1029,7 +1030,7 @@ describe("RunsReplicationService (part 1/2)", () => {
// Service A
const runsReplicationServiceA = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-shutdown-after-processed",
slotName: "task_runs_to_clickhouse_v1",
@@ -1131,7 +1132,7 @@ describe("RunsReplicationService (part 1/2)", () => {
// Service B
const runsReplicationServiceB = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-shutdown-after-processed",
slotName: "task_runs_to_clickhouse_v1",
@@ -1174,7 +1175,7 @@ describe("RunsReplicationService (part 1/2)", () => {
const metricsHelper = createInMemoryMetrics();
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-metrics",
slotName: "task_runs_to_clickhouse_v1",
apps/webapp/test/runsReplicationService.part2.test.ts
+11 -10
View File
@@ -6,6 +6,7 @@ import { setTimeout } from "node:timers/promises";
import { z } from "zod";
import { RunsReplicationService } from "~/services/runsReplicationService.server";
import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings";
import { TestReplicationClickhouseFactory } from "./utils/testReplicationClickhouseFactory";
vi.setConfig({ testTimeout: 60_000 });
@@ -23,7 +24,7 @@ describe("RunsReplicationService (part 2/2)", () => {
// Service A
const runsReplicationServiceA = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-shutdown-handover",
slotName: "task_runs_to_clickhouse_v1",
@@ -43,7 +44,7 @@ describe("RunsReplicationService (part 2/2)", () => {
// Service A
const runsReplicationServiceB = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-shutdown-handover",
slotName: "task_runs_to_clickhouse_v1",
@@ -152,7 +153,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-stress-bulk-insert",
slotName: "task_runs_to_clickhouse_v1",
@@ -267,7 +268,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-stress-bulk-insert",
slotName: "task_runs_to_clickhouse_v1",
@@ -388,7 +389,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-multi-event-tx",
slotName: "task_runs_to_clickhouse_v1",
@@ -522,7 +523,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-long-tx",
slotName: "task_runs_to_clickhouse_v1",
@@ -629,7 +630,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-stress-bulk-insert",
slotName: "task_runs_to_clickhouse_v1",
@@ -798,7 +799,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-merge-batch",
slotName: "task_runs_to_clickhouse_v1",
@@ -923,7 +924,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-sorting",
slotName: "task_runs_to_clickhouse_v1",
@@ -1136,7 +1137,7 @@ describe("RunsReplicationService (part 2/2)", () => {
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "runs-replication-exhaustive",
slotName: "task_runs_to_clickhouse_v1",
apps/webapp/test/sessionsReplicationService.test.ts
+3 -2
View File
@@ -3,6 +3,7 @@ import { containerTest } from "@internal/testcontainers";
import { setTimeout } from "node:timers/promises";
import { z } from "zod";
import { SessionsReplicationService } from "~/services/sessionsReplicationService.server";
import { TestReplicationClickhouseFactory } from "./utils/testReplicationClickhouseFactory";
vi.setConfig({ testTimeout: 60_000 });
@@ -21,7 +22,7 @@ describe("SessionsReplicationService", () => {
});
const service = new SessionsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "sessions-replication",
slotName: "sessions_to_clickhouse_v1",
@@ -128,7 +129,7 @@ describe("SessionsReplicationService", () => {
});
const service = new SessionsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: postgresContainer.getConnectionUri(),
serviceName: "sessions-replication",
slotName: "sessions_to_clickhouse_v1",
apps/webapp/test/utils/replicationUtils.ts
+2 -1
View File
@@ -2,6 +2,7 @@ import { ClickHouse } from "@internal/clickhouse";
import { RedisOptions } from "@internal/redis";
import { PrismaClient } from "~/db.server";
import { RunsReplicationService } from "~/services/runsReplicationService.server";
import { TestReplicationClickhouseFactory } from "./testReplicationClickhouseFactory";
import { afterEach } from "vitest";
export async function setupClickhouseReplication({
@@ -26,7 +27,7 @@ export async function setupClickhouseReplication({
});
const runsReplicationService = new RunsReplicationService({
clickhouse,
clickhouseFactory: new TestReplicationClickhouseFactory(clickhouse),
pgConnectionUrl: databaseUrl,
serviceName: "runs-replication",
slotName: "task_runs_to_clickhouse_v1",
apps/webapp/test/utils/testReplicationClickhouseFactory.ts
+32
View File
@@ -0,0 +1,32 @@
import type { ClickHouse } from "@internal/clickhouse";
import {
ClickhouseFactory,
type ClientType,
} from "~/services/clickhouse/clickhouseFactory.server";
import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
const testReplicationRegistryStub = {
isLoaded: true,
isReady: Promise.resolve(),
get: () => null,
} as unknown as OrganizationDataStoresRegistry;
/**
* Routes all `replication` and `sessions_replication` clients to a single test ClickHouse;
* other client types use the real factory defaults.
*/
export class TestReplicationClickhouseFactory extends ClickhouseFactory {
constructor(private readonly replicationClient: ClickHouse) {
super(testReplicationRegistryStub);
}
override getClickhouseForOrganizationSync(
organizationId: string,
clientType: ClientType
): ClickHouse {
if (clientType === "replication" || clientType === "sessions_replication") {
return this.replicationClient;
}
return super.getClickhouseForOrganizationSync(organizationId, clientType);
}
}
apps/webapp/test/utils/tracing.ts
+2
View File
@@ -39,6 +39,8 @@ export function createInMemoryTracing() {
});
provider.register();
// Use the provider's tracer so spans hit this exporter even when a global
// NodeTracerProvider was already registered (e.g. via tracer.server import chain).
const tracer = provider.getTracer("test-tracer");
return {
internal-packages/database/prisma/migrations/20260331212308_add_organization_data_stores/migration.sql
+21
View File
@@ -0,0 +1,21 @@
-- CreateEnum
CREATE TYPE "public"."DataStoreKind" AS ENUM ('CLICKHOUSE');
-- CreateTable
CREATE TABLE "public"."OrganizationDataStore" (
"id" TEXT NOT NULL,
"key" TEXT NOT NULL,
"organizationIds" TEXT[],
"kind" "public"."DataStoreKind" NOT NULL,
"config" JSONB NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "OrganizationDataStore_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "OrganizationDataStore_key_key" ON "public"."OrganizationDataStore"("key");
-- CreateIndex
CREATE INDEX "OrganizationDataStore_kind_idx" ON "public"."OrganizationDataStore"("kind");
internal-packages/database/prisma/schema.prisma
+22 -1
View File
@@ -3153,4 +3153,25 @@ model TaskIdentifier {
@@unique([runtimeEnvironmentId, slug])
@@index([runtimeEnvironmentId, isInLatestDeployment])
}
}
enum DataStoreKind {
CLICKHOUSE
}
/// Defines org-scoped data store overrides (e.g. dedicated ClickHouse for HIPAA orgs).
/// Multiple organizations can share a single data store row via organizationIds.
model OrganizationDataStore {
id String @id @default(cuid())
/// Human-readable unique key (e.g. "hipaa-clickhouse-us-east")
key String @unique
/// Organization IDs that use this data store
organizationIds String[]
kind DataStoreKind
/// Versioned config JSON. Structure is discriminated by the top-level `version` field.
config Json
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@index([kind])
}