github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: replicate task runs to clickhouse to power dashboard improvements (#2035)

Browse Source 
* WIP clickhouse package with test containers setup

* More clickhouse client setup now with otel and real tests, and the v1 of raw run events

* Add some additional columns to raw_run_events_v1

* WIP runs dashboard service

* Create a new run engine event bus event for the runs dashboard to hook into

* Track run events in the run engine

* make sure engine v1 runs get synced to CH

* Update the attemptNumber of v3 task runs

* Restructure the run events to be more sparse

* emit more stuff

* Setup replication package

* scaffold the replication package

* replication wip

* resolve conflicts

* more replication stuff

* Add ability to drop the replication slot completely on teardown

* Use the new single replacingmergetree task events table for replication

* get it working

* insert payloads into their own table only on insert and then join

* prepare for using clickhouse cloud and now running ch migrations during boot in the entrypoint.sh

* Handover WIP and tests

* Testing the replication service

* Remove the runs dashboard stuff that we aren't using anymore

* Added a test for large payloads

* hacky typecheck fix

* Fix new internal package typecheck issues and start adding telemetry to the replication service

* tracing over spans, some other improvements

* Improvements to the runs replication service, now ready for testing

* Some fixes and cleanups

* Don't need this code anymore

* move transaction types into the runs replication service

* only send spans where there are transaction events

* A couple of suggested tweaks
This commit is contained in:
Eric Allam
2025-05-12 22:12:36 +01:00
committed by GitHub
parent ccc9764383
commit 65da20c225
88 changed files with 7735 additions and 775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/webapp/app/env.server.ts
+41
View File
@@ -725,6 +725,47 @@ const EnvironmentSchema = z.object({
// BetterStack
BETTERSTACK_API_KEY: z.string().optional(),
BETTERSTACK_STATUS_PAGE_ID: z.string().optional(),
RUN_REPLICATION_REDIS_HOST: z
.string()
.optional()
.transform((v) => v ?? process.env.REDIS_HOST),
RUN_REPLICATION_REDIS_READER_HOST: z
.string()
.optional()
.transform((v) => v ?? process.env.REDIS_READER_HOST),
RUN_REPLICATION_REDIS_READER_PORT: z.coerce
.number()
.optional()
.transform(
(v) =>
v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined)
),
RUN_REPLICATION_REDIS_PORT: z.coerce
.number()
.optional()
.transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)),
RUN_REPLICATION_REDIS_USERNAME: z
.string()
.optional()
.transform((v) => v ?? process.env.REDIS_USERNAME),
RUN_REPLICATION_REDIS_PASSWORD: z
.string()
.optional()
.transform((v) => v ?? process.env.REDIS_PASSWORD),
RUN_REPLICATION_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"),
RUN_REPLICATION_CLICKHOUSE_URL: z.string().optional(),
RUN_REPLICATION_ENABLED: z.string().default("0"),
RUN_REPLICATION_SLOT_NAME: z.string().default("task_runs_to_clickhouse_v1"),
RUN_REPLICATION_PUBLICATION_NAME: z.string().default("task_runs_to_clickhouse_v1_publication"),
RUN_REPLICATION_MAX_FLUSH_CONCURRENCY: z.coerce.number().int().default(100),
RUN_REPLICATION_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
RUN_REPLICATION_FLUSH_BATCH_SIZE: z.coerce.number().int().default(100),
RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS: z.coerce.number().int().default(30_000),
RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS: z.coerce.number().int().default(10_000),
RUN_REPLICATION_ACK_INTERVAL_SECONDS: z.coerce.number().int().default(10),
RUN_REPLICATION_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
});
export type Environment = z.infer<typeof EnvironmentSchema>;
apps/webapp/app/metrics.server.ts
+3 -1
View File
@@ -4,7 +4,9 @@ import { env } from "./env.server";
export const metricsRegister = singleton("metricsRegister", initializeMetricsRegister);
function initializeMetricsRegister() {
export type MetricsRegister = Registry<OpenMetricsContentType>;
function initializeMetricsRegister(): MetricsRegister {
const registry = new Registry<OpenMetricsContentType>();
register.setDefaultLabels({
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.next.runs._index/route.tsx
+506
View File
@@ -0,0 +1,506 @@
import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
import { BeakerIcon, BookOpenIcon } from "@heroicons/react/24/solid";
import { Form, type MetaFunction, useNavigation } from "@remix-run/react";
import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
import { IconCircleX } from "@tabler/icons-react";
import { AnimatePresence, motion } from "framer-motion";
import { ListChecks, ListX } from "lucide-react";
import { Suspense, useState } from "react";
import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
import { TaskIcon } from "~/assets/icons/TaskIcon";
import { DevDisconnectedBanner, useDevPresence } from "~/components/DevPresence";
import { StepContentContainer } from "~/components/StepContentContainer";
import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
import { Button, LinkButton } from "~/components/primitives/Buttons";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTrigger,
} from "~/components/primitives/Dialog";
import { Header1, Header2 } from "~/components/primitives/Headers";
import { InfoPanel } from "~/components/primitives/InfoPanel";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
import {
SelectedItemsProvider,
useSelectedItems,
} from "~/components/primitives/SelectedItemsProvider";
import { Spinner, SpinnerWhite } from "~/components/primitives/Spinner";
import { StepNumber } from "~/components/primitives/StepNumber";
import { TextLink } from "~/components/primitives/TextLink";
import { RunsFilters, TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable";
import { BULK_ACTION_RUN_LIMIT } from "~/consts";
import { useEnvironment } from "~/hooks/useEnvironment";
import { useOrganization } from "~/hooks/useOrganizations";
import { useProject } from "~/hooks/useProject";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { RunListPresenter } from "~/presenters/v3/RunListPresenter.server";
import {
getRootOnlyFilterPreference,
setRootOnlyFilterPreference,
uiPreferencesStorage,
} from "~/services/preferences/uiPreferences.server";
import { requireUserId } from "~/services/session.server";
import { cn } from "~/utils/cn";
import {
docsPath,
EnvironmentParamSchema,
v3ProjectPath,
v3RunsNextPath,
v3TestPath,
} from "~/utils/pathBuilder";
import { ListPagination } from "../../components/ListPagination";
export const meta: MetaFunction = () => {
return [
{
title: `Runs | Trigger.dev`,
},
];
};
export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const userId = await requireUserId(request);
const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params);
const url = new URL(request.url);
let rootOnlyValue = false;
if (url.searchParams.has("rootOnly")) {
rootOnlyValue = url.searchParams.get("rootOnly") === "true";
} else {
rootOnlyValue = await getRootOnlyFilterPreference(request);
}
const project = await findProjectBySlug(organizationSlug, projectParam, userId);
if (!project) {
throw new Error("Project not found");
}
const environment = await findEnvironmentBySlug(project.id, envParam, userId);
if (!environment) {
throw new Error("Environment not found");
}
const s = {
cursor: url.searchParams.get("cursor") ?? undefined,
direction: url.searchParams.get("direction") ?? undefined,
statuses: url.searchParams.getAll("statuses"),
environments: [environment.id],
tasks: url.searchParams.getAll("tasks"),
period: url.searchParams.get("period") ?? undefined,
bulkId: url.searchParams.get("bulkId") ?? undefined,
tags: url.searchParams.getAll("tags").map((t) => decodeURIComponent(t)),
from: url.searchParams.get("from") ?? undefined,
to: url.searchParams.get("to") ?? undefined,
rootOnly: rootOnlyValue,
runId: url.searchParams.get("runId") ?? undefined,
batchId: url.searchParams.get("batchId") ?? undefined,
scheduleId: url.searchParams.get("scheduleId") ?? undefined,
};
const {
tasks,
versions,
statuses,
environments,
tags,
period,
bulkId,
from,
to,
cursor,
direction,
rootOnly,
runId,
batchId,
scheduleId,
} = TaskRunListSearchFilters.parse(s);
const presenter = new RunListPresenter();
const list = presenter.call({
userId,
projectId: project.id,
tasks,
versions,
statuses,
environments,
tags,
period,
bulkId,
from,
to,
batchId,
runIds: runId ? [runId] : undefined,
scheduleId,
rootOnly,
direction: direction,
cursor: cursor,
});
const session = await setRootOnlyFilterPreference(rootOnlyValue, request);
const cookieValue = await uiPreferencesStorage.commitSession(session);
return typeddefer(
{
data: list,
rootOnlyDefault: rootOnlyValue,
},
{
headers: {
"Set-Cookie": cookieValue,
},
}
);
};
export default function Page() {
const { data, rootOnlyDefault } = useTypedLoaderData<typeof loader>();
const navigation = useNavigation();
const isLoading = navigation.state !== "idle";
const { isConnected } = useDevPresence();
const project = useProject();
const environment = useEnvironment();
return (
<>
<NavBar>
<PageTitle title="Runs" />
{environment.type === "DEVELOPMENT" && project.engine === "V2" && (
<DevDisconnectedBanner isConnected={isConnected} />
)}
<PageAccessories>
<LinkButton
variant={"docs/small"}
LeadingIcon={BookOpenIcon}
to={docsPath("/runs-and-attempts")}
>
Runs docs
</LinkButton>
</PageAccessories>
</NavBar>
<PageBody scrollable={false}>
<SelectedItemsProvider
initialSelectedItems={[]}
maxSelectedItemCount={BULK_ACTION_RUN_LIMIT}
>
{({ selectedItems }) => (
<div
className={cn(
"grid h-full max-h-full overflow-hidden",
selectedItems.size === 0 ? "grid-rows-1" : "grid-rows-[1fr_auto]"
)}
>
<Suspense
fallback={
<div className="flex items-center justify-center py-2">
<div className="mx-auto flex items-center gap-2">
<Spinner />
<Paragraph variant="small">Loading runs</Paragraph>
</div>
</div>
}
>
<TypedAwait resolve={data}>
{(list) => (
<>
{list.runs.length === 0 && !list.hasAnyRuns ? (
list.possibleTasks.length === 0 ? (
<CreateFirstTaskInstructions />
) : (
<RunTaskInstructions />
)
) : (
<div
className={cn(
"grid h-full max-h-full grid-rows-[auto_1fr] overflow-hidden"
)}
>
<div className="flex items-start justify-between gap-x-2 p-2">
<RunsFilters
possibleTasks={list.possibleTasks}
bulkActions={list.bulkActions}
hasFilters={list.hasFilters}
rootOnlyDefault={rootOnlyDefault}
/>
<div className="flex items-center justify-end gap-x-2">
<ListPagination list={list} />
</div>
</div>
<TaskRunsTable
total={list.runs.length}
hasFilters={list.hasFilters}
filters={list.filters}
runs={list.runs}
isLoading={isLoading}
allowSelection
/>
</div>
)}
</>
)}
</TypedAwait>
</Suspense>
<BulkActionBar />
</div>
)}
</SelectedItemsProvider>
</PageBody>
</>
);
}
function BulkActionBar() {
const { selectedItems, deselectAll } = useSelectedItems();
const [barState, setBarState] = useState<"none" | "replay" | "cancel">("none");
const hasSelectedMaximum = selectedItems.size >= BULK_ACTION_RUN_LIMIT;
return (
<AnimatePresence>
{selectedItems.size > 0 && (
<motion.div
initial={{ translateY: "100%" }}
animate={{ translateY: 0 }}
exit={{ translateY: "100%" }}
className="flex items-center justify-between gap-3 border-t border-grid-bright bg-background-bright py-3 pl-4 pr-3"
>
<div className="flex items-center gap-1.5 text-sm text-text-bright">
<ListChecks className="mr-1 size-7 text-indigo-400" />
<Header2>Bulk actions:</Header2>
{hasSelectedMaximum ? (
<Paragraph className="text-warning">
Maximum of {selectedItems.size} runs selected
</Paragraph>
) : (
<Paragraph className="">{selectedItems.size} runs selected</Paragraph>
)}
</div>
<div className="flex items-center gap-3">
<CancelRuns
onOpen={(o) => {
if (o) {
setBarState("cancel");
} else {
setBarState("none");
}
}}
/>
<ReplayRuns
onOpen={(o) => {
if (o) {
setBarState("replay");
} else {
setBarState("none");
}
}}
/>
<Button
variant="tertiary/medium"
shortcut={{ key: "esc", enabledOnInputElements: true }}
onClick={() => {
if (barState !== "none") return;
deselectAll();
}}
LeadingIcon={ListX}
leadingIconClassName="text-indigo-400 w-6 h-6"
>
Clear selection
</Button>
</div>
</motion.div>
)}
</AnimatePresence>
);
}
function CancelRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
const { selectedItems } = useSelectedItems();
const organization = useOrganization();
const project = useProject();
const environment = useEnvironment();
const failedRedirect = v3RunsNextPath(organization, project, environment);
const formAction = `/resources/taskruns/bulk/cancel`;
const navigation = useNavigation();
const isLoading = navigation.formAction === formAction;
return (
<Dialog onOpenChange={(o) => onOpen(o)}>
<DialogTrigger asChild>
<Button
type="button"
variant="tertiary/medium"
shortcut={{ key: "c", enabledOnInputElements: true }}
LeadingIcon={IconCircleX}
leadingIconClassName="text-error w-[1.3rem] h-[1.3rem]"
>
Cancel runs
</Button>
</DialogTrigger>
<DialogContent key="replay">
<DialogHeader>Cancel {selectedItems.size} runs?</DialogHeader>
<DialogDescription className="pt-2">
Canceling these runs will stop them from running. Only runs that are not already finished
will be canceled, the others will remain in their existing state.
</DialogDescription>
<DialogFooter>
<Form action={formAction} method="post" reloadDocument>
<input type="hidden" name="failedRedirect" value={failedRedirect} />
<input type="hidden" name="organizationSlug" value={organization.slug} />
<input type="hidden" name="projectSlug" value={project.slug} />
<input type="hidden" name="environmentSlug" value={environment.slug} />
{[...selectedItems].map((runId) => (
<input key={runId} type="hidden" name="runIds" value={runId} />
))}
<Button
type="submit"
variant="danger/medium"
LeadingIcon={isLoading ? SpinnerWhite : StopCircleIcon}
disabled={isLoading}
shortcut={{ modifiers: ["mod"], key: "enter" }}
>
{isLoading ? "Canceling..." : `Cancel ${selectedItems.size} runs`}
</Button>
</Form>
</DialogFooter>
</DialogContent>
</Dialog>
);
}
function ReplayRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
const { selectedItems } = useSelectedItems();
const organization = useOrganization();
const project = useProject();
const environment = useEnvironment();
const failedRedirect = v3RunsNextPath(organization, project, environment);
const formAction = `/resources/taskruns/bulk/replay`;
const navigation = useNavigation();
const isLoading = navigation.formAction === formAction;
return (
<Dialog onOpenChange={(o) => onOpen(o)}>
<DialogTrigger asChild>
<Button
type="button"
variant="tertiary/medium"
shortcut={{ key: "r", enabledOnInputElements: true }}
LeadingIcon={ArrowPathIcon}
leadingIconClassName="text-blue-400 w-[1.3rem] h-[1.3rem]"
>
<span className="text-text-bright">Replay {selectedItems.size} runs</span>
</Button>
</DialogTrigger>
<DialogContent key="replay">
<DialogHeader>Replay runs?</DialogHeader>
<DialogDescription className="pt-2">
Replaying these runs will create a new run for each with the same payload and environment
as the original. It will use the latest version of the code for each task.
</DialogDescription>
<DialogFooter>
<Form action={formAction} method="post" reloadDocument>
<input type="hidden" name="failedRedirect" value={failedRedirect} />
<input type="hidden" name="organizationSlug" value={organization.slug} />
<input type="hidden" name="projectSlug" value={project.slug} />
<input type="hidden" name="environmentSlug" value={environment.slug} />
{[...selectedItems].map((runId) => (
<input key={runId} type="hidden" name="runIds" value={runId} />
))}
<Button
type="submit"
variant="primary/medium"
LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
disabled={isLoading}
shortcut={{ modifiers: ["mod"], key: "enter" }}
>
{isLoading ? "Replaying..." : `Replay ${selectedItems.size} runs`}
</Button>
</Form>
</DialogFooter>
</DialogContent>
</Dialog>
);
}
function CreateFirstTaskInstructions() {
const organization = useOrganization();
const project = useProject();
return (
<MainCenteredContainer className="max-w-md">
<InfoPanel
icon={TaskIcon}
iconClassName="text-blue-500"
panelClassName="max-full"
title="Create your first task"
accessory={
<LinkButton to={v3ProjectPath(organization, project)} variant="primary/small">
Create a task
</LinkButton>
}
>
<Paragraph variant="small">
Before running a task, you must first create one. Follow the instructions on the{" "}
<TextLink to={v3ProjectPath(organization, project)}>Tasks</TextLink> page to create a
task, then return here to run it.
</Paragraph>
</InfoPanel>
</MainCenteredContainer>
);
}
function RunTaskInstructions() {
const organization = useOrganization();
const project = useProject();
const environment = useEnvironment();
return (
<MainCenteredContainer className="max-w-prose">
<Header1 className="mb-6 border-b py-2">How to run your tasks</Header1>
<StepNumber stepNumber="A" title="Trigger a test run" />
<StepContentContainer>
<Paragraph spacing>
Perform a test run with a payload directly from the dashboard.
</Paragraph>
<LinkButton
to={v3TestPath(organization, project, environment)}
variant="secondary/medium"
LeadingIcon={BeakerIcon}
leadingIconClassName="text-lime-500"
className="inline-flex"
>
Test
</LinkButton>
<div className="mt-6 flex items-center gap-2">
<hr className="w-full" />
<Paragraph variant="extra-extra-small/dimmed/caps">OR</Paragraph>
<hr className="w-full" />
</div>
</StepContentContainer>
<StepNumber stepNumber="B" title="Trigger your task for real" />
<StepContentContainer>
<Paragraph spacing>
Performing a real run depends on the type of trigger your task is using.
</Paragraph>
<LinkButton
to={docsPath("/triggering")}
variant="docs/medium"
LeadingIcon={BookOpenIcon}
className="inline-flex"
>
How to trigger a task
</LinkButton>
</StepContentContainer>
</MainCenteredContainer>
);
}
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.next.runs/route.tsx
+10
View File
@@ -0,0 +1,10 @@
import { Outlet } from "@remix-run/react";
import { PageContainer } from "~/components/layout/AppLayout";
export default function Page() {
return (
<PageContainer>
<Outlet />
</PageContainer>
);
}
apps/webapp/app/routes/admin.api.v1.runs-replication.start.ts
+37
View File
@@ -0,0 +1,37 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { prisma } from "~/db.server";
import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
export async function action({ request }: ActionFunctionArgs) {
// Next authenticate the request
const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
if (!authenticationResult) {
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
const user = await prisma.user.findUnique({
where: {
id: authenticationResult.userId,
},
});
if (!user) {
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
if (!user.admin) {
return json({ error: "You must be an admin to perform this action" }, { status: 403 });
}
try {
await runsReplicationInstance?.start();
return json({
success: true,
});
} catch (error) {
return json({ error: error instanceof Error ? error.message : error }, { status: 400 });
}
}
apps/webapp/app/routes/admin.api.v1.runs-replication.stop.ts
+37
View File
@@ -0,0 +1,37 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { prisma } from "~/db.server";
import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
export async function action({ request }: ActionFunctionArgs) {
// Next authenticate the request
const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
if (!authenticationResult) {
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
const user = await prisma.user.findUnique({
where: {
id: authenticationResult.userId,
},
});
if (!user) {
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
if (!user.admin) {
return json({ error: "You must be an admin to perform this action" }, { status: 403 });
}
try {
await runsReplicationInstance?.stop();
return json({
success: true,
});
} catch (error) {
return json({ error: error instanceof Error ? error.message : error }, { status: 400 });
}
}
apps/webapp/app/routes/admin.api.v1.runs-replication.teardown.ts
+37
View File
@@ -0,0 +1,37 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import { prisma } from "~/db.server";
import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
export async function action({ request }: ActionFunctionArgs) {
// Next authenticate the request
const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
if (!authenticationResult) {
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
const user = await prisma.user.findUnique({
where: {
id: authenticationResult.userId,
},
});
if (!user) {
return json({ error: "Invalid or Missing API key" }, { status: 401 });
}
if (!user.admin) {
return json({ error: "You must be an admin to perform this action" }, { status: 403 });
}
try {
await runsReplicationInstance?.teardown();
return json({
success: true,
});
} catch (error) {
return json({ error: error instanceof Error ? error.message : error }, { status: 400 });
}
}
apps/webapp/app/routes/api.v1.runs.$runId.tags.ts
+1 -2
View File
@@ -4,7 +4,6 @@ import { z } from "zod";
import { prisma } from "~/db.server";
import { createTag, getTagsForRunId, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server";
import { authenticateApiRequest } from "~/services/apiAuth.server";
import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
const ParamsSchema = z.object({
runId: z.string(),
@@ -80,7 +79,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
}
}
const taskRun = await prisma.taskRun.update({
await prisma.taskRun.update({
where: {
friendlyId: parsedParams.data.runId,
runtimeEnvironmentId: authenticationResult.environment.id,
apps/webapp/app/runEngine/services/triggerTask.server.ts
+2
View File
@@ -305,6 +305,8 @@ export class RunEngineTriggerTaskService {
? parentRun.queueTimestamp ?? undefined
: undefined,
runChainState,
scheduleId: options.scheduleId,
scheduleInstanceId: options.scheduleInstanceId,
},
this.prisma
);
apps/webapp/app/runEngine/types.ts
-1
View File
@@ -7,7 +7,6 @@ import {
TriggerTaskRequestBody,
} from "@trigger.dev/core/v3";
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { z } from "zod";
export type TriggerTaskServiceOptions = {
idempotencyKey?: string;
apps/webapp/app/services/runsReplicationInstance.server.ts
+69
View File
@@ -0,0 +1,69 @@
import { ClickHouse } from "@internal/clickhouse";
import { RunsReplicationService } from "./runsReplicationService.server";
import { singleton } from "~/utils/singleton";
import invariant from "tiny-invariant";
import { env } from "~/env.server";
import { metricsRegister } from "~/metrics.server";
import { logger } from "./logger.server";
export const runsReplicationInstance = singleton(
"runsReplicationInstance",
initializeRunsReplicationInstance
);
function initializeRunsReplicationInstance() {
const { DATABASE_URL } = process.env;
invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");
if (!env.RUN_REPLICATION_CLICKHOUSE_URL) {
logger.info("🗃️ Runs replication service not enabled");
return;
}
const clickhouse = new ClickHouse({
url: env.RUN_REPLICATION_CLICKHOUSE_URL,
name: "runs-replication",
});
const service = new RunsReplicationService({
clickhouse: clickhouse,
pgConnectionUrl: DATABASE_URL,
serviceName: "runs-replication",
slotName: env.RUN_REPLICATION_SLOT_NAME,
publicationName: env.RUN_REPLICATION_PUBLICATION_NAME,
redisOptions: {
keyPrefix: "runs-replication:",
port: env.RUN_REPLICATION_REDIS_PORT ?? undefined,
host: env.RUN_REPLICATION_REDIS_HOST ?? undefined,
username: env.RUN_REPLICATION_REDIS_USERNAME ?? undefined,
password: env.RUN_REPLICATION_REDIS_PASSWORD ?? undefined,
enableAutoPipelining: true,
...(env.RUN_REPLICATION_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
},
maxFlushConcurrency: env.RUN_REPLICATION_MAX_FLUSH_CONCURRENCY,
flushIntervalMs: env.RUN_REPLICATION_FLUSH_INTERVAL_MS,
flushBatchSize: env.RUN_REPLICATION_FLUSH_BATCH_SIZE,
leaderLockTimeoutMs: env.RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS,
leaderLockExtendIntervalMs: env.RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS,
ackIntervalSeconds: env.RUN_REPLICATION_ACK_INTERVAL_SECONDS,
logLevel: env.RUN_REPLICATION_LOG_LEVEL,
});
if (env.RUN_REPLICATION_ENABLED === "1") {
service
.start()
.then(() => {
logger.info("🗃️ Runs replication service started");
})
.catch((error) => {
logger.error("🗃️ Runs replication service failed to start", {
error,
});
});
process.on("SIGTERM", service.shutdown.bind(service));
process.on("SIGINT", service.shutdown.bind(service));
}
return service;
}
apps/webapp/app/services/runsReplicationService.server.ts
+775
View File
@@ -0,0 +1,775 @@
import type { ClickHouse, RawTaskRunPayloadV1, TaskRunV1 } from "@internal/clickhouse";
import { RedisOptions } from "@internal/redis";
import {
LogicalReplicationClient,
type MessageDelete,
type MessageInsert,
type MessageUpdate,
type PgoutputMessage,
} from "@internal/replication";
import { startSpan, trace, type Tracer } from "@internal/tracing";
import { Logger, LogLevel } from "@trigger.dev/core/logger";
import { tryCatch } from "@trigger.dev/core/utils";
import { parsePacket } from "@trigger.dev/core/v3/utils/ioSerialization";
import { TaskRun } from "@trigger.dev/database";
import { nanoid } from "nanoid";
import EventEmitter from "node:events";
import pLimit from "p-limit";
interface TransactionEvent<T = any> {
tag: "insert" | "update" | "delete";
data: T;
raw: MessageInsert | MessageUpdate | MessageDelete;
}
interface Transaction<T = any> {
beginStartTimestamp: number;
commitLsn: string | null;
commitEndLsn: string | null;
xid: number;
events: TransactionEvent<T>[];
replicationLagMs: number;
}
export type RunsReplicationServiceOptions = {
clickhouse: ClickHouse;
pgConnectionUrl: string;
serviceName: string;
slotName: string;
publicationName: string;
redisOptions: RedisOptions;
maxFlushConcurrency?: number;
flushIntervalMs?: number;
flushBatchSize?: number;
leaderLockTimeoutMs?: number;
leaderLockExtendIntervalMs?: number;
ackIntervalSeconds?: number;
acknowledgeTimeoutMs?: number;
logger?: Logger;
logLevel?: LogLevel;
tracer?: Tracer;
};
type TaskRunInsert = { _version: bigint; run: TaskRun; event: "insert" | "update" | "delete" };
export type RunsReplicationServiceEvents = {
message: [{ lsn: string; message: PgoutputMessage; service: RunsReplicationService }];
};
export class RunsReplicationService {
private _isSubscribed = false;
private _currentTransaction:
| (Omit<Transaction<TaskRun>, "commitEndLsn" | "replicationLagMs"> & {
commitEndLsn?: string | null;
replicationLagMs?: number;
})
| null = null;
private _replicationClient: LogicalReplicationClient;
private _concurrentFlushScheduler: ConcurrentFlushScheduler<TaskRunInsert>;
private logger: Logger;
private _isShuttingDown = false;
private _isShutDownComplete = false;
private _tracer: Tracer;
private _currentParseDurationMs: number | null = null;
private _lastAcknowledgedAt: number | null = null;
private _acknowledgeTimeoutMs: number;
private _latestCommitEndLsn: string | null = null;
private _lastAcknowledgedLsn: string | null = null;
private _acknowledgeInterval: NodeJS.Timeout | null = null;
public readonly events: EventEmitter<RunsReplicationServiceEvents>;
constructor(private readonly options: RunsReplicationServiceOptions) {
this.logger =
options.logger ?? new Logger("RunsReplicationService", options.logLevel ?? "info");
this.events = new EventEmitter();
this._tracer = options.tracer ?? trace.getTracer("runs-replication-service");
this._acknowledgeTimeoutMs = options.acknowledgeTimeoutMs ?? 1_000;
this._replicationClient = new LogicalReplicationClient({
pgConfig: {
connectionString: options.pgConnectionUrl,
},
name: options.serviceName,
slotName: options.slotName,
publicationName: options.publicationName,
table: "TaskRun",
redisOptions: options.redisOptions,
autoAcknowledge: false,
publicationActions: ["insert", "update", "delete"],
logger: new Logger("LogicalReplicationClient", options.logLevel ?? "info"),
leaderLockTimeoutMs: options.leaderLockTimeoutMs ?? 30_000,
leaderLockExtendIntervalMs: options.leaderLockExtendIntervalMs ?? 10_000,
ackIntervalSeconds: options.ackIntervalSeconds ?? 10,
});
this._concurrentFlushScheduler = new ConcurrentFlushScheduler<TaskRunInsert>({
batchSize: options.flushBatchSize ?? 50,
flushInterval: options.flushIntervalMs ?? 100,
maxConcurrency: options.maxFlushConcurrency ?? 100,
callback: this.#flushBatch.bind(this),
logger: new Logger("ConcurrentFlushScheduler", options.logLevel ?? "info"),
});
this._replicationClient.events.on("data", async ({ lsn, log, parseDuration }) => {
this.#handleData(lsn, log, parseDuration);
});
this._replicationClient.events.on("heartbeat", async ({ lsn, shouldRespond }) => {
if (this._isShuttingDown) return;
if (this._isShutDownComplete) return;
if (shouldRespond) {
this._lastAcknowledgedLsn = lsn;
await this._replicationClient.acknowledge(lsn);
}
});
this._replicationClient.events.on("error", (error) => {
this.logger.error("Replication client error", {
error,
});
});
this._replicationClient.events.on("start", () => {
this.logger.debug("Replication client started");
});
this._replicationClient.events.on("acknowledge", ({ lsn }) => {
this.logger.debug("Acknowledged", { lsn });
});
this._replicationClient.events.on("leaderElection", (isLeader) => {
this.logger.debug("Leader election", { isLeader });
});
}
public async shutdown() {
this._isShuttingDown = true;
this.logger.info("Initiating shutdown of runs replication service");
if (!this._currentTransaction) {
this.logger.info("No transaction to commit, shutting down immediately");
await this._replicationClient.stop();
this._isShutDownComplete = true;
return;
}
this._concurrentFlushScheduler.shutdown();
}
async start() {
this.logger.info("Starting replication client", {
lastLsn: this._latestCommitEndLsn,
});
await this._replicationClient.subscribe(this._latestCommitEndLsn ?? undefined);
this._acknowledgeInterval = setInterval(this.#acknowledgeLatestTransaction.bind(this), 1000);
this._concurrentFlushScheduler.start();
}
async stop() {
this.logger.info("Stopping replication client");
await this._replicationClient.stop();
if (this._acknowledgeInterval) {
clearInterval(this._acknowledgeInterval);
}
}
async teardown() {
this.logger.info("Teardown replication client");
await this._replicationClient.teardown();
if (this._acknowledgeInterval) {
clearInterval(this._acknowledgeInterval);
}
}
#handleData(lsn: string, message: PgoutputMessage, parseDuration: bigint) {
this.logger.debug("Handling data", {
lsn,
tag: message.tag,
parseDuration,
});
this.events.emit("message", { lsn, message, service: this });
switch (message.tag) {
case "begin": {
if (this._isShuttingDown || this._isShutDownComplete) {
return;
}
this._currentTransaction = {
beginStartTimestamp: Date.now(),
commitLsn: message.commitLsn,
xid: message.xid,
events: [],
};
this._currentParseDurationMs = Number(parseDuration) / 1_000_000;
break;
}
case "insert": {
if (!this._currentTransaction) {
return;
}
if (this._currentParseDurationMs) {
this._currentParseDurationMs =
this._currentParseDurationMs + Number(parseDuration) / 1_000_000;
}
this._currentTransaction.events.push({
tag: message.tag,
data: message.new as TaskRun,
raw: message,
});
break;
}
case "update": {
if (!this._currentTransaction) {
return;
}
if (this._currentParseDurationMs) {
this._currentParseDurationMs =
this._currentParseDurationMs + Number(parseDuration) / 1_000_000;
}
this._currentTransaction.events.push({
tag: message.tag,
data: message.new as TaskRun,
raw: message,
});
break;
}
case "delete": {
if (!this._currentTransaction) {
return;
}
if (this._currentParseDurationMs) {
this._currentParseDurationMs =
this._currentParseDurationMs + Number(parseDuration) / 1_000_000;
}
this._currentTransaction.events.push({
tag: message.tag,
data: message.old as TaskRun,
raw: message,
});
break;
}
case "commit": {
if (!this._currentTransaction) {
return;
}
if (this._currentParseDurationMs) {
this._currentParseDurationMs =
this._currentParseDurationMs + Number(parseDuration) / 1_000_000;
}
const replicationLagMs = Date.now() - Number(message.commitTime / 1000n);
this._currentTransaction.commitEndLsn = message.commitEndLsn;
this._currentTransaction.replicationLagMs = replicationLagMs;
const transaction = this._currentTransaction as Transaction<TaskRun>;
this._currentTransaction = null;
if (transaction.commitEndLsn) {
this._latestCommitEndLsn = transaction.commitEndLsn;
}
this.#handleTransaction(transaction);
break;
}
default: {
this.logger.debug("Unknown message tag", {
pgMessage: message,
});
}
}
}
#handleTransaction(transaction: Transaction<TaskRun>) {
if (this._isShutDownComplete) return;
if (this._isShuttingDown) {
this._replicationClient.stop().finally(() => {
this._isShutDownComplete = true;
});
}
// If there are no events, do nothing
if (transaction.events.length === 0) {
return;
}
if (!transaction.commitEndLsn) {
this.logger.error("Transaction has no commit end lsn", {
transaction,
});
return;
}
this.logger.debug("Handling transaction", {
transaction,
});
const lsnToUInt64Start = process.hrtime.bigint();
// If there are events, we need to handle them
const _version = lsnToUInt64(transaction.commitEndLsn);
const lsnToUInt64DurationMs = Number(process.hrtime.bigint() - lsnToUInt64Start) / 1_000_000;
this._concurrentFlushScheduler.addToBatch(
transaction.events.map((event) => ({
_version,
run: event.data,
event: event.tag,
}))
);
const currentSpan = this._tracer.startSpan("handle_transaction", {
attributes: {
"transaction.xid": transaction.xid,
"transaction.replication_lag_ms": transaction.replicationLagMs,
"transaction.events": transaction.events.length,
"transaction.commit_end_lsn": transaction.commitEndLsn,
"transaction.parse_duration_ms": this._currentParseDurationMs ?? undefined,
"transaction.lsn_to_uint64_ms": lsnToUInt64DurationMs,
"transaction.version": _version.toString(),
},
startTime: transaction.beginStartTimestamp,
});
currentSpan.end();
}
async #acknowledgeLatestTransaction() {
if (!this._latestCommitEndLsn) {
return;
}
if (this._lastAcknowledgedLsn === this._latestCommitEndLsn) {
return;
}
const now = Date.now();
if (this._lastAcknowledgedAt) {
const timeSinceLastAcknowledged = now - this._lastAcknowledgedAt;
// If we've already acknowledged within the last second, don't acknowledge again
if (timeSinceLastAcknowledged < this._acknowledgeTimeoutMs) {
return;
}
}
this._lastAcknowledgedAt = now;
this._lastAcknowledgedLsn = this._latestCommitEndLsn;
this.logger.debug("Acknowledging transaction", {
commitEndLsn: this._latestCommitEndLsn,
lastAcknowledgedAt: this._lastAcknowledgedAt,
});
const [ackError] = await tryCatch(
this._replicationClient.acknowledge(this._latestCommitEndLsn)
);
if (ackError) {
this.logger.error("Error acknowledging transaction", { ackError });
}
if (this._isShutDownComplete && this._acknowledgeInterval) {
clearInterval(this._acknowledgeInterval);
}
}
async #flushBatch(flushId: string, batch: Array<TaskRunInsert>) {
if (batch.length === 0) {
this.logger.debug("No runs to flush", {
flushId,
});
return;
}
this.logger.debug("Flushing batch", {
flushId,
batchSize: batch.length,
});
await startSpan(this._tracer, "flushBatch", async (span) => {
const preparedInserts = await startSpan(this._tracer, "prepare_inserts", async (span) => {
return await Promise.all(batch.map(this.#prepareRunInserts.bind(this)));
});
const taskRunInserts = preparedInserts
.map(({ taskRunInsert }) => taskRunInsert)
.filter(Boolean);
const payloadInserts = preparedInserts
.map(({ payloadInsert }) => payloadInsert)
.filter(Boolean);
span.setAttribute("task_run_inserts", taskRunInserts.length);
span.setAttribute("payload_inserts", payloadInserts.length);
this.logger.debug("Flushing inserts", {
flushId,
taskRunInserts: taskRunInserts.length,
payloadInserts: payloadInserts.length,
});
await Promise.all([
this.#insertTaskRunInserts(taskRunInserts),
this.#insertPayloadInserts(payloadInserts),
]);
this.logger.debug("Flushed inserts", {
flushId,
taskRunInserts: taskRunInserts.length,
payloadInserts: payloadInserts.length,
});
});
}
async #insertTaskRunInserts(taskRunInserts: TaskRunV1[]) {
const [insertError, insertResult] = await this.options.clickhouse.taskRuns.insert(
taskRunInserts,
{
params: {
clickhouse_settings: {
wait_for_async_insert: 1,
},
},
}
);
if (insertError) {
this.logger.error("Error inserting task run inserts", {
error: insertError,
});
}
return insertResult;
}
async #insertPayloadInserts(payloadInserts: RawTaskRunPayloadV1[]) {
const [insertError, insertResult] = await this.options.clickhouse.taskRuns.insertPayloads(
payloadInserts,
{
params: {
clickhouse_settings: {
wait_for_async_insert: 1,
},
},
}
);
if (insertError) {
this.logger.error("Error inserting payload inserts", {
error: insertError,
});
}
return insertResult;
}
async #prepareRunInserts(
batchedRun: TaskRunInsert
): Promise<{ taskRunInsert?: TaskRunV1; payloadInsert?: RawTaskRunPayloadV1 }> {
this.logger.debug("Preparing run", {
batchedRun,
});
const { run, _version, event } = batchedRun;
if (!run.environmentType) {
return {
taskRunInsert: undefined,
payloadInsert: undefined,
};
}
if (!run.organizationId) {
return {
taskRunInsert: undefined,
payloadInsert: undefined,
};
}
if (event === "update" || event === "delete") {
const taskRunInsert = await this.#prepareTaskRunInsert(
run,
run.organizationId,
run.environmentType,
event,
_version
);
return {
taskRunInsert,
payloadInsert: undefined,
};
}
const [taskRunInsert, payloadInsert] = await Promise.all([
this.#prepareTaskRunInsert(run, run.organizationId, run.environmentType, event, _version),
this.#preparePayloadInsert(run, _version),
]);
return {
taskRunInsert,
payloadInsert,
};
}
async #prepareTaskRunInsert(
run: TaskRun,
organizationId: string,
environmentType: string,
event: "insert" | "update" | "delete",
_version: bigint
): Promise<TaskRunV1> {
const output = await this.#prepareJson(run.output, run.outputType);
return {
environment_id: run.runtimeEnvironmentId,
organization_id: organizationId,
project_id: run.projectId,
run_id: run.id,
updated_at: run.updatedAt.getTime(),
created_at: run.createdAt.getTime(),
status: run.status,
environment_type: environmentType,
friendly_id: run.friendlyId,
engine: run.engine,
task_identifier: run.taskIdentifier,
queue: run.queue,
span_id: run.spanId,
trace_id: run.traceId,
error: { data: run.error },
attempt: run.attemptNumber ?? 1,
schedule_id: run.scheduleId ?? "",
batch_id: run.batchId ?? "",
completed_at: run.completedAt?.getTime(),
started_at: run.startedAt?.getTime(),
executed_at: run.executedAt?.getTime(),
delay_until: run.delayUntil?.getTime(),
queued_at: run.queuedAt?.getTime(),
expired_at: run.expiredAt?.getTime(),
usage_duration_ms: run.usageDurationMs,
cost_in_cents: run.costInCents,
base_cost_in_cents: run.baseCostInCents,
tags: run.runTags ?? [],
task_version: run.taskVersion ?? "",
sdk_version: run.sdkVersion ?? "",
cli_version: run.cliVersion ?? "",
machine_preset: run.machinePreset ?? "",
root_run_id: run.rootTaskRunId ?? "",
parent_run_id: run.parentTaskRunId ?? "",
depth: run.depth,
is_test: run.isTest,
idempotency_key: run.idempotencyKey ?? "",
expiration_ttl: run.ttl ?? "",
output,
_version: _version.toString(),
_is_deleted: event === "delete" ? 1 : 0,
};
}
async #preparePayloadInsert(run: TaskRun, _version: bigint): Promise<RawTaskRunPayloadV1> {
const payload = await this.#prepareJson(run.payload, run.payloadType);
return {
run_id: run.id,
created_at: run.createdAt.getTime(),
payload,
};
}
async #prepareJson(
data: string | undefined | null,
dataType: string
): Promise<{ data: unknown }> {
if (!data) {
return { data: undefined };
}
if (dataType !== "application/json" && dataType !== "application/super+json") {
return { data: undefined };
}
const packet = {
data,
dataType,
};
const [parseError, parsedData] = await tryCatch(parsePacket(packet));
if (parseError) {
this.logger.error("Error parsing packet", {
error: parseError,
packet,
});
return { data: undefined };
}
return { data: parsedData };
}
}
export type ConcurrentFlushSchedulerConfig<T> = {
batchSize: number;
flushInterval: number;
maxConcurrency?: number;
callback: (flushId: string, batch: T[]) => Promise<void>;
tracer?: Tracer;
logger?: Logger;
};
export class ConcurrentFlushScheduler<T> {
private currentBatch: T[]; // Adjust the type according to your data structure
private readonly BATCH_SIZE: number;
private readonly flushInterval: number;
private readonly MAX_CONCURRENCY: number;
private readonly concurrencyLimiter: ReturnType<typeof pLimit>;
private flushTimer: NodeJS.Timeout | null;
private failedBatchCount;
private logger: Logger;
private _tracer: Tracer;
private _isShutDown = false;
constructor(private readonly config: ConcurrentFlushSchedulerConfig<T>) {
this.logger = config.logger ?? new Logger("ConcurrentFlushScheduler", "info");
this._tracer = config.tracer ?? trace.getTracer("concurrent-flush-scheduler");
this.currentBatch = [];
this.BATCH_SIZE = config.batchSize;
this.flushInterval = config.flushInterval;
this.MAX_CONCURRENCY = config.maxConcurrency || 1;
this.concurrencyLimiter = pLimit(this.MAX_CONCURRENCY);
this.flushTimer = null;
this.failedBatchCount = 0;
}
addToBatch(items: T[]): void {
this.currentBatch = this.currentBatch.concat(items);
this.#flushNextBatchIfNeeded();
}
start(): void {
this.logger.info("Starting ConcurrentFlushScheduler", {
batchSize: this.BATCH_SIZE,
flushInterval: this.flushInterval,
maxConcurrency: this.MAX_CONCURRENCY,
});
this.#startFlushTimer();
}
shutdown(): void {
this.logger.info("Shutting down ConcurrentFlushScheduler");
this._isShutDown = true;
this.#clearTimer();
this.#flushNextBatchIfNeeded();
}
#flushNextBatchIfNeeded(): void {
if (this.currentBatch.length >= this.BATCH_SIZE || this._isShutDown) {
this.logger.debug("Batch size threshold reached, initiating flush", {
batchSize: this.BATCH_SIZE,
currentSize: this.currentBatch.length,
isShutDown: this._isShutDown,
});
this.#flushNextBatch().catch((error) => {
this.logger.error("Error flushing next batch", {
error,
});
});
}
}
#startFlushTimer(): void {
this.flushTimer = setInterval(() => this.#checkAndFlush().catch(() => {}), this.flushInterval);
this.logger.debug("Started flush timer", { interval: this.flushInterval });
}
#clearTimer(): void {
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.logger.debug("Flush timer cleared");
}
}
async #checkAndFlush(): Promise<void> {
if (this.currentBatch.length > 0) {
this.logger.debug("Periodic flush check triggered", {
currentBatchSize: this.currentBatch.length,
});
await this.#flushNextBatch();
}
}
async #flushNextBatch(): Promise<void> {
if (this.currentBatch.length === 0) return;
const batch = this.currentBatch;
this.currentBatch = [];
const callback = this.config.callback;
const promise = this.concurrencyLimiter(async () => {
await startSpan(this._tracer, "flushNextBatch", async (span) => {
const batchId = nanoid();
span.setAttribute("batch_id", batchId);
span.setAttribute("batch_size", batch.length);
span.setAttribute("concurrency_active_count", this.concurrencyLimiter.activeCount);
span.setAttribute("concurrency_pending_count", this.concurrencyLimiter.pendingCount);
span.setAttribute("concurrency_concurrency", this.concurrencyLimiter.concurrency);
await callback(batchId, batch);
});
});
const [error] = await tryCatch(promise);
if (error) {
this.logger.error("Error flushing batch", {
error,
});
this.failedBatchCount++;
}
this.logger.debug("Batch flush complete", {
totalBatches: 1,
successfulBatches: 1,
failedBatches: 0,
totalFailedBatches: this.failedBatchCount,
});
}
}
function lsnToUInt64(lsn: string): bigint {
const [seg, off] = lsn.split("/");
return (BigInt("0x" + seg) << 32n) | BigInt("0x" + off);
}
apps/webapp/app/utils/pathBuilder.ts
+11
View File
@@ -233,6 +233,17 @@ export function v3RunsPath(
return `${v3EnvironmentPath(organization, project, environment)}/runs${query}`;
}
export function v3RunsNextPath(
organization: OrgForPath,
project: ProjectForPath,
environment: EnvironmentForPath,
filters?: TaskRunListSearchFilters
) {
const searchParams = objectToSearchParams(filters);
const query = searchParams ? `?${searchParams.toString()}` : "";
return `${v3EnvironmentPath(organization, project, environment)}/runs/next${query}`;
}
export function v3RunPath(
organization: OrgForPath,
project: ProjectForPath,
apps/webapp/app/v3/marqs/devQueueConsumer.server.ts
+8 -5
View File
@@ -6,11 +6,14 @@ import {
TaskRunFailedExecutionResult,
serverWebsocketMessages,
} from "@trigger.dev/core/v3";
import { getMaxDuration } from "@trigger.dev/core/v3/isomorphic";
import { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
import { BackgroundWorker, BackgroundWorkerTask } from "@trigger.dev/database";
import { z } from "zod";
import { prisma } from "~/db.server";
import { createNewSession, disconnectSession } from "~/models/runtimeEnvironment.server";
import { findQueueInEnvironment, sanitizeQueueName } from "~/models/taskQueue.server";
import { RedisClient, createRedisClient } from "~/redis.server";
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { marqs } from "~/v3/marqs/index.server";
@@ -19,10 +22,7 @@ import { FailedTaskRunService } from "../failedTaskRun.server";
import { CancelDevSessionRunsService } from "../services/cancelDevSessionRuns.server";
import { CompleteAttemptService } from "../services/completeAttempt.server";
import { attributesFromAuthenticatedEnv, tracer } from "../tracer.server";
import { getMaxDuration } from "@trigger.dev/core/v3/isomorphic";
import { DevSubscriber, devPubSub } from "./devPubSub.server";
import { findQueueInEnvironment, sanitizeQueueName } from "~/models/taskQueue.server";
import { createRedisClient, RedisClient } from "~/redis.server";
const MessageBody = z.discriminatedUnion("type", [
z.object({
@@ -440,19 +440,22 @@ export class DevQueueConsumer {
return;
}
const lockedAt = new Date();
const startedAt = existingTaskRun.startedAt ?? new Date();
const lockedTaskRun = await prisma.taskRun.update({
where: {
id: message.messageId,
},
data: {
lockedAt: new Date(),
lockedAt,
lockedById: backgroundTask.id,
status: "EXECUTING",
lockedToVersionId: backgroundWorker.id,
taskVersion: backgroundWorker.version,
sdkVersion: backgroundWorker.sdkVersion,
cliVersion: backgroundWorker.cliVersion,
startedAt: existingTaskRun.startedAt ?? new Date(),
startedAt,
maxDurationInSeconds: getMaxDuration(
existingTaskRun.maxDurationInSeconds,
backgroundTask.maxDurationInSeconds
apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts
+17 -11
View File
@@ -707,26 +707,32 @@ export class SharedQueueConsumer {
};
}
const lockedAt = new Date();
const machinePreset =
existingTaskRun.machinePreset ??
machinePresetFromConfig(backgroundTask.machineConfig ?? {}).name;
const maxDurationInSeconds = getMaxDuration(
existingTaskRun.maxDurationInSeconds,
backgroundTask.maxDurationInSeconds
);
const startedAt = existingTaskRun.startedAt ?? dequeuedAt;
const baseCostInCents = env.CENTS_PER_RUN;
const lockedTaskRun = await prisma.taskRun.update({
where: {
id: message.messageId,
},
data: {
lockedAt: new Date(),
lockedAt,
lockedById: backgroundTask.id,
lockedToVersionId: worker.id,
taskVersion: worker.version,
sdkVersion: worker.sdkVersion,
cliVersion: worker.cliVersion,
startedAt: existingTaskRun.startedAt ?? dequeuedAt,
baseCostInCents: env.CENTS_PER_RUN,
machinePreset:
existingTaskRun.machinePreset ??
machinePresetFromConfig(backgroundTask.machineConfig ?? {}).name,
maxDurationInSeconds: getMaxDuration(
existingTaskRun.maxDurationInSeconds,
backgroundTask.maxDurationInSeconds
),
startedAt: startedAt,
baseCostInCents: baseCostInCents,
machinePreset: machinePreset,
maxDurationInSeconds,
},
include: {
runtimeEnvironment: true,
@@ -1430,7 +1436,7 @@ export class SharedQueueConsumer {
async #markRunAsWaitingForDeploy(runId: string) {
logger.debug("Marking run as waiting for deploy", { runId });
return await prisma.taskRun.update({
const run = await prisma.taskRun.update({
where: {
id: runId,
},
apps/webapp/app/v3/services/completeAttempt.server.ts
+2 -3
View File
@@ -10,7 +10,6 @@ import {
TaskRunFailedExecutionResult,
TaskRunSuccessfulExecutionResult,
flattenAttributes,
isManualOutOfMemoryError,
isOOMRunError,
sanitizeError,
shouldRetryError,
@@ -26,14 +25,14 @@ import { safeJsonParse } from "~/utils/json";
import { marqs } from "~/v3/marqs/index.server";
import { createExceptionPropertiesFromError, eventRepository } from "../eventRepository.server";
import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
import { socketIo } from "../handleSocketIo.server";
import { getTaskEventStoreTableForRun } from "../taskEventStore.server";
import { FAILED_RUN_STATUSES, isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
import { BaseService } from "./baseService.server";
import { CancelAttemptService } from "./cancelAttempt.server";
import { CreateCheckpointService } from "./createCheckpoint.server";
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
import { RetryAttemptService } from "./retryAttempt.server";
import { getTaskEventStoreTableForRun } from "../taskEventStore.server";
import { socketIo } from "../handleSocketIo.server";
type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
apps/webapp/app/v3/services/createTaskRunAttempt.server.ts
+3 -2
View File
@@ -2,16 +2,16 @@ import { parsePacket, TaskRunExecution } from "@trigger.dev/core/v3";
import { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
import { $transaction, prisma, PrismaClientOrTransaction } from "~/db.server";
import { findQueueInEnvironment } from "~/models/taskQueue.server";
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { reportInvocationUsage } from "~/services/platform.v3.server";
import { generateFriendlyId } from "../friendlyIdentifiers";
import { machinePresetFromConfig, machinePresetFromRun } from "../machinePresets.server";
import { FINAL_RUN_STATUSES } from "../taskStatus";
import { BaseService, ServiceValidationError } from "./baseService.server";
import { CrashTaskRunService } from "./crashTaskRun.server";
import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
import { findQueueInEnvironment } from "~/models/taskQueue.server";
import { FINAL_RUN_STATUSES } from "../taskStatus";
export class CreateTaskRunAttemptService extends BaseService {
public async call({
@@ -159,6 +159,7 @@ export class CreateTaskRunAttemptService extends BaseService {
data: {
status: setToExecuting ? "EXECUTING" : undefined,
executedAt: taskRun.executedAt ?? new Date(),
attemptNumber: nextAttemptNumber,
},
});
apps/webapp/app/v3/services/enqueueDelayedRun.server.ts
+19 -23
View File
@@ -1,12 +1,10 @@
import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic";
import { $transaction } from "~/db.server";
import { logger } from "~/services/logger.server";
import { marqs } from "~/v3/marqs/index.server";
import { BaseService } from "./baseService.server";
import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
import { commonWorker } from "../commonWorker.server";
import { workerQueue } from "~/services/worker.server";
import { commonWorker } from "../commonWorker.server";
import { BaseService } from "./baseService.server";
import { enqueueRun } from "./enqueueRun.server";
import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
export class EnqueueDelayedRunService extends BaseService {
public static async enqueue(runId: string, runAt?: Date) {
@@ -82,26 +80,24 @@ export class EnqueueDelayedRunService extends BaseService {
return;
}
await $transaction(this._prisma, "delayed run enqueue", async (tx) => {
await tx.taskRun.update({
where: {
id: run.id,
},
data: {
status: "PENDING",
queuedAt: new Date(),
},
});
if (run.ttl) {
const expireAt = parseNaturalLanguageDuration(run.ttl);
if (expireAt) {
await ExpireEnqueuedRunService.enqueue(run.id, expireAt);
}
}
await this._prisma.taskRun.update({
where: {
id: run.id,
},
data: {
status: "PENDING",
queuedAt: new Date(),
},
});
if (run.ttl) {
const expireAt = parseNaturalLanguageDuration(run.ttl);
if (expireAt) {
await ExpireEnqueuedRunService.enqueue(run.id, expireAt);
}
}
await enqueueRun({
env: run.runtimeEnvironment,
run: run,
apps/webapp/app/v3/services/executeTasksWaitingForDeploy.ts
+4 -2
View File
@@ -1,9 +1,9 @@
import { PrismaClientOrTransaction } from "~/db.server";
import { env } from "~/env.server";
import { logger } from "~/services/logger.server";
import { workerQueue } from "~/services/worker.server";
import { marqs } from "~/v3/marqs/index.server";
import { BaseService } from "./baseService.server";
import { logger } from "~/services/logger.server";
import { env } from "~/env.server";
export class ExecuteTasksWaitingForDeployService extends BaseService {
public async call(backgroundWorkerId: string) {
@@ -51,6 +51,8 @@ export class ExecuteTasksWaitingForDeployService extends BaseService {
taskIdentifier: true,
concurrencyKey: true,
queue: true,
updatedAt: true,
createdAt: true,
},
take: maxCount + 1,
});
apps/webapp/app/v3/services/finalizeTaskRun.server.ts
+3 -1
View File
@@ -94,9 +94,11 @@ export class FinalizeTaskRunService extends BaseService {
// - A single update is more efficient than two
// - If the status updates to a final status, realtime will receive that status and then shut down the stream
// before the error is updated, which would cause the error to be lost
const taskRunError = error ? sanitizeError(error) : undefined;
const run = await this._prisma.taskRun.update({
where: { id },
data: { status, expiredAt, completedAt, error: error ? sanitizeError(error) : undefined },
data: { status, expiredAt, completedAt, error: taskRunError },
...(include ? { include } : {}),
});
apps/webapp/app/v3/services/triggerScheduledTask.server.ts
+5 -11
View File
@@ -147,7 +147,11 @@ export class TriggerScheduledTaskService extends BaseService {
instance.taskSchedule.taskIdentifier,
instance.environment,
{ payload: payloadPacket.data, options: { payloadType: payloadPacket.dataType } },
{ customIcon: "scheduled" }
{
customIcon: "scheduled",
scheduleId: instance.taskSchedule.id,
scheduleInstanceId: instance.id,
}
);
if (!result) {
@@ -157,16 +161,6 @@ export class TriggerScheduledTaskService extends BaseService {
payloadPacket,
});
} else {
await this._prisma.taskRun.update({
where: {
id: result.run.id,
},
data: {
scheduleId: instance.taskSchedule.id,
scheduleInstanceId: instance.id,
},
});
await this._prisma.taskSchedule.update({
where: {
id: instance.taskSchedule.id,
apps/webapp/app/v3/services/triggerTask.server.ts
+2
View File
@@ -29,6 +29,8 @@ export type TriggerTaskServiceOptions = {
runFriendlyId?: string;
skipChecks?: boolean;
oneTimeUseToken?: string;
scheduleId?: string;
scheduleInstanceId?: string;
};
export class OutOfEntitlementError extends Error {
apps/webapp/app/v3/services/triggerTaskV1.server.ts
+4
View File
@@ -375,6 +375,8 @@ export class TriggerTaskServiceV1 extends BaseService {
number: num,
friendlyId: runFriendlyId,
runtimeEnvironmentId: environment.id,
environmentType: environment.type,
organizationId: environment.organizationId,
projectId: environment.projectId,
idempotencyKey,
idempotencyKeyExpiresAt: idempotencyKey ? idempotencyKeyExpiresAt : undefined,
@@ -434,6 +436,8 @@ export class TriggerTaskServiceV1 extends BaseService {
runTags: bodyTags,
oneTimeUseToken: options.oneTimeUseToken,
machinePreset: body.options?.machine,
scheduleId: options.scheduleId,
scheduleInstanceId: options.scheduleInstanceId,
},
});
apps/webapp/package.json
+5 -1
View File
@@ -105,6 +105,7 @@
"@trigger.dev/otlp-importer": "workspace:*",
"@trigger.dev/platform": "1.0.14",
"@trigger.dev/sdk": "workspace:*",
"@internal/tracing": "workspace:*",
"@types/pg": "8.6.6",
"@uiw/react-codemirror": "^4.19.5",
"@unkey/cache": "^1.5.0",
@@ -146,6 +147,7 @@
"ohash": "^1.1.3",
"openai": "^4.33.1",
"parse-duration": "^1.1.0",
"p-limit": "^6.2.0",
"posthog-js": "^1.93.3",
"posthog-node": "^3.1.3",
"prism-react-renderer": "^2.3.1",
@@ -193,6 +195,8 @@
},
"devDependencies": {
"@internal/testcontainers": "workspace:*",
"@internal/replication": "workspace:*",
"@internal/clickhouse": "workspace:*",
"@remix-run/dev": "2.1.0",
"@remix-run/eslint-config": "2.1.0",
"@remix-run/testing": "^2.1.0",
@@ -258,4 +262,4 @@
"engines": {
"node": ">=16.0.0"
}
}
}
apps/webapp/remix.config.js
+1
View File
@@ -23,6 +23,7 @@ module.exports = {
"superjson",
"prismjs/components/prism-json",
"prismjs/components/prism-typescript",
"redlock",
],
browserNodeBuiltinsPolyfill: { modules: { path: true, os: true, crypto: true } },
};
apps/webapp/test/engine/triggerTask.test.ts
-1
View File
@@ -32,7 +32,6 @@ import {
ValidationResult,
} from "~/runEngine/types";
import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server";
import { getEntitlement } from "~/services/platform.v3.server";
vi.setConfig({ testTimeout: 30_000 }); // 30 seconds timeout
apps/webapp/test/runsReplicationService.test.ts
+1506
View File
File diff suppressed because it is too large Load Diff
apps/webapp/test/utils/tracing.ts
+19
View File
@@ -0,0 +1,19 @@
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
import { InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
import { context, trace } from "@opentelemetry/api";
export function createInMemoryTracing() {
// Initialize the tracer provider and exporter
const provider = new NodeTracerProvider();
const exporter = new InMemorySpanExporter();
provider.addSpanProcessor(new SimpleSpanProcessor(exporter));
provider.register();
// Retrieve the tracer
const tracer = trace.getTracer("test-tracer");
return {
exporter,
tracer,
};
}
apps/webapp/tsconfig.json
+2 -2
View File
@@ -3,14 +3,14 @@
"include": ["remix.env.d.ts", "global.d.ts", "**/*.ts", "**/*.tsx"],
"compilerOptions": {
"types": ["vitest/globals"],
"lib": ["DOM", "DOM.Iterable", "DOM.AsyncIterable", "ES2019"],
"lib": ["DOM", "DOM.Iterable", "DOM.AsyncIterable", "ES2020"],
"isolatedModules": true,
"esModuleInterop": true,
"jsx": "react-jsx",
"module": "esnext",
"moduleResolution": "Bundler",
"resolveJsonModule": true,
"target": "ES2019",
"target": "ES2020",
"strict": true,
"allowJs": true,
"forceConsistentCasingInFileNames": true,
docker/Dockerfile
+12
View File
@@ -1,5 +1,8 @@
ARG NODE_IMAGE=node:20.11.1-bullseye-slim@sha256:5a5a92b3a8d392691c983719dbdc65d9f30085d6dcd65376e7a32e6fe9bf4cbe
FROM golang:1.23-alpine AS goose_builder
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
FROM ${NODE_IMAGE} AS pruner
WORKDIR /triggerdotdev
@@ -43,6 +46,11 @@ WORKDIR /triggerdotdev
# Corepack is used to install pnpm
RUN corepack enable
# Goose and schemas
COPY --from=goose_builder /go/bin/goose /usr/local/bin/goose
RUN chmod +x /usr/local/bin/goose
COPY --chown=node:node internal-packages/clickhouse/schema /triggerdotdev/internal-packages/clickhouse/schema
COPY --from=pruner --chown=node:node /triggerdotdev/out/full/ .
COPY --from=dev-deps --chown=node:node /triggerdotdev/ .
COPY --chown=node:node turbo.json turbo.json
@@ -70,6 +78,10 @@ COPY --from=builder --chown=node:node /triggerdotdev/apps/webapp/public ./apps/w
COPY --from=builder --chown=node:node /triggerdotdev/apps/webapp/prisma/seed.js ./apps/webapp/prisma/seed.js
COPY --from=builder --chown=node:node /triggerdotdev/scripts ./scripts
# Goose and schemas
COPY --from=builder /usr/local/bin/goose /usr/local/bin/goose
COPY --from=builder --chown=node:node /triggerdotdev/internal-packages/clickhouse/schema /triggerdotdev/internal-packages/clickhouse/schema
EXPOSE 3000
USER node
docker/dev-compose.yml
+95 -25
View File
@@ -3,6 +3,7 @@ version: "3"
volumes:
database-data:
redis-data:
clickhouse:
networks:
app_network:
@@ -10,8 +11,10 @@ networks:
services:
db:
container_name: devdb
image: postgres:14
container_name: db-dev
build:
context: .
dockerfile: Dockerfile.postgres
restart: always
volumes:
- database-data:/var/lib/postgresql/data/
@@ -23,29 +26,63 @@ services:
- app_network
ports:
- 5432:5432
app:
build:
context: ../
dockerfile: ./docker/Dockerfile
ports:
- 3030:3030
depends_on:
- db
env_file:
- ../.env
command:
- -c
- listen_addresses=*
- -c
- wal_level=logical
- -c
- shared_preload_libraries=pg_partman_bgw
electric:
container_name: electric-dev
image: electricsql/electric:1.0.0-beta.15@sha256:4ae0f895753b82684aa31ea1c708e9e86d0a9bca355acb7270dcb24062520810
restart: always
environment:
DATABASE_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
DIRECT_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
SESSION_SECRET: secret123
MAGIC_LINK_SECRET: secret123
ENCRYPTION_KEY: secret123
REMIX_APP_PORT: 3030
PORT: 3030
DATABASE_URL: postgresql://postgres:postgres@db:5432/postgres?sslmode=disable
networks:
- app_network
ports:
- "3060:3000"
depends_on:
- db
clickhouse:
image: bitnami/clickhouse:latest
container_name: clickhouse-dev
environment:
CLICKHOUSE_ADMIN_USER: default
CLICKHOUSE_ADMIN_PASSWORD: password
ports:
- "8123:8123"
- "9000:9000"
volumes:
- clickhouse:/bitnami/clickhouse
networks:
- app_network
healthcheck:
test:
[
"CMD",
"clickhouse-client",
"--host",
"localhost",
"--port",
"9000",
"--user",
"default",
"--password",
"password",
"--query",
"SELECT 1",
]
interval: 3s
timeout: 5s
retries: 5
start_period: 10s
redis:
container_name: redis
container_name: redis-dev
image: redis:7
restart: always
volumes:
@@ -55,9 +92,42 @@ services:
ports:
- 6379:6379
redisinsight:
image: redislabs/redisinsight:latest
app:
build:
context: ../
dockerfile: ./docker/Dockerfile
ports:
- "8001:8001"
volumes:
- redis-data:/redisinsight
- 3030:3030
depends_on:
- db
- electric
- clickhouse
- redis
env_file:
- ../.env
environment:
DATABASE_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
DIRECT_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
CLICKHOUSE_URL: http://default:password@clickhouse:8123
SESSION_SECRET: secret123
MAGIC_LINK_SECRET: secret123
ENCRYPTION_KEY: secret123
REMIX_APP_PORT: 3030
PORT: 3030
networks:
- app_network
ch-ui:
image: ghcr.io/caioricciuti/ch-ui:latest
container_name: ch-ui-dev
restart: always
ports:
- "5521:5521"
environment:
VITE_CLICKHOUSE_URL: "http://clickhouse:8123"
VITE_CLICKHOUSE_USER: "default"
VITE_CLICKHOUSE_PASS: "password"
depends_on:
- clickhouse
networks:
- app_network
docker/docker-compose.yml
+58 -93
View File
@@ -5,12 +5,7 @@ volumes:
database-data-alt:
pgadmin-data:
redis-data:
redis-cluster_data-0:
redis-cluster_data-1:
redis-cluster_data-2:
redis-cluster_data-3:
redis-cluster_data-4:
redis-cluster_data-5:
clickhouse:
networks:
app_network:
@@ -52,93 +47,6 @@ services:
ports:
- 6379:6379
# redis-node-0:
# image: docker.io/bitnami/redis-cluster:7.0
# container_name: redis-node-0
# networks:
# - app_network
# ports:
# - "6378:6379"
# volumes:
# - redis-cluster_data-0:/bitnami/redis/data
# environment:
# - "REDIS_PASSWORD=bitnami"
# - "REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
# redis-node-1:
# image: docker.io/bitnami/redis-cluster:7.0
# container_name: redis-node-1
# networks:
# - app_network
# ports:
# - "6380:6379"
# volumes:
# - redis-cluster_data-1:/bitnami/redis/data
# environment:
# - "REDIS_PASSWORD=bitnami"
# - "REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
# redis-node-2:
# image: docker.io/bitnami/redis-cluster:7.0
# container_name: redis-node-2
# networks:
# - app_network
# ports:
# - "6381:6379"
# volumes:
# - redis-cluster_data-2:/bitnami/redis/data
# environment:
# - "REDIS_PASSWORD=bitnami"
# - "REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
# redis-node-3:
# image: docker.io/bitnami/redis-cluster:7.0
# container_name: redis-node-3
# networks:
# - app_network
# ports:
# - "6382:6379"
# volumes:
# - redis-cluster_data-3:/bitnami/redis/data
# environment:
# - "REDIS_PASSWORD=bitnami"
# - "REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
# redis-node-4:
# image: docker.io/bitnami/redis-cluster:7.0
# container_name: redis-node-4
# networks:
# - app_network
# ports:
# - "6383:6379"
# volumes:
# - redis-cluster_data-4:/bitnami/redis/data
# environment:
# - "REDIS_PASSWORD=bitnami"
# - "REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
# redis-node-5:
# image: docker.io/bitnami/redis-cluster:7.0
# container_name: redis-node-5
# networks:
# - app_network
# ports:
# - "6384:6379"
# volumes:
# - redis-cluster_data-5:/bitnami/redis/data
# depends_on:
# - redis-node-0
# - redis-node-1
# - redis-node-2
# - redis-node-3
# - redis-node-4
# environment:
# - "REDIS_PASSWORD=bitnami"
# - "REDISCLI_AUTH=bitnami"
# - "REDIS_CLUSTER_REPLICAS=1"
# - "REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
# - "REDIS_CLUSTER_CREATOR=yes"
electric:
container_name: electric
image: electricsql/electric:1.0.0-beta.15@sha256:4ae0f895753b82684aa31ea1c708e9e86d0a9bca355acb7270dcb24062520810
@@ -152,6 +60,63 @@ services:
depends_on:
- database
clickhouse:
image: bitnami/clickhouse:latest
container_name: clickhouse
environment:
CLICKHOUSE_ADMIN_USER: default
CLICKHOUSE_ADMIN_PASSWORD: password
ports:
- "8123:8123"
- "9000:9000"
volumes:
- clickhouse:/bitnami/clickhouse
networks:
- app_network
healthcheck:
test:
[
"CMD",
"clickhouse-client",
"--host",
"localhost",
"--port",
"9000",
"--user",
"default",
"--password",
"password",
"--query",
"SELECT 1",
]
interval: 3s
timeout: 5s
retries: 5
start_period: 10s
clickhouse_migrator:
build:
context: ../internal-packages/clickhouse
dockerfile: ./Dockerfile
depends_on:
clickhouse:
condition: service_healthy
networks:
- app_network
command: ["goose", "${GOOSE_COMMAND:-up}"]
ch-ui:
image: ghcr.io/caioricciuti/ch-ui:latest
restart: always
ports:
- "5521:5521"
environment:
VITE_CLICKHOUSE_URL: "http://clickhouse:8123"
VITE_CLICKHOUSE_USER: "default"
VITE_CLICKHOUSE_PASS: "password"
networks:
- app_network
# otel-collector:
# container_name: otel-collector
# image: otel/opentelemetry-collector-contrib:latest
docker/scripts/entrypoint.sh
+14
View File
@@ -6,7 +6,21 @@ if [ -n "$DATABASE_HOST" ]; then
fi
# Run migrations
echo "Running prisma migrations"
pnpm --filter @trigger.dev/database db:migrate:deploy
echo "Prisma migrations done"
if [ -n "$CLICKHOUSE_URL" ]; then
# Run ClickHouse migrations
echo "Running ClickHouse migrations..."
export GOOSE_DRIVER=clickhouse
export GOOSE_DBSTRING="$CLICKHOUSE_URL" # Use the full URL provided by the env var
export GOOSE_MIGRATION_DIR=/triggerdotdev/internal-packages/clickhouse/schema
/usr/local/bin/goose up
echo "ClickHouse migrations complete."
else
echo "CLICKHOUSE_URL not set, skipping ClickHouse migrations."
fi
# Copy over required prisma files
cp internal-packages/database/prisma/schema.prisma apps/webapp/prisma/
internal-packages/clickhouse/Dockerfile
+12
View File
@@ -0,0 +1,12 @@
FROM golang
RUN go install github.com/pressly/goose/v3/cmd/goose@latest
COPY ./schema ./schema
ENV GOOSE_DRIVER=clickhouse
ENV GOOSE_DBSTRING="tcp://default:password@clickhouse:9000"
ENV GOOSE_MIGRATION_DIR=./schema
CMD ["goose", "up"]
internal-packages/clickhouse/README.md
+64
View File
@@ -0,0 +1,64 @@
# ClickHouse Table Naming Conventions
The following document is heavily inspired by the [Unkey](https://unkey.dev) ClickHouse naming conventions.
This document outlines the naming conventions for tables and materialized views in our ClickHouse setup. Adhering to these conventions ensures consistency, clarity, and ease of management across our data infrastructure.
## General Rules
1. Use lowercase letters and separate words with underscores.
2. Avoid ClickHouse reserved words and special characters in names.
3. Be descriptive but concise.
## Table Naming Convention
Format: `[prefix]_[domain]_[description]_[version]`
### Prefixes
- `raw_`: Input data tables
- `tmp_{yourname}_`: Temporary tables for experiments, add your name, so it's easy to identify ownership.
### Versioning
- Version numbers: `_v1`, `_v2`, etc.
### Aggregation Suffixes
For aggregated or summary tables, use suffixes like:
- `_per_day`
- `_per_month`
- `_summary`
## Materialized View Naming Convention
Format: `[description]_[aggregation]_mv_[version]`
- Always suffix with `mv_[version]`
- Include a description of the view's purpose
- Add aggregation level if applicable
## Examples
1. Raw Data Table:
`raw_sales_transactions_v1`
2. Materialized View:
`active_users_per_day_mv_v2`
3. Temporary Table:
`tmp_eric_user_analysis_v1`
4. Aggregated Table:
`sales_summary_per_hour_mv_v1`
## Consistency Across Related Objects
Maintain consistent naming across related tables, views, and other objects:
- `raw_user_activity_v1`
- `user_activity_per_day_v1`
- `user_activity_per_day_mv_v1`
By following these conventions, we ensure a clear, consistent, and scalable naming structure for our ClickHouse setup.
internal-packages/clickhouse/package.json
+31
View File
@@ -0,0 +1,31 @@
{
"name": "@internal/clickhouse",
"private": true,
"version": "0.0.2",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
"type": "module",
"dependencies": {
"@clickhouse/client": "^1.11.1",
"@internal/tracing": "workspace:*",
"@trigger.dev/core": "workspace:*",
"zod": "3.23.8",
"zod-error": "1.5.0"
},
"devDependencies": {
"@internal/testcontainers": "workspace:*",
"@vitest/coverage-v8": "^3.0.8",
"rimraf": "6.0.1",
"vitest": "^3.0.8"
},
"scripts": {
"clean": "rimraf dist",
"typecheck": "tsc --noEmit -p tsconfig.build.json",
"build": "pnpm run clean && tsc -p tsconfig.build.json",
"dev": "tsc --watch -p tsconfig.build.json",
"db:migrate": "docker compose -p triggerdotdev-docker -f ../../docker/docker-compose.yml up clickhouse_migrator --build",
"db:migrate:down": "GOOSE_COMMAND=down pnpm run db:migrate",
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
"test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled"
}
}
internal-packages/clickhouse/schema/001_create_databases.sql
+6
View File
@@ -0,0 +1,6 @@
-- +goose up
CREATE DATABASE trigger_dev;
-- +goose down
DROP DATABASE trigger_dev;
internal-packages/clickhouse/schema/002_create_smoke_test.sql
+11
View File
@@ -0,0 +1,11 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.smoke_test (
id UUID DEFAULT generateUUIDv4(),
timestamp DateTime64(3) DEFAULT now64(3),
message String,
number UInt32
) ENGINE = MergeTree()
ORDER BY (timestamp, id);
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.smoke_test;
internal-packages/clickhouse/schema/003_create_task_runs_v1.sql
+100
View File
@@ -0,0 +1,100 @@
-- +goose Up
CREATE TABLE trigger_dev.task_runs_v1
(
/* ─── ids & hierarchy ─────────────────────────────────────── */
environment_id String,
organization_id String,
project_id String,
run_id String,
environment_type LowCardinality(String),
friendly_id String,
attempt UInt8 DEFAULT 1,
/* ─── enums / status ──────────────────────────────────────── */
engine LowCardinality(String),
status LowCardinality(String),
/* ─── queue / concurrency / schedule ─────────────────────── */
task_identifier String,
queue String,
schedule_id String,
batch_id String,
/* ─── related runs ─────────────────────────────────────────────── */
root_run_id String,
parent_run_id String,
depth UInt8 DEFAULT 0,
/* ─── telemetry ─────────────────────────────────────────────── */
span_id String,
trace_id String,
idempotency_key String,
/* ─── timing ─────────────────────────────────────────────── */
created_at DateTime64(3),
updated_at DateTime64(3),
started_at Nullable(DateTime64(3)),
executed_at Nullable(DateTime64(3)),
completed_at Nullable(DateTime64(3)),
delay_until Nullable(DateTime64(3)),
queued_at Nullable(DateTime64(3)),
expired_at Nullable(DateTime64(3)),
expiration_ttl String,
/* ─── cost / usage ───────────────────────────────────────── */
usage_duration_ms UInt32 DEFAULT 0,
cost_in_cents Float64 DEFAULT 0,
base_cost_in_cents Float64 DEFAULT 0,
/* ─── payload & context ──────────────────────────────────── */
output JSON(max_dynamic_paths = 1024),
error JSON(max_dynamic_paths = 64),
/* ─── tagging / versions ─────────────────────────────────── */
tags Array(String) CODEC(ZSTD(1)),
task_version String CODEC(LZ4),
sdk_version String CODEC(LZ4),
cli_version String CODEC(LZ4),
machine_preset LowCardinality(String) CODEC(LZ4),
is_test UInt8 DEFAULT 0,
/* ─── commit lsn ─────────────────────────────────────────────── */
_version UInt64,
_is_deleted UInt8 DEFAULT 0
)
ENGINE = ReplacingMergeTree(_version, _is_deleted)
PARTITION BY toYYYYMM(created_at)
ORDER BY (toDate(created_at), environment_id, task_identifier, created_at, run_id)
SETTINGS enable_json_type = 1;
/* Fast tag filtering */
ALTER TABLE trigger_dev.task_runs_v1
ADD INDEX idx_tags tags TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 4;
CREATE TABLE trigger_dev.raw_task_runs_payload_v1
(
run_id String,
created_at DateTime64(3),
payload JSON(max_dynamic_paths = 1024)
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (run_id)
SETTINGS enable_json_type = 1;
CREATE VIEW trigger_dev.tmp_eric_task_runs_full_v1 AS
SELECT
s.*,
p.payload as payload
FROM trigger_dev.task_runs_v1 AS s FINAL
LEFT JOIN trigger_dev.raw_task_runs_payload_v1 AS p ON s.run_id = p.run_id
SETTINGS enable_json_type = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.task_runs_v1;
DROP TABLE IF EXISTS trigger_dev.raw_task_runs_payload_v1;
DROP VIEW IF EXISTS trigger_dev.tmp_eric_task_runs_full_v1;
internal-packages/clickhouse/src/client/client.test.ts
+149
View File
@@ -0,0 +1,149 @@
import { clickhouseTest } from "@internal/testcontainers";
import { ClickhouseClient } from "./client.js";
import { z } from "zod";
import { setTimeout } from "timers/promises";
describe("ClickHouse Client", () => {
clickhouseTest("should be able to insert and query data", async ({ clickhouseContainer }) => {
const client = new ClickhouseClient({
name: "test",
url: clickhouseContainer.getConnectionUrl(),
});
const insertSmokeTest = client.insert({
name: "insert-smoke-test",
table: "trigger_dev.smoke_test",
schema: z.object({
message: z.string(),
number: z.number(),
}),
});
const querySmokeTest = client.query({
name: "query-smoke-test",
query: "SELECT * FROM trigger_dev.smoke_test",
schema: z.object({
message: z.string(),
number: z.number(),
timestamp: z.string(),
id: z.string(),
}),
});
const [insertError, insertResult] = await insertSmokeTest([
{ message: "hello", number: 42 },
{ message: "world", number: 100 },
]);
expect(insertError).toBeNull();
expect(insertResult).toEqual(
expect.objectContaining({
executed: true,
query_id: expect.any(String),
summary: expect.objectContaining({ read_rows: "2", elapsed_ns: expect.any(String) }),
})
);
const [queryError, result] = await querySmokeTest({});
expect(queryError).toBeNull();
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
message: "hello",
number: 42,
timestamp: expect.any(String),
id: expect.any(String),
}),
expect.objectContaining({
message: "world",
number: 100,
timestamp: expect.any(String),
id: expect.any(String),
}),
])
);
const insertSmokeTestAsyncWaiting = client.insert({
name: "insert-smoke-test-async-waiting",
table: "trigger_dev.smoke_test",
schema: z.object({
message: z.string(),
number: z.number(),
}),
settings: {
async_insert: 1,
wait_for_async_insert: 1,
async_insert_busy_timeout_ms: 1000,
},
});
const [insertErrorAsyncWaiting, insertResultAsyncWaiting] = await insertSmokeTestAsyncWaiting([
{ message: "async-waiting-hello", number: 42 },
{ message: "async-waiting-world", number: 100 },
]);
expect(insertErrorAsyncWaiting).toBeNull();
expect(insertResultAsyncWaiting).toEqual(expect.objectContaining({ executed: true }));
// Should be able to query for the data right away
const [queryErrorAsyncWaiting, resultAsyncWaiting] = await querySmokeTest({});
expect(queryErrorAsyncWaiting).toBeNull();
expect(resultAsyncWaiting).toEqual(
expect.arrayContaining([
expect.objectContaining({ message: "async-waiting-hello", number: 42 }),
expect.objectContaining({ message: "async-waiting-world", number: 100 }),
])
);
const insertSmokeTestAsyncDontWait = client.insert({
name: "insert-smoke-test-async-dont-wait",
table: "trigger_dev.smoke_test",
schema: z.object({
message: z.string(),
number: z.number(),
}),
settings: {
async_insert: 1,
wait_for_async_insert: 0,
async_insert_busy_timeout_ms: 1000,
},
});
const [insertErrorAsyncDontWait, insertResultAsyncDontWait] =
await insertSmokeTestAsyncDontWait([
{ message: "async-dont-wait-hello", number: 42 },
{ message: "async-dont-wait-world", number: 100 },
]);
expect(insertErrorAsyncDontWait).toBeNull();
expect(insertResultAsyncDontWait).toEqual(expect.objectContaining({ executed: true }));
// Querying now should return an array without the data
const [queryErrorAsyncDontWait, resultAsyncDontWait] = await querySmokeTest({});
expect(queryErrorAsyncDontWait).toBeNull();
expect(resultAsyncDontWait).toEqual(
expect.not.arrayContaining([
expect.objectContaining({ message: "async-dont-wait-hello", number: 42 }),
expect.objectContaining({ message: "async-dont-wait-world", number: 100 }),
])
);
// Now we wait for the data to be flushed
await setTimeout(2000);
// Querying now should return the data
const [queryErrorAsyncDontWait2, resultAsyncDontWait2] = await querySmokeTest({});
expect(queryErrorAsyncDontWait2).toBeNull();
expect(resultAsyncDontWait2).toEqual(
expect.arrayContaining([
expect.objectContaining({ message: "async-dont-wait-hello", number: 42 }),
expect.objectContaining({ message: "async-dont-wait-world", number: 100 }),
])
);
});
});
internal-packages/clickhouse/src/client/client.ts
+281
View File
@@ -0,0 +1,281 @@
import {
type ClickHouseClient,
ClickHouseError,
type ClickHouseSettings,
createClient,
} from "@clickhouse/client";
import { recordSpanError, Span, startSpan, trace, Tracer } from "@internal/tracing";
import { flattenAttributes, tryCatch } from "@trigger.dev/core/v3";
import { z } from "zod";
import { InsertError, QueryError } from "./errors.js";
import type {
ClickhouseInsertFunction,
ClickhouseQueryFunction,
ClickhouseReader,
ClickhouseWriter,
} from "./types.js";
import { generateErrorMessage } from "zod-error";
import { Logger } from "@trigger.dev/core/logger";
export type ClickhouseConfig = {
name: string;
url: string;
tracer?: Tracer;
clickhouseSettings?: ClickHouseSettings;
logger?: Logger;
};
export class ClickhouseClient implements ClickhouseReader, ClickhouseWriter {
public readonly client: ClickHouseClient;
private readonly tracer: Tracer;
private readonly name: string;
private readonly logger: Logger;
constructor(config: ClickhouseConfig) {
this.name = config.name;
this.logger = config.logger ?? new Logger("ClickhouseClient", "debug");
this.client = createClient({
url: config.url,
clickhouse_settings: {
...config.clickhouseSettings,
output_format_json_quote_64bit_integers: 0,
output_format_json_quote_64bit_floats: 0,
},
});
this.tracer = config.tracer ?? trace.getTracer("@internal/clickhouse");
}
public async close() {
await this.client.close();
}
public query<TIn extends z.ZodSchema<any>, TOut extends z.ZodSchema<any>>(req: {
/**
* The name of the operation.
* This will be used to identify the operation in the span.
*/
name: string;
/**
* The SQL query to run.
* Use {paramName: Type} to define parameters
* Example: `SELECT * FROM table WHERE id = {id: String}`
*/
query: string;
/**
* The schema of the parameters
* Example: z.object({ id: z.string() })
*/
params?: TIn;
/**
* The schema of the output of each row
* Example: z.object({ id: z.string() })
*/
schema: TOut;
/**
* The settings to use for the query.
* These will be merged with the default settings.
*/
settings?: ClickHouseSettings;
}): ClickhouseQueryFunction<z.input<TIn>, z.output<TOut>> {
return async (params, options) => {
return await startSpan(this.tracer, "query", async (span) => {
span.setAttributes({
"clickhouse.clientName": this.name,
"clickhouse.operationName": req.name,
...flattenAttributes(req.settings, "clickhouse.settings"),
...flattenAttributes(options?.attributes),
});
const validParams = req.params?.safeParse(params);
if (validParams?.error) {
recordSpanError(span, validParams.error);
this.logger.error("Error parsing query params", {
name: req.name,
error: validParams.error,
query: req.query,
params,
});
return [
new QueryError(`Bad params: ${generateErrorMessage(validParams.error.issues)}`, {
query: req.query,
}),
null,
];
}
let unparsedRows: Array<TOut> = [];
const [clickhouseError, res] = await tryCatch(
this.client.query({
query: req.query,
query_params: validParams?.data,
format: "JSONEachRow",
...options?.params,
clickhouse_settings: {
...req.settings,
...options?.params?.clickhouse_settings,
},
})
);
if (clickhouseError) {
this.logger.error("Error querying clickhouse", {
name: req.name,
error: clickhouseError,
query: req.query,
params,
});
recordClickhouseError(span, clickhouseError);
return [
new QueryError(`Unable to query clickhouse: ${clickhouseError.message}`, {
query: req.query,
}),
null,
];
}
unparsedRows = await res.json();
span.setAttributes({
"clickhouse.query_id": res.query_id,
...flattenAttributes(res.response_headers, "clickhouse.response_headers"),
});
const summaryHeader = res.response_headers["x-clickhouse-summary"];
if (typeof summaryHeader === "string") {
span.setAttributes({
...flattenAttributes(JSON.parse(summaryHeader), "clickhouse.summary"),
});
}
const parsed = z.array(req.schema).safeParse(unparsedRows);
if (parsed.error) {
this.logger.error("Error parsing clickhouse query result", {
name: req.name,
error: parsed.error,
query: req.query,
params,
});
const queryError = new QueryError(generateErrorMessage(parsed.error.issues), {
query: req.query,
});
recordSpanError(span, queryError);
return [queryError, null];
}
span.setAttributes({
"clickhouse.rows": unparsedRows.length,
});
return [null, parsed.data];
});
};
}
public insert<TSchema extends z.ZodSchema<any>>(req: {
name: string;
table: string;
schema: TSchema;
settings?: ClickHouseSettings;
}): ClickhouseInsertFunction<z.input<TSchema>> {
return async (events, options) => {
return await startSpan(this.tracer, "insert", async (span) => {
span.setAttributes({
"clickhouse.clientName": this.name,
"clickhouse.tableName": req.table,
"clickhouse.operationName": req.name,
...flattenAttributes(req.settings, "clickhouse.settings"),
...flattenAttributes(options?.attributes),
});
let validatedEvents: z.output<TSchema> | z.output<TSchema>[] | undefined = undefined;
const v = Array.isArray(events)
? req.schema.array().safeParse(events)
: req.schema.safeParse(events);
if (!v.success) {
this.logger.error("Error validating insert events", {
name: req.name,
table: req.table,
error: v.error,
});
const error = new InsertError(generateErrorMessage(v.error.issues));
recordSpanError(span, error);
return [error, null];
}
validatedEvents = v.data;
const [clickhouseError, result] = await tryCatch(
this.client.insert({
table: req.table,
format: "JSONEachRow",
values: Array.isArray(validatedEvents) ? validatedEvents : [validatedEvents],
...options?.params,
clickhouse_settings: {
...req.settings,
...options?.params?.clickhouse_settings,
},
})
);
if (clickhouseError) {
this.logger.error("Error inserting into clickhouse", {
name: req.name,
error: clickhouseError,
table: req.table,
});
recordClickhouseError(span, clickhouseError);
return [new InsertError(clickhouseError.message), null];
}
span.setAttributes({
"clickhouse.query_id": result.query_id,
"clickhouse.executed": result.executed,
"clickhouse.summary.read_rows": result.summary?.read_rows,
"clickhouse.summary.read_bytes": result.summary?.read_bytes,
"clickhouse.summary.written_rows": result.summary?.written_rows,
"clickhouse.summary.written_bytes": result.summary?.written_bytes,
"clickhouse.summary.total_rows_to_read": result.summary?.total_rows_to_read,
"clickhouse.summary.result_rows": result.summary?.result_rows,
"clickhouse.summary.result_bytes": result.summary?.result_bytes,
"clickhouse.summary.elapsed_ns": result.summary?.elapsed_ns,
});
return [null, result];
});
};
}
}
function recordClickhouseError(span: Span, error: Error) {
if (error instanceof ClickHouseError) {
span.setAttributes({
"clickhouse.error.code": error.code,
"clickhouse.error.message": error.message,
"clickhouse.error.type": error.type,
});
recordSpanError(span, error);
} else {
recordSpanError(span, error);
}
}
internal-packages/clickhouse/src/client/errors.ts
+42
View File
@@ -0,0 +1,42 @@
export type ErrorContext = Record<string, unknown>;
export abstract class BaseError<TContext extends ErrorContext = ErrorContext> extends Error {
public abstract readonly retry: boolean;
public readonly cause: BaseError | undefined;
public readonly context: TContext | undefined;
public readonly message: string;
public abstract readonly name: string;
constructor(opts: { message: string; cause?: BaseError; context?: TContext }) {
super(opts.message);
this.message = opts.message;
this.cause = opts.cause;
this.context = opts.context;
}
public toString(): string {
return `${this.name}: ${this.message} - ${JSON.stringify(
this.context
)} - caused by ${this.cause?.toString()}`;
}
}
export class InsertError extends BaseError {
public readonly retry = true;
public readonly name = InsertError.name;
constructor(message: string) {
super({
message,
});
}
}
export class QueryError extends BaseError<{ query: string }> {
public readonly retry = true;
public readonly name = QueryError.name;
constructor(message: string, context: { query: string }) {
super({
message,
context,
});
}
}
internal-packages/clickhouse/src/client/noop.ts
+66
View File
@@ -0,0 +1,66 @@
import { Result } from "@trigger.dev/core/v3";
import { InsertError, QueryError } from "./errors.js";
import { ClickhouseWriter } from "./types.js";
import { ClickhouseReader } from "./types.js";
import { z } from "zod";
import { ClickHouseSettings, InsertResult } from "@clickhouse/client";
export class NoopClient implements ClickhouseReader, ClickhouseWriter {
public async close() {
return;
}
public query<TIn extends z.ZodSchema<any>, TOut extends z.ZodSchema<any>>(req: {
query: string;
params?: TIn;
schema: TOut;
}): (params: z.input<TIn>) => Promise<Result<z.output<TOut>[], QueryError>> {
return async (params: z.input<TIn>) => {
const validParams = req.params?.safeParse(params);
if (validParams?.error) {
return [new QueryError(`Bad params: ${validParams.error.message}`, { query: "" }), null];
}
return [null, []];
};
}
public insert<TSchema extends z.ZodSchema<any>>(req: {
name: string;
table: string;
schema: TSchema;
settings?: ClickHouseSettings;
}): (
events: z.input<TSchema> | z.input<TSchema>[]
) => Promise<Result<InsertResult, InsertError>> {
return async (events: z.input<TSchema> | z.input<TSchema>[]) => {
const v = Array.isArray(events)
? req.schema.array().safeParse(events)
: req.schema.safeParse(events);
if (!v.success) {
return [new InsertError(v.error.message), null];
}
return [
null,
{
executed: true,
query_id: "noop",
summary: {
read_rows: "0",
read_bytes: "0",
written_rows: "0",
written_bytes: "0",
total_rows_to_read: "0",
result_rows: "0",
result_bytes: "0",
elapsed_ns: "0",
},
response_headers: {},
},
];
};
}
}
internal-packages/clickhouse/src/client/types.ts
+65
View File
@@ -0,0 +1,65 @@
import type { Result } from "@trigger.dev/core/v3";
import type { z } from "zod";
import type { InsertError, QueryError } from "./errors.js";
import { ClickHouseSettings } from "@clickhouse/client";
import type { BaseQueryParams, InsertResult } from "@clickhouse/client";
export type ClickhouseQueryFunction<TInput, TOutput> = (
params: TInput,
options?: {
attributes?: Record<string, string | number | boolean>;
params?: BaseQueryParams;
}
) => Promise<Result<TOutput[], QueryError>>;
export interface ClickhouseReader {
query<TIn extends z.ZodSchema<any>, TOut extends z.ZodSchema<any>>(req: {
/**
* The name of the operation.
* This will be used to identify the operation in the span.
*/
name: string;
/**
* The SQL query to run.
* Use {paramName: Type} to define parameters
* Example: `SELECT * FROM table WHERE id = {id: String}`
*/
query: string;
/**
* The schema of the parameters
* Example: z.object({ id: z.string() })
*/
params?: TIn;
/**
* The schema of the output of each row
* Example: z.object({ id: z.string() })
*/
schema: TOut;
/**
* The settings to use for the query.
* These will be merged with the default settings.
*/
settings?: ClickHouseSettings;
}): ClickhouseQueryFunction<z.input<TIn>, z.output<TOut>>;
close(): Promise<void>;
}
export type ClickhouseInsertFunction<TInput> = (
events: TInput | TInput[],
options?: {
attributes?: Record<string, string | number | boolean>;
params?: BaseQueryParams;
}
) => Promise<Result<InsertResult, InsertError>>;
export interface ClickhouseWriter {
insert<TSchema extends z.ZodSchema<any>>(req: {
name: string;
table: string;
schema: TSchema;
settings?: ClickHouseSettings;
}): ClickhouseInsertFunction<z.input<TSchema>>;
close(): Promise<void>;
}
internal-packages/clickhouse/src/index.ts
+111
View File
@@ -0,0 +1,111 @@
import { ClickHouseSettings } from "@clickhouse/client";
import { ClickhouseClient } from "./client/client.js";
import { ClickhouseReader, ClickhouseWriter } from "./client/types.js";
import { NoopClient } from "./client/noop.js";
import { insertTaskRuns, insertRawTaskRunPayloads } from "./taskRuns.js";
import { Logger } from "@trigger.dev/core/logger";
export type * from "./taskRuns.js";
export type ClickHouseConfig =
| {
name?: string;
url?: string;
writerUrl?: never;
readerUrl?: never;
clickhouseSettings?: ClickHouseSettings;
logger?: Logger;
}
| {
name?: never;
url?: never;
writerName?: string;
writerUrl: string;
readerName?: string;
readerUrl: string;
clickhouseSettings?: ClickHouseSettings;
logger?: Logger;
};
export class ClickHouse {
public readonly reader: ClickhouseReader;
public readonly writer: ClickhouseWriter;
private readonly logger: Logger;
private _splitClients: boolean;
constructor(config: ClickHouseConfig) {
this.logger = config.logger ?? new Logger("ClickHouse", "debug");
if (config.url) {
const url = new URL(config.url);
url.password = "redacted";
this.logger.info("🏠 Initializing ClickHouse client with url", { url: url.toString() });
const client = new ClickhouseClient({
name: config.name ?? "clickhouse",
url: config.url,
clickhouseSettings: config.clickhouseSettings,
logger: this.logger,
});
this.reader = client;
this.writer = client;
this._splitClients = false;
} else if (config.writerUrl && config.readerUrl) {
this.reader = new ClickhouseClient({
name: config.readerName ?? "clickhouse-reader",
url: config.readerUrl,
clickhouseSettings: config.clickhouseSettings,
logger: this.logger,
});
this.writer = new ClickhouseClient({
name: config.writerName ?? "clickhouse-writer",
url: config.writerUrl,
clickhouseSettings: config.clickhouseSettings,
logger: this.logger,
});
this._splitClients = true;
} else {
this.reader = new NoopClient();
this.writer = new NoopClient();
this._splitClients = true;
}
}
static fromEnv(): ClickHouse {
if (
typeof process.env.CLICKHOUSE_WRITER_URL === "string" &&
typeof process.env.CLICKHOUSE_READER_URL === "string"
) {
return new ClickHouse({
writerUrl: process.env.CLICKHOUSE_WRITER_URL,
readerUrl: process.env.CLICKHOUSE_READER_URL,
writerName: process.env.CLICKHOUSE_WRITER_NAME,
readerName: process.env.CLICKHOUSE_READER_NAME,
});
}
return new ClickHouse({
url: process.env.CLICKHOUSE_URL,
name: process.env.CLICKHOUSE_NAME,
});
}
async close() {
if (this._splitClients) {
await Promise.all([this.reader.close(), this.writer.close()]);
} else {
await this.reader.close();
}
}
get taskRuns() {
return {
insert: insertTaskRuns(this.writer),
insertPayloads: insertRawTaskRunPayloads(this.writer),
};
}
}
internal-packages/clickhouse/src/taskRuns.test.ts
+253
View File
@@ -0,0 +1,253 @@
import { clickhouseTest } from "@internal/testcontainers";
import { z } from "zod";
import { ClickhouseClient } from "./client/client.js";
import { insertRawTaskRunPayloads, insertTaskRuns } from "./taskRuns.js";
describe("Task Runs V1", () => {
clickhouseTest("should be able to insert task runs", async ({ clickhouseContainer }) => {
const client = new ClickhouseClient({
name: "test",
url: clickhouseContainer.getConnectionUrl(),
});
const insert = insertTaskRuns(client, {
async_insert: 0, // turn off async insert for this test
});
const insertPayloads = insertRawTaskRunPayloads(client, {
async_insert: 0, // turn off async insert for this test
});
const [insertError, insertResult] = await insert([
{
environment_id: "env_1234",
environment_type: "DEVELOPMENT",
organization_id: "org_1234",
project_id: "project_1234",
run_id: "run_1234",
friendly_id: "friendly_1234",
attempt: 1,
engine: "V2",
status: "PENDING",
task_identifier: "my-task",
queue: "my-queue",
schedule_id: "schedule_1234",
batch_id: "batch_1234",
created_at: Date.now(),
updated_at: Date.now(),
completed_at: undefined,
tags: ["tag1", "tag2"],
output: {
key: "value",
},
error: {
type: "BUILT_IN_ERROR",
name: "Error",
message: "error",
stackTrace: "stack trace",
},
usage_duration_ms: 1000,
cost_in_cents: 100,
task_version: "1.0.0",
sdk_version: "1.0.0",
cli_version: "1.0.0",
machine_preset: "small-1x",
is_test: true,
span_id: "span_1234",
trace_id: "trace_1234",
idempotency_key: "idempotency_key_1234",
expiration_ttl: "1h",
root_run_id: "root_run_1234",
parent_run_id: "parent_run_1234",
depth: 1,
_version: "1",
},
]);
expect(insertError).toBeNull();
expect(insertResult).toEqual(expect.objectContaining({ executed: true }));
expect(insertResult?.summary?.written_rows).toEqual("1");
const query = client.query({
name: "query-task-runs",
query: "SELECT * FROM trigger_dev.task_runs_v1",
schema: z.object({
environment_id: z.string(),
run_id: z.string(),
}),
params: z.object({
run_id: z.string(),
}),
});
const [queryError, result] = await query({ run_id: "run_1234" });
expect(queryError).toBeNull();
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
environment_id: "env_1234",
run_id: "run_1234",
}),
])
);
const [insertPayloadsError, insertPayloadsResult] = await insertPayloads([
{
run_id: "run_1234",
created_at: Date.now(),
payload: {
key: "value",
},
},
]);
expect(insertPayloadsError).toBeNull();
expect(insertPayloadsResult).toEqual(expect.objectContaining({ executed: true }));
expect(insertPayloadsResult?.summary?.written_rows).toEqual("1");
const queryPayloads = client.query({
name: "query-raw-task-run-payloads",
query: "SELECT * FROM trigger_dev.raw_task_runs_payload_v1",
schema: z.object({
run_id: z.string(),
created_at: z.coerce.date(),
payload: z.unknown(),
}),
});
const [queryPayloadsError, resultPayloads] = await queryPayloads({ run_id: "run_1234" });
expect(queryPayloadsError).toBeNull();
expect(resultPayloads).toEqual(
expect.arrayContaining([expect.objectContaining({ run_id: "run_1234" })])
);
});
clickhouseTest("should deduplicate on the _version column", async ({ clickhouseContainer }) => {
const client = new ClickhouseClient({
name: "test",
url: clickhouseContainer.getConnectionUrl(),
});
const insert = insertTaskRuns(client, {
async_insert: 0, // turn off async insert for this test
});
const [insertError, insertResult] = await insert([
{
environment_id: "cm9kddfcs01zqdy88ld9mmrli",
organization_id: "cm8zs78wb0002dy616dg75tv3",
project_id: "cm9kddfbz01zpdy88t9dstecu",
run_id: "cma45oli70002qrdy47w0j4n7",
environment_type: "PRODUCTION",
friendly_id: "run_cma45oli70002qrdy47w0j4n7",
attempt: 1,
engine: "V2",
status: "PENDING",
task_identifier: "retry-task",
queue: "task/retry-task",
schedule_id: "",
batch_id: "",
root_run_id: "",
parent_run_id: "",
depth: 0,
span_id: "538677637f937f54",
trace_id: "20a28486b0b9f50c647b35e8863e36a5",
idempotency_key: "",
created_at: new Date("2025-04-30 16:34:04.312").getTime(),
updated_at: new Date("2025-04-30 16:34:04.312").getTime(),
started_at: null,
executed_at: null,
completed_at: null,
delay_until: null,
queued_at: new Date("2025-04-30 16:34:04.311").getTime(),
expired_at: null,
expiration_ttl: "",
usage_duration_ms: 0,
cost_in_cents: 0,
base_cost_in_cents: 0,
output: null,
error: null,
tags: [],
task_version: "",
sdk_version: "",
cli_version: "",
machine_preset: "",
is_test: true,
_version: "1",
},
{
environment_id: "cm9kddfcs01zqdy88ld9mmrli",
organization_id: "cm8zs78wb0002dy616dg75tv3",
project_id: "cm9kddfbz01zpdy88t9dstecu",
run_id: "cma45oli70002qrdy47w0j4n7",
environment_type: "PRODUCTION",
friendly_id: "run_cma45oli70002qrdy47w0j4n7",
attempt: 1,
engine: "V2",
status: "COMPLETED_SUCCESSFULLY",
task_identifier: "retry-task",
queue: "task/retry-task",
schedule_id: "",
batch_id: "",
root_run_id: "",
parent_run_id: "",
depth: 0,
span_id: "538677637f937f54",
trace_id: "20a28486b0b9f50c647b35e8863e36a5",
idempotency_key: "",
created_at: new Date("2025-04-30 16:34:04.312").getTime(),
updated_at: new Date("2025-04-30 16:34:04.312").getTime(),
started_at: null,
executed_at: null,
completed_at: null,
delay_until: null,
queued_at: new Date("2025-04-30 16:34:04.311").getTime(),
expired_at: null,
expiration_ttl: "",
usage_duration_ms: 0,
cost_in_cents: 0,
base_cost_in_cents: 0,
output: null,
error: null,
tags: [],
task_version: "",
sdk_version: "",
cli_version: "",
machine_preset: "",
is_test: true,
_version: "2",
},
]);
expect(insertError).toBeNull();
expect(insertResult).toEqual(expect.objectContaining({ executed: true }));
const query = client.query({
name: "query-task-runs",
query: "SELECT * FROM trigger_dev.task_runs_v1 FINAL",
schema: z.object({
environment_id: z.string(),
run_id: z.string(),
status: z.string(),
}),
params: z.object({
run_id: z.string(),
}),
});
const [queryError, result] = await query({ run_id: "cma45oli70002qrdy47w0j4n7" });
expect(queryError).toBeNull();
expect(result).toEqual(
expect.arrayContaining([
expect.objectContaining({
environment_id: "cm9kddfcs01zqdy88ld9mmrli",
run_id: "cma45oli70002qrdy47w0j4n7",
status: "COMPLETED_SUCCESSFULLY",
}),
])
);
});
});
internal-packages/clickhouse/src/taskRuns.ts
+89
View File
@@ -0,0 +1,89 @@
import { ClickHouseSettings } from "@clickhouse/client";
import { z } from "zod";
import { ClickhouseWriter } from "./client/types.js";
export const TaskRunV1 = z.object({
environment_id: z.string(),
organization_id: z.string(),
project_id: z.string(),
run_id: z.string(),
updated_at: z.number().int(),
created_at: z.number().int(),
status: z.string(),
environment_type: z.string(),
friendly_id: z.string(),
attempt: z.number().int().default(1),
engine: z.string(),
task_identifier: z.string(),
queue: z.string(),
schedule_id: z.string(),
batch_id: z.string(),
completed_at: z.number().int().nullish(),
started_at: z.number().int().nullish(),
executed_at: z.number().int().nullish(),
delay_until: z.number().int().nullish(),
queued_at: z.number().int().nullish(),
expired_at: z.number().int().nullish(),
usage_duration_ms: z.number().int().default(0),
cost_in_cents: z.number().default(0),
base_cost_in_cents: z.number().default(0),
output: z.unknown(),
error: z.unknown(),
tags: z.array(z.string()).default([]),
task_version: z.string(),
sdk_version: z.string(),
cli_version: z.string(),
machine_preset: z.string(),
root_run_id: z.string(),
parent_run_id: z.string(),
depth: z.number().int().default(0),
span_id: z.string(),
trace_id: z.string(),
idempotency_key: z.string(),
expiration_ttl: z.string(),
is_test: z.boolean().default(false),
_version: z.string(),
_is_deleted: z.number().int().default(0),
});
export type TaskRunV1 = z.input<typeof TaskRunV1>;
export function insertTaskRuns(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
return ch.insert({
name: "insertTaskRuns",
table: "trigger_dev.task_runs_v1",
schema: TaskRunV1,
settings: {
async_insert: 1,
wait_for_async_insert: 0,
async_insert_max_data_size: "1000000",
async_insert_busy_timeout_ms: 1000,
enable_json_type: 1,
...settings,
},
});
}
export const RawTaskRunPayloadV1 = z.object({
run_id: z.string(),
created_at: z.number().int(),
payload: z.unknown(),
});
export type RawTaskRunPayloadV1 = z.infer<typeof RawTaskRunPayloadV1>;
export function insertRawTaskRunPayloads(ch: ClickhouseWriter, settings?: ClickHouseSettings) {
return ch.insert({
name: "insertRawTaskRunPayloads",
table: "trigger_dev.raw_task_runs_payload_v1",
schema: RawTaskRunPayloadV1,
settings: {
async_insert: 1,
wait_for_async_insert: 0,
async_insert_max_data_size: "1000000",
async_insert_busy_timeout_ms: 1000,
enable_json_type: 1,
...settings,
},
});
}
internal-packages/clickhouse/tsconfig.build.json
+21
View File
@@ -0,0 +1,21 @@
{
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts"],
"compilerOptions": {
"composite": true,
"target": "ES2019",
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"outDir": "dist",
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"declaration": true
}
}
internal-packages/clickhouse/tsconfig.json
+8
View File
@@ -0,0 +1,8 @@
{
"references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
"compilerOptions": {
"moduleResolution": "Node16",
"module": "Node16",
"customConditions": ["@triggerdotdev/source"]
}
}
internal-packages/clickhouse/tsconfig.src.json
+20
View File
@@ -0,0 +1,20 @@
{
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "src/**/*.test.ts"],
"compilerOptions": {
"composite": true,
"target": "ES2019",
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"customConditions": ["@triggerdotdev/source"]
}
}
internal-packages/clickhouse/tsconfig.test.json
+21
View File
@@ -0,0 +1,21 @@
{
"include": ["src/**/*.test.ts", "vitest.config.ts"],
"references": [{ "path": "./tsconfig.src.json" }],
"compilerOptions": {
"composite": true,
"target": "ES2019",
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"types": ["vitest/globals"],
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"customConditions": ["@triggerdotdev/source"]
}
}
internal-packages/clickhouse/vitest.config.ts
+19
View File
@@ -0,0 +1,19 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
include: ["**/*.test.ts"],
globals: true,
isolate: true,
fileParallelism: false,
poolOptions: {
threads: {
singleThread: true,
},
},
testTimeout: 60_000,
coverage: {
provider: "v8",
},
},
});
internal-packages/database/prisma/migrations/20250428211853_add_environment_type_and_org_id_to_task_run/migration.sql
+4
View File
@@ -0,0 +1,4 @@
-- AlterTable
ALTER TABLE "TaskRun" ADD COLUMN "environmentType" "RuntimeEnvironmentType",
ADD COLUMN "organizationId" TEXT;
internal-packages/database/prisma/schema.prisma
+4
View File
@@ -1731,9 +1731,13 @@ model TaskRun {
runtimeEnvironment RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
runtimeEnvironmentId String
environmentType RuntimeEnvironmentType?
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade)
projectId String
organizationId String?
// The specific queue this run is in
queue String
// The queueId is set when the run is locked to a specific queue
internal-packages/replication/README.md
+1
View File
@@ -0,0 +1 @@
# Replication
internal-packages/replication/package.json
+30
View File
@@ -0,0 +1,30 @@
{
"name": "@internal/replication",
"private": true,
"version": "0.0.1",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
"type": "module",
"dependencies": {
"@internal/redis": "workspace:*",
"@internal/tracing": "workspace:*",
"@trigger.dev/core": "workspace:*",
"pg": "8.15.6",
"redlock": "5.0.0-beta.2"
},
"devDependencies": {
"@internal/testcontainers": "workspace:*",
"@vitest/coverage-v8": "^3.0.8",
"rimraf": "6.0.1",
"vitest": "^3.0.8",
"@types/pg": "8.11.14"
},
"scripts": {
"clean": "rimraf dist",
"typecheck": "tsc --noEmit -p tsconfig.build.json",
"build": "pnpm run clean && tsc -p tsconfig.build.json",
"dev": "tsc --watch -p tsconfig.build.json",
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
"test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled"
}
}
internal-packages/replication/src/client.test.ts
+184
View File
@@ -0,0 +1,184 @@
import { postgresAndRedisTest } from "@internal/testcontainers";
import { LogicalReplicationClient } from "./client.js";
import { setTimeout } from "timers/promises";
describe("Replication Client", () => {
postgresAndRedisTest(
"should be able to subscribe to changes on a table",
async ({ postgresContainer, prisma, redisOptions }) => {
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
const client = new LogicalReplicationClient({
name: "test",
slotName: "test_slot",
publicationName: "test_publication",
redisOptions,
table: "TaskRun",
pgConfig: {
connectionString: postgresContainer.getConnectionUri(),
},
});
const logs: Array<{
lsn: string;
log: unknown;
}> = [];
client.events.on("data", (data) => {
console.log(data);
logs.push(data);
});
client.events.on("error", (error) => {
console.error(error);
});
await client.subscribe();
const organization = await prisma.organization.create({
data: {
title: "test",
slug: "test",
},
});
const project = await prisma.project.create({
data: {
name: "test",
slug: "test",
organizationId: organization.id,
externalRef: "test",
},
});
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
data: {
slug: "test",
type: "DEVELOPMENT",
projectId: project.id,
organizationId: organization.id,
apiKey: "test",
pkApiKey: "test",
shortcode: "test",
},
});
// Now we insert a row into the table
await prisma.taskRun.create({
data: {
friendlyId: "run_1234",
taskIdentifier: "my-task",
payload: JSON.stringify({ foo: "bar" }),
traceId: "1234",
spanId: "1234",
queue: "test",
runtimeEnvironmentId: runtimeEnvironment.id,
projectId: project.id,
},
});
// Wait for a bit of time
await setTimeout(50);
// Now we should see the row in the logs
expect(logs.length).toBeGreaterThan(0);
await client.stop();
}
);
postgresAndRedisTest(
"should be able to teardown",
async ({ postgresContainer, prisma, redisOptions }) => {
await prisma.$executeRawUnsafe(`ALTER TABLE public."TaskRun" REPLICA IDENTITY FULL;`);
const client = new LogicalReplicationClient({
name: "test",
slotName: "test_slot",
publicationName: "test_publication",
redisOptions,
table: "TaskRun",
pgConfig: {
connectionString: postgresContainer.getConnectionUri(),
},
});
const logs: Array<{
lsn: string;
log: unknown;
}> = [];
client.events.on("data", (data) => {
console.log(data);
logs.push(data);
});
client.events.on("error", (error) => {
console.error(error);
});
await client.subscribe();
const organization = await prisma.organization.create({
data: {
title: "test",
slug: "test",
},
});
const project = await prisma.project.create({
data: {
name: "test",
slug: "test",
organizationId: organization.id,
externalRef: "test",
},
});
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
data: {
slug: "test",
type: "DEVELOPMENT",
projectId: project.id,
organizationId: organization.id,
apiKey: "test",
pkApiKey: "test",
shortcode: "test",
},
});
// Now we insert a row into the table
await prisma.taskRun.create({
data: {
friendlyId: "run_1234",
taskIdentifier: "my-task",
payload: JSON.stringify({ foo: "bar" }),
traceId: "1234",
spanId: "1234",
queue: "test",
runtimeEnvironmentId: runtimeEnvironment.id,
projectId: project.id,
},
});
// Wait for a bit of time
await setTimeout(50);
// Now we should see the row in the logs
expect(logs.length).toBeGreaterThan(0);
const slotDropped = await client.teardown();
expect(slotDropped).toBe(true);
// Now the replication slot should be gone
const slotExists = await prisma.$queryRaw<
{ exists: boolean }[]
>`SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'test_slot');`;
console.log(slotExists);
expect(slotExists[0].exists).toBe(false);
}
);
});
internal-packages/replication/src/client.ts
+662
View File
@@ -0,0 +1,662 @@
import { tryCatch } from "@trigger.dev/core/utils";
import { Redis, type RedisOptions } from "@internal/redis";
import EventEmitter from "node:events";
import { Client, ClientConfig, Connection } from "pg";
import Redlock, { Lock } from "redlock";
import { createRedisClient } from "@internal/redis";
import { Logger } from "@trigger.dev/core/logger";
import { LogicalReplicationClientError } from "./errors.js";
import { PgoutputMessage, PgoutputParser, getPgoutputStartReplicationSQL } from "./pgoutput.js";
import { startSpan, trace, Tracer } from "@internal/tracing";
export interface LogicalReplicationClientOptions {
/**
* The pg client config.
*/
pgConfig: ClientConfig;
/**
* The name of this LogicalReplicationClient instance, used for leader election.
*/
name: string;
/**
* The table to replicate (for publication creation).
*/
table: string;
/**
* The name of the replication slot to use.
*/
slotName: string;
/**
* The name of the publication to use.
*/
publicationName: string;
/**
* A connected Redis client instance for Redlock.
*/
redisOptions: RedisOptions;
/**
* Whether to automatically acknowledge messages.
*/
autoAcknowledge?: boolean;
/**
* A logger instance for logging.
*/
logger?: Logger;
/**
* The initial leader lock timeout in ms (default: 30000)
*/
leaderLockTimeoutMs?: number;
/**
* The interval in ms to extend the leader lock (default: 10000)
*/
leaderLockExtendIntervalMs?: number;
/**
* The number of times to retry acquiring the leader lock (default: 120)
*/
leaderLockRetryCount?: number;
/**
* The interval in ms to retry acquiring the leader lock (default: 500)
*/
leaderLockRetryIntervalMs?: number;
/**
* The interval in seconds to automatically acknowledge the last LSN if no ack has been sent (default: 10)
*/
ackIntervalSeconds?: number;
/**
* The actions to publish to the publication.
*/
publicationActions?: Array<"insert" | "update" | "delete" | "truncate">;
tracer?: Tracer;
}
export type LogicalReplicationClientEvents = {
leaderElection: [boolean];
error: [Error];
data: [{ lsn: string; log: PgoutputMessage; parseDuration: bigint }];
start: [];
acknowledge: [{ lsn: string }];
heartbeat: [{ lsn: string; timestamp: number; shouldRespond: boolean }];
};
export class LogicalReplicationClient {
private readonly options: LogicalReplicationClientOptions;
private client: Client | null = null;
private connection: Connection | null = null;
private redis: Redis;
private redlock: Redlock;
private leaderLock: Lock | null = null;
public readonly events: EventEmitter<LogicalReplicationClientEvents>;
private logger: Logger;
private autoAcknowledge: boolean;
private lastAcknowledgedLsn: string | null = null;
private leaderLockTimeoutMs: number;
private leaderLockExtendIntervalMs: number;
private leaderLockRetryCount: number;
private leaderLockRetryIntervalMs: number;
private leaderLockHeartbeatTimer: NodeJS.Timeout | null = null;
private ackIntervalSeconds: number;
private lastAckTimestamp: number = 0;
private ackIntervalTimer: NodeJS.Timeout | null = null;
private _isStopped: boolean = false;
private _tracer: Tracer;
public get lastLsn(): string {
return this.lastAcknowledgedLsn ?? "0/00000000";
}
public get isStopped(): boolean {
return this._isStopped;
}
constructor(options: LogicalReplicationClientOptions) {
this.options = options;
this.logger = options.logger ?? new Logger("LogicalReplicationClient", "info");
this._tracer = options.tracer ?? trace.getTracer("logical-replication-client");
this.autoAcknowledge =
typeof options.autoAcknowledge === "boolean" ? options.autoAcknowledge : true;
this.leaderLockTimeoutMs = options.leaderLockTimeoutMs ?? 30000;
this.leaderLockExtendIntervalMs = options.leaderLockExtendIntervalMs ?? 10000;
this.leaderLockRetryCount = options.leaderLockRetryCount ?? 120;
this.leaderLockRetryIntervalMs = options.leaderLockRetryIntervalMs ?? 500;
this.ackIntervalSeconds = options.ackIntervalSeconds ?? 10;
this.redis = createRedisClient(
{
...options.redisOptions,
keyPrefix: `${options.redisOptions.keyPrefix}logical-replication-client:`,
},
{
onError: (error) => {
this.logger.error(`RunLock redis client error:`, {
error,
keyPrefix: options.redisOptions.keyPrefix,
});
},
}
);
this.redlock = new Redlock([this.redis], {
retryCount: 0,
});
this.events = new EventEmitter<LogicalReplicationClientEvents>();
}
public async stop(): Promise<this> {
return await startSpan(this._tracer, "logical_replication_client.stop", async (span) => {
if (this._isStopped) return this;
span.setAttribute("replication_client.name", this.options.name);
span.setAttribute("replication_client.table", this.options.table);
span.setAttribute("replication_client.slot_name", this.options.slotName);
span.setAttribute("replication_client.publication_name", this.options.publicationName);
this._isStopped = true;
// Clean up leader lock heartbeat
if (this.leaderLockHeartbeatTimer) {
clearInterval(this.leaderLockHeartbeatTimer);
this.leaderLockHeartbeatTimer = null;
}
// Clean up ack interval
if (this.ackIntervalTimer) {
clearInterval(this.ackIntervalTimer);
this.ackIntervalTimer = null;
}
// Release leader lock if held
await this.#releaseLeaderLock();
this.connection?.removeAllListeners();
this.connection = null;
if (this.client) {
this.client.removeAllListeners();
const [endError] = await tryCatch(this.client.end());
if (endError) {
this.logger.error("Failed to end client", {
name: this.options.name,
error: endError,
});
} else {
this.logger.info("Ended client", {
name: this.options.name,
});
}
this.client = null;
}
// clear any intervals
if (this.leaderLockHeartbeatTimer) {
clearInterval(this.leaderLockHeartbeatTimer);
this.leaderLockHeartbeatTimer = null;
}
if (this.ackIntervalTimer) {
clearInterval(this.ackIntervalTimer);
this.ackIntervalTimer = null;
}
return this;
});
}
public async teardown(): Promise<boolean> {
await this.stop();
// Acquire the leaderLock
const leaderLockAcquired = await this.#acquireLeaderLock();
if (!leaderLockAcquired) {
return false;
}
this.client = new Client({
...this.options.pgConfig,
// @ts-expect-error
replication: "database",
application_name: this.options.name,
});
await this.client.connect();
// Drop the slot
const slotDropped = await this.#dropSlot();
await this.client.end();
this.client = null;
await this.#releaseLeaderLock();
return slotDropped;
}
public async subscribe(startLsn?: string): Promise<this> {
await this.stop();
this.lastAcknowledgedLsn = startLsn ?? this.lastAcknowledgedLsn;
this.logger.info("Subscribing to logical replication", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
startLsn,
});
// 1. Leader election
const leaderLockAcquired = await this.#acquireLeaderLock();
if (!leaderLockAcquired) {
this.events.emit("leaderElection", false);
return this.stop();
}
this.events.emit("leaderElection", true);
this.logger.info("Leader election successful", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
startLsn,
});
// Start leader lock heartbeat
this.#startLeaderLockHeartbeat();
// Start auto-acknowledge interval
this.#startAckInterval();
// 2. Connect pg client
this.client = new Client({
...this.options.pgConfig,
// @ts-expect-error
replication: "database",
application_name: this.options.name,
});
await this.client.connect();
// @ts-ignore
this.connection = this.client.connection;
const publicationCreated = await this.#createPublication();
if (!publicationCreated) {
return this.stop();
}
this.logger.info("Publication created", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
startLsn,
});
const slotCreated = await this.#createSlot();
if (!slotCreated) {
return this.stop();
}
this.logger.info("Slot created", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
startLsn,
});
// 5. Start replication (pgoutput)
const parser = new PgoutputParser();
const sql = getPgoutputStartReplicationSQL(this.options.slotName, this.lastLsn, {
protoVersion: 1,
publicationNames: [this.options.publicationName],
messages: false,
});
// 6. Listen for replication events (copyData, etc.)
if (!this.connection) {
this.events.emit(
"error",
new LogicalReplicationClientError("No connection after starting replication")
);
return this.stop();
}
this.connection.once("replicationStart", () => {
this._isStopped = false;
this.events.emit("start");
});
this.connection.on(
"copyData",
async ({ chunk: buffer }: { length: number; chunk: Buffer; name: string }) => {
// pgoutput protocol: 0x77 = XLogData, 0x6b = Primary keepalive
if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {
this.logger.warn("Unknown replication message type", { byte: buffer[0] });
return;
}
const lsn =
buffer.readUInt32BE(1).toString(16).toUpperCase() +
"/" +
buffer.readUInt32BE(5).toString(16).toUpperCase();
if (buffer[0] === 0x77) {
// XLogData
try {
const start = process.hrtime.bigint();
const log = parser.parse(buffer.subarray(25));
const duration = process.hrtime.bigint() - start;
this.events.emit("data", { lsn, log, parseDuration: duration });
await this.#acknowledge(lsn);
} catch (err) {
this.logger.error("Failed to parse XLogData", { error: err });
this.events.emit("error", err instanceof Error ? err : new Error(String(err)));
}
} else if (buffer[0] === 0x6b) {
// Primary keepalive message
const timestamp = Math.floor(
buffer.readUInt32BE(9) * 4294967.296 + buffer.readUInt32BE(13) / 1000 + 946080000000
);
const shouldRespond = !!buffer.readInt8(17);
this.events.emit("heartbeat", { lsn, timestamp, shouldRespond });
if (shouldRespond) {
await this.#acknowledge(lsn);
}
}
this.lastAcknowledgedLsn = lsn;
}
);
// 7. Handle errors and cleanup
this.client.on("error", (err) => {
this.events.emit("error", err);
});
this.logger.info("Started replication", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
startLsn,
sql: sql.replace(/\s+/g, " "),
});
// Start the replication stream
this.client.query(sql).catch((err) => {
this.logger.error("Failed to start replication", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
error: err,
});
this.events.emit("error", err);
return this.stop();
});
return this;
}
async #createPublication(): Promise<boolean> {
if (!this.client) {
this.events.emit("error", new LogicalReplicationClientError("Client not connected"));
return false;
}
if (await this.#doesPublicationExist()) {
return true;
}
const [createError] = await tryCatch(
this.client.query(
`CREATE PUBLICATION "${this.options.publicationName}" FOR TABLE "${this.options.table}" ${
this.options.publicationActions
? `WITH (publish = '${this.options.publicationActions.join(", ")}')`
: ""
};`
)
);
if (createError) {
this.logger.error("Failed to create publication", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
error: createError,
});
this.events.emit("error", createError);
return false;
}
return true;
}
async #doesPublicationExist(): Promise<boolean> {
if (!this.client) {
this.events.emit(
"error",
new LogicalReplicationClientError("Cannot check if publication exists")
);
return false;
}
const res = await this.client.query(
`SELECT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = '${this.options.publicationName}');`
);
return res.rows[0].exists;
}
async #createSlot(): Promise<boolean> {
if (!this.client) {
this.events.emit("error", new LogicalReplicationClientError("Cannot create slot"));
return false;
}
if (await this.#doesSlotExist()) {
return true;
}
const [createError] = await tryCatch(
this.client.query(
`SELECT * FROM pg_create_logical_replication_slot('${this.options.slotName}', 'pgoutput')`
)
);
if (createError) {
this.logger.error("Failed to create slot", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
error: createError,
});
this.events.emit("error", createError);
return false;
}
return true;
}
async #doesSlotExist(): Promise<boolean> {
if (!this.client) {
this.events.emit("error", new LogicalReplicationClientError("Cannot check if slot exists"));
return false;
}
const res = await this.client.query(
`SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = '${this.options.slotName}');`
);
return res.rows[0].exists;
}
async #dropSlot(): Promise<boolean> {
if (!this.client) {
this.events.emit("error", new LogicalReplicationClientError("Cannot drop slot"));
return false;
}
const [dropError] = await tryCatch(
this.client.query(`SELECT pg_drop_replication_slot('${this.options.slotName}');`)
);
if (dropError) {
this.logger.error("Failed to drop slot", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
error: dropError,
});
this.events.emit("error", dropError);
}
return true;
}
async #acknowledge(lsn: string): Promise<void> {
if (!this.autoAcknowledge) return;
this.events.emit("acknowledge", { lsn });
await this.acknowledge(lsn);
}
public async acknowledge(lsn: string): Promise<boolean> {
if (this._isStopped) return false;
if (!this.connection) return false;
return await startSpan(this._tracer, "logical_replication_client.acknowledge", async (span) => {
span.setAttribute("replication_client.lsn", lsn);
span.setAttribute("replication_client.name", this.options.name);
span.setAttribute("replication_client.table", this.options.table);
span.setAttribute("replication_client.slot_name", this.options.slotName);
span.setAttribute("replication_client.publication_name", this.options.publicationName);
// WAL LSN split
const slice = lsn.split("/");
let [upperWAL, lowerWAL]: [number, number] = [parseInt(slice[0], 16), parseInt(slice[1], 16)];
// Timestamp as microseconds since midnight 2000-01-01
const now = Date.now() - 946080000000;
const upperTimestamp = Math.floor(now / 4294967.296);
const lowerTimestamp = Math.floor(now - upperTimestamp * 4294967.296);
if (lowerWAL === 4294967295) {
upperWAL = upperWAL + 1;
lowerWAL = 0;
} else {
lowerWAL = lowerWAL + 1;
}
const response = Buffer.alloc(34);
response.fill(0x72); // 'r'
response.writeUInt32BE(upperWAL, 1);
response.writeUInt32BE(lowerWAL, 5);
response.writeUInt32BE(upperWAL, 9);
response.writeUInt32BE(lowerWAL, 13);
response.writeUInt32BE(upperWAL, 17);
response.writeUInt32BE(lowerWAL, 21);
response.writeUInt32BE(upperTimestamp, 25);
response.writeUInt32BE(lowerTimestamp, 29);
response.writeInt8(0, 33);
// @ts-ignore
this.connection.sendCopyFromChunk(response);
this.lastAckTimestamp = Date.now();
return true;
});
}
async #acquireLeaderLock(): Promise<boolean> {
try {
this.leaderLock = await this.redlock.acquire(
[`logical-replication-client:${this.options.name}`],
this.leaderLockTimeoutMs,
{
retryCount: this.leaderLockRetryCount,
retryDelay: this.leaderLockRetryIntervalMs,
}
);
} catch (err) {
this.logger.error("Leader election failed", {
name: this.options.name,
table: this.options.table,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
error: err,
});
return false;
}
return true;
}
async #releaseLeaderLock() {
if (!this.leaderLock) return;
const [releaseError] = await tryCatch(this.leaderLock.release());
this.leaderLock = null;
if (releaseError) {
this.logger.error("Failed to release leader lock", {
name: this.options.name,
error: releaseError,
});
}
}
async #startLeaderLockHeartbeat() {
if (this.leaderLockHeartbeatTimer) {
clearInterval(this.leaderLockHeartbeatTimer);
}
if (!this.leaderLock) return;
this.leaderLockHeartbeatTimer = setInterval(async () => {
if (!this.leaderLock) return;
if (this._isStopped) return;
try {
this.leaderLock = await this.leaderLock.extend(this.leaderLockTimeoutMs);
this.logger.debug("Extended leader lock", {
name: this.options.name,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
});
} catch (err) {
this.logger.error("Failed to extend leader lock", {
name: this.options.name,
slotName: this.options.slotName,
publicationName: this.options.publicationName,
error: err,
});
// Optionally emit an error or handle loss of leadership
this.events.emit("error", err instanceof Error ? err : new Error(String(err)));
}
}, this.leaderLockExtendIntervalMs);
}
#startAckInterval() {
if (this.ackIntervalTimer) {
clearInterval(this.ackIntervalTimer);
}
if (!this.autoAcknowledge || this.ackIntervalSeconds <= 0) return;
this.ackIntervalTimer = setInterval(async () => {
if (this._isStopped) return;
const now = Date.now();
if (
this.lastAcknowledgedLsn &&
now - this.lastAckTimestamp > this.ackIntervalSeconds * 1000
) {
await this.acknowledge(this.lastAcknowledgedLsn);
}
}, 1000);
}
}
internal-packages/replication/src/errors.ts
+5
View File
@@ -0,0 +1,5 @@
export class LogicalReplicationClientError extends Error {
constructor(message: string) {
super(message);
}
}
internal-packages/replication/src/index.ts
+3
View File
@@ -0,0 +1,3 @@
export * from "./client.js";
export * from "./errors.js";
export type * from "./pgoutput.js";
internal-packages/replication/src/pgoutput.ts
+403
View File
@@ -0,0 +1,403 @@
// NOTE: This file requires ES2020 or higher for BigInt literals (used in BinaryReader.readTime)
import { Client } from "pg";
import { types } from "pg";
export interface PgoutputOptions {
protoVersion: 1 | 2;
publicationNames: string[];
messages?: boolean;
}
export type PgoutputMessage =
| MessageBegin
| MessageCommit
| MessageDelete
| MessageInsert
| MessageMessage
| MessageOrigin
| MessageRelation
| MessageTruncate
| MessageType
| MessageUpdate;
export interface MessageBegin {
tag: "begin";
commitLsn: string | null;
commitTime: bigint;
xid: number;
}
export interface MessageCommit {
tag: "commit";
flags: number;
commitLsn: string | null;
commitEndLsn: string | null;
commitTime: bigint;
}
export interface MessageDelete {
tag: "delete";
relation: MessageRelation;
key: Record<string, any> | null;
old: Record<string, any> | null;
}
export interface MessageInsert {
tag: "insert";
relation: MessageRelation;
new: Record<string, any>;
}
export interface MessageMessage {
tag: "message";
flags: number;
transactional: boolean;
messageLsn: string | null;
prefix: string;
content: Uint8Array;
}
export interface MessageOrigin {
tag: "origin";
originLsn: string | null;
originName: string;
}
export interface MessageRelation {
tag: "relation";
relationOid: number;
schema: string;
name: string;
replicaIdentity: "default" | "nothing" | "full" | "index";
columns: RelationColumn[];
keyColumns: string[];
}
export interface RelationColumn {
name: string;
flags: number;
typeOid: number;
typeMod: number;
typeSchema: string | null;
typeName: string | null;
parser: (raw: any) => any;
}
export interface MessageTruncate {
tag: "truncate";
cascade: boolean;
restartIdentity: boolean;
relations: MessageRelation[];
}
export interface MessageType {
tag: "type";
typeOid: number;
typeSchema: string;
typeName: string;
}
export interface MessageUpdate {
tag: "update";
relation: MessageRelation;
key: Record<string, any> | null;
old: Record<string, any> | null;
new: Record<string, any>;
}
class BinaryReader {
private offset = 0;
constructor(private buf: Buffer) {}
readUint8(): number {
return this.buf.readUInt8(this.offset++);
}
readInt16(): number {
const v = this.buf.readInt16BE(this.offset);
this.offset += 2;
return v;
}
readInt32(): number {
const v = this.buf.readInt32BE(this.offset);
this.offset += 4;
return v;
}
readString(): string {
let end = this.buf.indexOf(0, this.offset);
if (end === -1) throw new Error("Null-terminated string not found");
const str = this.buf.toString("utf8", this.offset, end);
this.offset = end + 1;
return str;
}
read(len: number): Buffer {
const b = this.buf.subarray(this.offset, this.offset + len);
this.offset += len;
return b;
}
decodeText(buf: Buffer): string {
return buf.toString("utf8");
}
array<T>(n: number, fn: () => T): T[] {
return Array.from({ length: n }, fn);
}
readLsn(): string | null {
const upper = this.readUint32();
const lower = this.readUint32();
if (upper === 0 && lower === 0) {
return null;
}
return (
upper.toString(16).padStart(8, "0").toUpperCase() +
"/" +
lower.toString(16).padStart(8, "0").toUpperCase()
);
}
readUint32(): number {
// >>> 0 ensures unsigned
return this.readInt32() >>> 0;
}
readUint64(): bigint {
// Combine two unsigned 32-bit ints into a 64-bit bigint
return (BigInt(this.readUint32()) << 32n) | BigInt(this.readUint32());
}
readTime(): bigint {
// (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY == 946684800000000
const microsSinceUnixEpoch = this.readUint64() + 946684800000000n;
return microsSinceUnixEpoch;
}
}
export class PgoutputParser {
private _typeCache = new Map<number, { typeSchema: string; typeName: string }>();
private _relationCache = new Map<number, MessageRelation>();
public parse(buf: Buffer): PgoutputMessage {
const reader = new BinaryReader(buf);
const tag = reader.readUint8();
switch (tag) {
case 0x42:
return this.msgBegin(reader);
case 0x4f:
return this.msgOrigin(reader);
case 0x59:
return this.msgType(reader);
case 0x52:
return this.msgRelation(reader);
case 0x49:
return this.msgInsert(reader);
case 0x55:
return this.msgUpdate(reader);
case 0x44:
return this.msgDelete(reader);
case 0x54:
return this.msgTruncate(reader);
case 0x4d:
return this.msgMessage(reader);
case 0x43:
return this.msgCommit(reader);
default:
throw Error("unknown pgoutput message");
}
}
private msgBegin(reader: BinaryReader): MessageBegin {
return {
tag: "begin",
commitLsn: reader.readLsn(),
commitTime: reader.readTime(),
xid: reader.readInt32(),
};
}
private msgOrigin(reader: BinaryReader): MessageOrigin {
return {
tag: "origin",
originLsn: reader.readLsn(),
originName: reader.readString(),
};
}
private msgType(reader: BinaryReader): MessageType {
const typeOid = reader.readInt32();
const typeSchema = reader.readString();
const typeName = reader.readString();
this._typeCache.set(typeOid, { typeSchema, typeName });
return { tag: "type", typeOid, typeSchema, typeName };
}
private msgRelation(reader: BinaryReader): MessageRelation {
const relationOid = reader.readInt32();
const schema = reader.readString();
const name = reader.readString();
const replicaIdentity = this.readRelationReplicaIdentity(reader);
const columns = reader.array(reader.readInt16(), () => this.readRelationColumn(reader));
const keyColumns = columns.filter((it) => it.flags & 0b1).map((it) => it.name);
const msg: MessageRelation = {
tag: "relation",
relationOid,
schema,
name,
replicaIdentity,
columns,
keyColumns,
};
this._relationCache.set(relationOid, msg);
return msg;
}
private readRelationReplicaIdentity(reader: BinaryReader) {
const ident = reader.readUint8();
switch (ident) {
case 0x64:
return "default";
case 0x6e:
return "nothing";
case 0x66:
return "full";
case 0x69:
return "index";
default:
throw Error(`unknown replica identity ${String.fromCharCode(ident)}`);
}
}
private readRelationColumn(reader: BinaryReader): RelationColumn {
const flags = reader.readUint8();
const name = reader.readString();
const typeOid = reader.readInt32();
const typeMod = reader.readInt32();
return {
flags,
name,
typeOid,
typeMod,
typeSchema: null,
typeName: null,
...this._typeCache.get(typeOid),
parser: types.getTypeParser(typeOid),
};
}
private msgInsert(reader: BinaryReader): MessageInsert {
const relation = this._relationCache.get(reader.readInt32());
if (!relation) throw Error("missing relation");
reader.readUint8(); // consume the 'N' key
return {
tag: "insert",
relation,
new: this.readTuple(reader, relation),
};
}
private msgUpdate(reader: BinaryReader): MessageUpdate {
const relation = this._relationCache.get(reader.readInt32());
if (!relation) throw Error("missing relation");
let key: Record<string, any> | null = null;
let old: Record<string, any> | null = null;
let new_: Record<string, any> | null = null;
const subMsgKey = reader.readUint8();
if (subMsgKey === 0x4b) {
key = this.readKeyTuple(reader, relation);
reader.readUint8();
new_ = this.readTuple(reader, relation);
} else if (subMsgKey === 0x4f) {
old = this.readTuple(reader, relation);
reader.readUint8();
new_ = this.readTuple(reader, relation, old);
} else if (subMsgKey === 0x4e) {
new_ = this.readTuple(reader, relation);
} else {
throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);
}
return { tag: "update", relation, key, old, new: new_ };
}
private msgDelete(reader: BinaryReader): MessageDelete {
const relation = this._relationCache.get(reader.readInt32());
if (!relation) throw Error("missing relation");
let key: Record<string, any> | null = null;
let old: Record<string, any> | null = null;
const subMsgKey = reader.readUint8();
if (subMsgKey === 0x4b) {
key = this.readKeyTuple(reader, relation);
} else if (subMsgKey === 0x4f) {
old = this.readTuple(reader, relation);
} else {
throw Error(`unknown submessage key ${String.fromCharCode(subMsgKey)}`);
}
return { tag: "delete", relation, key, old };
}
private readKeyTuple(reader: BinaryReader, relation: MessageRelation): Record<string, any> {
const tuple = this.readTuple(reader, relation);
const key = Object.create(null);
for (const k of relation.keyColumns) {
key[k] = tuple[k] === null ? undefined : tuple[k];
}
return key;
}
private readTuple(
reader: BinaryReader,
{ columns }: MessageRelation,
unchangedToastFallback?: Record<string, any> | null
): Record<string, any> {
const nfields = reader.readInt16();
const tuple = Object.create(null);
for (let i = 0; i < nfields; i++) {
const { name, parser } = columns[i];
const kind = reader.readUint8();
switch (kind) {
case 0x62: // 'b' binary
const bsize = reader.readInt32();
const bval = reader.read(bsize);
tuple[name] = bval;
break;
case 0x74: // 't' text
const valsize = reader.readInt32();
const valbuf = reader.read(valsize);
const valtext = reader.decodeText(valbuf);
tuple[name] = parser(valtext);
break;
case 0x6e: // 'n' null
tuple[name] = null;
break;
case 0x75: // 'u' unchanged toast datum
tuple[name] = unchangedToastFallback?.[name];
break;
default:
throw Error(`unknown attribute kind ${String.fromCharCode(kind)}`);
}
}
return tuple;
}
private msgTruncate(reader: BinaryReader): MessageTruncate {
const nrels = reader.readInt32();
const flags = reader.readUint8();
return {
tag: "truncate",
cascade: Boolean(flags & 0b1),
restartIdentity: Boolean(flags & 0b10),
relations: reader.array(
nrels,
() => this._relationCache.get(reader.readInt32()) as MessageRelation
),
};
}
private msgMessage(reader: BinaryReader): MessageMessage {
const flags = reader.readUint8();
return {
tag: "message",
flags,
transactional: Boolean(flags & 0b1),
messageLsn: reader.readLsn(),
prefix: reader.readString(),
content: reader.read(reader.readInt32()),
};
}
private msgCommit(reader: BinaryReader): MessageCommit {
return {
tag: "commit",
flags: reader.readUint8(),
commitLsn: reader.readLsn(),
commitEndLsn: reader.readLsn(),
commitTime: reader.readTime(),
};
}
}
export function getPgoutputStartReplicationSQL(
slotName: string,
lastLsn: string,
options: PgoutputOptions
): string {
const opts = [
`proto_version '${options.protoVersion}'`,
`publication_names '${options.publicationNames.join(",")}'`,
`messages '${options.messages ?? false}'`,
];
return `START_REPLICATION SLOT "${slotName}" LOGICAL ${lastLsn} (${opts.join(", ")});`;
}
internal-packages/replication/tsconfig.build.json
+21
View File
@@ -0,0 +1,21 @@
{
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts"],
"compilerOptions": {
"composite": true,
"target": "ES2020",
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"outDir": "dist",
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"declaration": true
}
}
internal-packages/replication/tsconfig.json
+8
View File
@@ -0,0 +1,8 @@
{
"references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
"compilerOptions": {
"moduleResolution": "Node16",
"module": "Node16",
"customConditions": ["@triggerdotdev/source"]
}
}
internal-packages/replication/tsconfig.src.json
+20
View File
@@ -0,0 +1,20 @@
{
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "src/**/*.test.ts"],
"compilerOptions": {
"composite": true,
"target": "ES2020",
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"customConditions": ["@triggerdotdev/source"]
}
}
internal-packages/replication/tsconfig.test.json
+21
View File
@@ -0,0 +1,21 @@
{
"include": ["src/**/*.test.ts", "vitest.config.ts"],
"references": [{ "path": "./tsconfig.src.json" }],
"compilerOptions": {
"composite": true,
"target": "ES2020",
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"module": "Node16",
"moduleResolution": "Node16",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"types": ["vitest/globals"],
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"strict": true,
"customConditions": ["@triggerdotdev/source"]
}
}
internal-packages/replication/vitest.config.ts
+19
View File
@@ -0,0 +1,19 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
include: ["**/*.test.ts"],
globals: true,
isolate: true,
fileParallelism: false,
poolOptions: {
threads: {
singleThread: true,
},
},
testTimeout: 60_000,
coverage: {
provider: "v8",
},
},
});
internal-packages/run-engine/src/engine/eventBus.ts
+172 -2
View File
@@ -1,20 +1,134 @@
import { TaskRunExecutionStatus, TaskRunStatus } from "@trigger.dev/database";
import { AuthenticatedEnvironment } from "../shared/index.js";
import { FlushedRunMetadata, TaskRunError } from "@trigger.dev/core/v3";
import {
RuntimeEnvironmentType,
TaskRunExecutionStatus,
TaskRunStatus,
} from "@trigger.dev/database";
import { EventEmitter } from "events";
import { AuthenticatedEnvironment } from "../shared/index.js";
export type EventBusEvents = {
runCreated: [
{
time: Date;
runId: string;
},
];
runEnqueuedAfterDelay: [
{
time: Date;
run: {
id: string;
status: TaskRunStatus;
queuedAt: Date;
updatedAt: Date;
createdAt: Date;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
runDelayRescheduled: [
{
time: Date;
run: {
id: string;
status: TaskRunStatus;
delayUntil: Date;
updatedAt: Date;
createdAt: Date;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
runLocked: [
{
time: Date;
run: {
id: string;
updatedAt: Date;
status: TaskRunStatus;
lockedAt: Date;
lockedById: string;
lockedToVersionId: string;
lockedQueueId: string;
startedAt: Date;
baseCostInCents: number;
machinePreset: string;
taskVersion: string;
sdkVersion: string;
cliVersion: string;
maxDurationInSeconds?: number;
maxAttempts?: number;
createdAt: Date;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
runStatusChanged: [
{
time: Date;
run: {
id: string;
status: TaskRunStatus;
updatedAt: Date;
createdAt: Date;
};
organization: {
id?: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
runAttemptStarted: [
{
time: Date;
run: {
id: string;
status: TaskRunStatus;
createdAt: Date;
updatedAt: Date;
attemptNumber: number;
baseCostInCents: number;
executedAt: Date | undefined;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
runAttemptFailed: [
@@ -29,6 +143,7 @@ export type EventBusEvents = {
taskEventStore: string;
createdAt: Date;
completedAt: Date | null;
updatedAt: Date;
};
},
];
@@ -37,11 +152,23 @@ export type EventBusEvents = {
time: Date;
run: {
id: string;
status: TaskRunStatus;
spanId: string;
ttl: string | null;
taskEventStore: string;
createdAt: Date;
completedAt: Date | null;
expiredAt: Date | null;
updatedAt: Date;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
@@ -50,12 +177,26 @@ export type EventBusEvents = {
time: Date;
run: {
id: string;
status: TaskRunStatus;
spanId: string;
output: string | undefined;
outputType: string;
taskEventStore: string;
createdAt: Date;
completedAt: Date | null;
usageDurationMs: number;
costInCents: number;
updatedAt: Date;
attemptNumber: number;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
@@ -70,6 +211,19 @@ export type EventBusEvents = {
taskEventStore: string;
createdAt: Date;
completedAt: Date | null;
updatedAt: Date;
attemptNumber: number;
usageDurationMs: number;
costInCents: number;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
@@ -78,6 +232,7 @@ export type EventBusEvents = {
time: Date;
run: {
id: string;
status: TaskRunStatus;
friendlyId: string;
spanId: string;
attemptNumber: number;
@@ -86,6 +241,9 @@ export type EventBusEvents = {
taskIdentifier: string;
baseCostInCents: number;
nextMachineAfterOOM?: string;
updatedAt: Date;
createdAt: Date;
error: TaskRunError;
};
organization: {
id: string;
@@ -99,12 +257,24 @@ export type EventBusEvents = {
time: Date;
run: {
id: string;
status: TaskRunStatus;
friendlyId: string;
spanId: string;
error: TaskRunError;
taskEventStore: string;
createdAt: Date;
completedAt: Date | null;
updatedAt: Date;
attemptNumber: number;
};
organization: {
id: string;
};
project: {
id: string;
};
environment: {
id: string;
};
},
];
internal-packages/run-engine/src/engine/index.ts
+12 -5
View File
@@ -28,11 +28,7 @@ import { FairQueueSelectionStrategy } from "../run-queue/fairQueueSelectionStrat
import { RunQueue } from "../run-queue/index.js";
import { RunQueueFullKeyProducer } from "../run-queue/keyProducer.js";
import { MinimalAuthenticatedEnvironment } from "../shared/index.js";
import {
NotImplementedError,
RunDuplicateIdempotencyKeyError,
ServiceValidationError,
} from "./errors.js";
import { NotImplementedError, RunDuplicateIdempotencyKeyError } from "./errors.js";
import { EventBus, EventBusEvents } from "./eventBus.js";
import { RunLocker } from "./locking.js";
import { BatchSystem } from "./systems/batchSystem.js";
@@ -369,6 +365,8 @@ export class RunEngine {
runnerId,
releaseConcurrency,
runChainState,
scheduleId,
scheduleInstanceId,
}: TriggerParams,
tx?: PrismaClientOrTransaction
): Promise<TaskRun> {
@@ -407,6 +405,8 @@ export class RunEngine {
number,
friendlyId,
runtimeEnvironmentId: environment.id,
environmentType: environment.type,
organizationId: environment.organization.id,
projectId: environment.project.id,
idempotencyKey,
idempotencyKeyExpiresAt,
@@ -455,6 +455,8 @@ export class RunEngine {
maxDurationInSeconds,
machinePreset: machine,
runChainState,
scheduleId,
scheduleInstanceId,
executionSnapshots: {
create: {
engine: "V2",
@@ -552,6 +554,11 @@ export class RunEngine {
}
});
this.eventBus.emit("runCreated", {
time: new Date(),
runId: taskRun.id,
});
return taskRun;
},
{
internal-packages/run-engine/src/engine/systems/checkpointSystem.ts
+43
View File
@@ -143,6 +143,25 @@ export class CheckpointSystem {
throw new ServiceValidationError("Run not found", 404);
}
this.$.eventBus.emit("runStatusChanged", {
time: new Date(),
run: {
id: runId,
status: run.status,
updatedAt: run.updatedAt,
createdAt: run.createdAt,
},
organization: {
id: run.runtimeEnvironment.organizationId,
},
project: {
id: run.runtimeEnvironment.projectId,
},
environment: {
id: run.runtimeEnvironment.id,
},
});
// Create the checkpoint
const taskRunCheckpoint = await prisma.taskRunCheckpoint.create({
data: {
@@ -261,6 +280,11 @@ export class CheckpointSystem {
id: true,
status: true,
attemptNumber: true,
organizationId: true,
runtimeEnvironmentId: true,
projectId: true,
updatedAt: true,
createdAt: true,
},
});
@@ -272,6 +296,25 @@ export class CheckpointSystem {
throw new ServiceValidationError("Run not found", 404);
}
this.$.eventBus.emit("runStatusChanged", {
time: new Date(),
run: {
id: runId,
status: run.status,
updatedAt: run.updatedAt,
createdAt: run.createdAt,
},
organization: {
id: run.organizationId ?? undefined,
},
project: {
id: run.projectId,
},
environment: {
id: run.runtimeEnvironmentId,
},
});
const newSnapshot = await this.executionSnapshotSystem.createExecutionSnapshot(prisma, {
run,
snapshot: {
internal-packages/run-engine/src/engine/systems/delayedRunSystem.ts
+44 -2
View File
@@ -68,6 +68,26 @@ export class DelayedRunSystem {
await this.$.worker.reschedule(`enqueueDelayedRun:${updatedRun.id}`, delayUntil);
this.$.eventBus.emit("runDelayRescheduled", {
time: new Date(),
run: {
id: updatedRun.id,
status: updatedRun.status,
delayUntil: delayUntil,
updatedAt: updatedRun.updatedAt,
createdAt: updatedRun.createdAt,
},
organization: {
id: snapshot.organizationId,
},
project: {
id: updatedRun.projectId,
},
environment: {
id: updatedRun.runtimeEnvironmentId,
},
});
return updatedRun;
});
},
@@ -101,11 +121,33 @@ export class DelayedRunSystem {
batchId: run.batchId ?? undefined,
});
await this.$.prisma.taskRun.update({
const queuedAt = new Date();
const updatedRun = await this.$.prisma.taskRun.update({
where: { id: runId },
data: {
status: "PENDING",
queuedAt: new Date(),
queuedAt,
},
});
this.$.eventBus.emit("runEnqueuedAfterDelay", {
time: new Date(),
run: {
id: runId,
status: "PENDING",
queuedAt,
updatedAt: updatedRun.updatedAt,
createdAt: updatedRun.createdAt,
},
organization: {
id: run.runtimeEnvironment.organizationId,
},
project: {
id: run.runtimeEnvironment.projectId,
},
environment: {
id: run.runtimeEnvironmentId,
},
});
internal-packages/run-engine/src/engine/systems/dequeueSystem.ts
+62 -6
View File
@@ -329,25 +329,29 @@ export class DequeueSystem {
maxAttempts = parsedConfig.data?.maxAttempts;
}
//update the run
const lockedAt = new Date();
const startedAt = result.run.startedAt ?? lockedAt;
const maxDurationInSeconds = getMaxDuration(
result.run.maxDurationInSeconds,
result.task.maxDurationInSeconds
);
const lockedTaskRun = await prisma.taskRun.update({
where: {
id: runId,
},
data: {
lockedAt: new Date(),
lockedAt,
lockedById: result.task.id,
lockedToVersionId: result.worker.id,
lockedQueueId: result.queue.id,
startedAt: result.run.startedAt ?? new Date(),
startedAt,
baseCostInCents: this.options.machines.baseCostInCents,
machinePreset: machinePreset.name,
taskVersion: result.worker.version,
sdkVersion: result.worker.sdkVersion,
cliVersion: result.worker.cliVersion,
maxDurationInSeconds: getMaxDuration(
result.run.maxDurationInSeconds,
result.task.maxDurationInSeconds
),
maxDurationInSeconds,
maxAttempts: maxAttempts ?? undefined,
},
include: {
@@ -356,6 +360,37 @@ export class DequeueSystem {
},
});
this.$.eventBus.emit("runLocked", {
time: new Date(),
run: {
id: runId,
status: lockedTaskRun.status,
lockedAt,
lockedById: result.task.id,
lockedToVersionId: result.worker.id,
lockedQueueId: result.queue.id,
startedAt,
baseCostInCents: this.options.machines.baseCostInCents,
machinePreset: machinePreset.name,
taskVersion: result.worker.version,
sdkVersion: result.worker.sdkVersion,
cliVersion: result.worker.cliVersion,
maxDurationInSeconds: lockedTaskRun.maxDurationInSeconds ?? undefined,
maxAttempts: lockedTaskRun.maxAttempts ?? undefined,
updatedAt: lockedTaskRun.updatedAt,
createdAt: lockedTaskRun.createdAt,
},
organization: {
id: orgId,
},
project: {
id: lockedTaskRun.projectId,
},
environment: {
id: lockedTaskRun.runtimeEnvironmentId,
},
});
if (!lockedTaskRun) {
this.$.logger.error(
"RunEngine.dequeueFromMasterQueue(): Failed to lock task run",
@@ -539,6 +574,8 @@ export class DequeueSystem {
id: true,
status: true,
attemptNumber: true,
updatedAt: true,
createdAt: true,
runtimeEnvironment: {
select: {
id: true,
@@ -573,6 +610,25 @@ export class DequeueSystem {
//we ack because when it's deployed it will be requeued
await this.$.runQueue.acknowledgeMessage(orgId, runId);
this.$.eventBus.emit("runStatusChanged", {
time: new Date(),
run: {
id: runId,
status: run.status,
updatedAt: run.updatedAt,
createdAt: run.createdAt,
},
organization: {
id: run.runtimeEnvironment.project.organizationId,
},
project: {
id: run.runtimeEnvironment.projectId,
},
environment: {
id: run.runtimeEnvironment.id,
},
});
});
},
{
internal-packages/run-engine/src/engine/systems/pendingVersionSystem.ts
+19
View File
@@ -100,6 +100,25 @@ export class PendingVersionSystem {
tx,
});
});
this.$.eventBus.emit("runStatusChanged", {
time: new Date(),
run: {
id: run.id,
status: "PENDING",
updatedAt: run.updatedAt,
createdAt: run.createdAt,
},
organization: {
id: backgroundWorker.runtimeEnvironment.organizationId,
},
project: {
id: backgroundWorker.runtimeEnvironment.projectId,
},
environment: {
id: backgroundWorker.runtimeEnvironmentId,
},
});
}
//enqueue more if needed
internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts
+77 -12
View File
@@ -175,18 +175,6 @@ export class RunAttemptSystem {
throw new ServiceValidationError("Max attempts reached", 400);
}
this.$.eventBus.emit("runAttemptStarted", {
time: new Date(),
run: {
id: taskRun.id,
attemptNumber: nextAttemptNumber,
baseCostInCents: taskRun.baseCostInCents,
},
organization: {
id: environment.organization.id,
},
});
const result = await $transaction(
prisma,
async (tx) => {
@@ -258,6 +246,28 @@ export class RunAttemptSystem {
const { run, snapshot } = result;
this.$.eventBus.emit("runAttemptStarted", {
time: new Date(),
run: {
id: run.id,
status: run.status,
createdAt: run.createdAt,
updatedAt: run.updatedAt,
attemptNumber: nextAttemptNumber,
baseCostInCents: run.baseCostInCents,
executedAt: run.executedAt ?? undefined,
},
organization: {
id: environment.organization.id,
},
project: {
id: environment.project.id,
},
environment: {
id: environment.id,
},
});
const machinePreset = getMachinePreset({
machines: this.options.machines.machines,
defaultMachine: this.options.machines.defaultMachine,
@@ -455,6 +465,7 @@ export class RunAttemptSystem {
status: true,
attemptNumber: true,
spanId: true,
updatedAt: true,
associatedWaitpoint: {
select: {
id: true,
@@ -470,6 +481,10 @@ export class RunAttemptSystem {
completedAt: true,
taskEventStore: true,
parentTaskRunId: true,
usageDurationMs: true,
costInCents: true,
runtimeEnvironmentId: true,
projectId: true,
},
});
const newSnapshot = await getLatestExecutionSnapshot(prisma, runId);
@@ -503,12 +518,26 @@ export class RunAttemptSystem {
time: completedAt,
run: {
id: runId,
status: run.status,
spanId: run.spanId,
output: completion.output,
outputType: completion.outputType,
createdAt: run.createdAt,
completedAt: run.completedAt,
taskEventStore: run.taskEventStore,
usageDurationMs: run.usageDurationMs,
costInCents: run.costInCents,
updatedAt: run.updatedAt,
attemptNumber: run.attemptNumber ?? 1,
},
organization: {
id: run.project.organizationId,
},
project: {
id: run.projectId,
},
environment: {
id: run.runtimeEnvironmentId,
},
});
@@ -593,6 +622,7 @@ export class RunAttemptSystem {
taskEventStore: true,
createdAt: true,
completedAt: true,
updatedAt: true,
},
});
@@ -611,6 +641,7 @@ export class RunAttemptSystem {
createdAt: minimalRun.createdAt,
completedAt: minimalRun.completedAt,
taskEventStore: minimalRun.taskEventStore,
updatedAt: minimalRun.updatedAt,
},
});
}
@@ -679,6 +710,7 @@ export class RunAttemptSystem {
createdAt: run.createdAt,
completedAt: run.completedAt,
taskEventStore: run.taskEventStore,
updatedAt: run.updatedAt,
},
});
}
@@ -687,6 +719,7 @@ export class RunAttemptSystem {
time: failedAt,
run: {
id: run.id,
status: run.status,
friendlyId: run.friendlyId,
attemptNumber: nextAttemptNumber,
queue: run.queue,
@@ -695,6 +728,9 @@ export class RunAttemptSystem {
baseCostInCents: run.baseCostInCents,
spanId: run.spanId,
nextMachineAfterOOM: retryResult.machine,
updatedAt: run.updatedAt,
error: completion.error,
createdAt: run.createdAt,
},
organization: {
id: run.runtimeEnvironment.organizationId,
@@ -974,6 +1010,7 @@ export class RunAttemptSystem {
taskEventStore: true,
parentTaskRunId: true,
delayUntil: true,
updatedAt: true,
runtimeEnvironment: {
select: {
organizationId: true,
@@ -1056,12 +1093,24 @@ export class RunAttemptSystem {
time: new Date(),
run: {
id: run.id,
status: run.status,
friendlyId: run.friendlyId,
spanId: run.spanId,
taskEventStore: run.taskEventStore,
createdAt: run.createdAt,
completedAt: run.completedAt,
error,
updatedAt: run.updatedAt,
attemptNumber: run.attemptNumber ?? 1,
},
organization: {
id: latestSnapshot.organizationId,
},
project: {
id: latestSnapshot.projectId,
},
environment: {
id: latestSnapshot.environmentId,
},
});
@@ -1121,6 +1170,9 @@ export class RunAttemptSystem {
spanId: true,
batchId: true,
parentTaskRunId: true,
updatedAt: true,
usageDurationMs: true,
costInCents: true,
associatedWaitpoint: {
select: {
id: true,
@@ -1181,6 +1233,19 @@ export class RunAttemptSystem {
taskEventStore: run.taskEventStore,
createdAt: run.createdAt,
completedAt: run.completedAt,
updatedAt: run.updatedAt,
attemptNumber: run.attemptNumber ?? 1,
usageDurationMs: run.usageDurationMs,
costInCents: run.costInCents,
},
organization: {
id: run.runtimeEnvironment.project.organizationId,
},
project: {
id: run.runtimeEnvironment.project.id,
},
environment: {
id: run.runtimeEnvironment.id,
},
});
internal-packages/run-engine/src/engine/systems/ttlSystem.ts
+16 -6
View File
@@ -1,11 +1,10 @@
import { startSpan } from "@internal/tracing";
import { SystemResources } from "./systems.js";
import { PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database";
import { getLatestExecutionSnapshot } from "./executionSnapshotSystem.js";
import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic";
import { TaskRunError } from "@trigger.dev/core/v3/schemas";
import { PrismaClientOrTransaction } from "@trigger.dev/database";
import { ServiceValidationError } from "../errors.js";
import { isExecuting } from "../statuses.js";
import { TaskRunError } from "@trigger.dev/core/v3/schemas";
import { getLatestExecutionSnapshot } from "./executionSnapshotSystem.js";
import { SystemResources } from "./systems.js";
import { WaitpointSystem } from "./waitpointSystem.js";
export type TtlSystemOptions = {
@@ -85,6 +84,7 @@ export class TtlSystem {
id: true,
spanId: true,
ttl: true,
updatedAt: true,
associatedWaitpoint: {
select: {
id: true,
@@ -93,12 +93,16 @@ export class TtlSystem {
runtimeEnvironment: {
select: {
organizationId: true,
projectId: true,
id: true,
},
},
createdAt: true,
completedAt: true,
taskEventStore: true,
parentTaskRunId: true,
expiredAt: true,
status: true,
},
});
@@ -113,7 +117,13 @@ export class TtlSystem {
output: { value: JSON.stringify(error), isError: true },
});
this.$.eventBus.emit("runExpired", { run: updatedRun, time: new Date() });
this.$.eventBus.emit("runExpired", {
run: updatedRun,
time: new Date(),
organization: { id: updatedRun.runtimeEnvironment.organizationId },
project: { id: updatedRun.runtimeEnvironment.projectId },
environment: { id: updatedRun.runtimeEnvironment.id },
});
});
}
internal-packages/run-engine/src/engine/types.ts
+2
View File
@@ -118,6 +118,8 @@ export type TriggerParams = {
runnerId?: string;
releaseConcurrency?: boolean;
runChainState?: RunChainState;
scheduleId?: string;
scheduleInstanceId?: string;
};
export type EngineWorker = Worker<typeof workerCatalog>;
internal-packages/run-engine/src/index.ts
+1 -1
View File
@@ -1,4 +1,4 @@
export { RunEngine } from "./engine/index.js";
export { RunDuplicateIdempotencyKeyError, RunOneTimeUseTokenError } from "./engine/errors.js";
export type { EventBusEventArgs } from "./engine/eventBus.js";
export type { EventBusEventArgs, EventBusEvents } from "./engine/eventBus.js";
export type { AuthenticatedEnvironment } from "./shared/index.js";
internal-packages/testcontainers/package.json
+1
View File
@@ -5,6 +5,7 @@
"main": "./src/index.ts",
"types": "./src/index.ts",
"dependencies": {
"@clickhouse/client": "^1.11.1",
"@opentelemetry/api": "^1.9.0",
"@trigger.dev/database": "workspace:*",
"ioredis": "^5.3.2"
internal-packages/testcontainers/src/clickhouse.ts
+185
View File
@@ -0,0 +1,185 @@
import { ClickHouseClient } from "@clickhouse/client";
import { readdir, readFile } from "node:fs/promises";
import { resolve } from "node:path";
import {
AbstractStartedContainer,
GenericContainer,
StartedTestContainer,
Wait,
} from "testcontainers";
const CLICKHOUSE_PORT = 9000;
const CLICKHOUSE_HTTP_PORT = 8123;
export class ClickHouseContainer extends GenericContainer {
private username = "test";
private password = "test";
private database = "test";
constructor(image = "clickhouse/clickhouse-server:25.4-alpine") {
super(image);
this.withExposedPorts(CLICKHOUSE_PORT, CLICKHOUSE_HTTP_PORT);
this.withWaitStrategy(
Wait.forHttp("/", CLICKHOUSE_HTTP_PORT).forResponsePredicate(
(response) => response === "Ok.\n"
)
);
this.withStartupTimeout(120_000);
// Setting this high ulimits value proactively prevents the "Too many open files" error,
// especially under potentially heavy load during testing.
this.withUlimits({
nofile: {
hard: 262144,
soft: 262144,
},
});
}
public withDatabase(database: string): this {
this.database = database;
return this;
}
public withUsername(username: string): this {
this.username = username;
return this;
}
public withPassword(password: string): this {
this.password = password;
return this;
}
public override async start(): Promise<StartedClickHouseContainer> {
this.withEnvironment({
CLICKHOUSE_USER: this.username,
CLICKHOUSE_PASSWORD: this.password,
CLICKHOUSE_DB: this.database,
});
return new StartedClickHouseContainer(
await super.start(),
this.database,
this.username,
this.password
);
}
}
export class StartedClickHouseContainer extends AbstractStartedContainer {
constructor(
startedTestContainer: StartedTestContainer,
private readonly database: string,
private readonly username: string,
private readonly password: string
) {
super(startedTestContainer);
}
public getPort(): number {
return super.getMappedPort(CLICKHOUSE_PORT);
}
public getHttpPort(): number {
return super.getMappedPort(CLICKHOUSE_HTTP_PORT);
}
public getUsername(): string {
return this.username;
}
public getPassword(): string {
return this.password;
}
public getDatabase(): string {
return this.database;
}
/**
* Gets the base HTTP URL (protocol, host and mapped port) for the ClickHouse container's HTTP interface.
* Example: `http://localhost:32768`
*/
public getHttpUrl(): string {
const protocol = "http";
const host = this.getHost();
const port = this.getHttpPort();
return `${protocol}://${host}:${port}`;
}
/**
* Gets configuration options suitable for passing directly to `createClient({...})`
* from `@clickhouse/client`. Uses the HTTP interface.
*/
public getClientOptions(): {
url?: string;
username: string;
password: string;
database: string;
} {
return {
url: this.getHttpUrl(),
username: this.getUsername(),
password: this.getPassword(),
database: this.getDatabase(),
};
}
/**
* Gets a ClickHouse connection URL for the HTTP interface with format:
* http://username:password@hostname:port/database
* @returns The ClickHouse HTTP URL string.
*/
public getConnectionUrl(): string {
const url = new URL(this.getHttpUrl());
url.username = this.getUsername();
url.password = this.getPassword();
const dbName = this.getDatabase();
url.pathname = dbName.startsWith("/") ? dbName : `/${dbName}`;
return url.toString();
}
}
export async function runClickhouseMigrations(client: ClickHouseClient, migrationsPath: string) {
// Get all the *.sql files in the migrations path
const queries = await getAllClickhouseMigrationQueries(migrationsPath);
for (const query of queries) {
await client.command({
query,
});
}
}
async function getAllClickhouseMigrationQueries(migrationsPath: string) {
const queries: string[] = [];
// Get all the *.sql files in the migrations path
const migrations = await readdir(migrationsPath);
for (const migration of migrations) {
const migrationPath = resolve(migrationsPath, migration);
const migrationContent = await readFile(migrationPath, "utf-8");
// Split content by goose markers
const parts = migrationContent.split(/--\s*\+goose\s+(Up|Down)/i);
// The array will be: ["", "Up", "up queries", "Down", "down queries"]
// We want the "up queries" part which is at index 2
if (parts.length >= 3) {
const upQueries = parts[2]!.trim();
queries.push(
...upQueries
.split(";")
.filter((q) => q.trim())
.map((q) => q.trim())
);
}
}
return queries;
}
internal-packages/testcontainers/src/index.ts
+56 -1
View File
@@ -5,6 +5,7 @@ import { RedisOptions } from "ioredis";
import { Network, type StartedNetwork } from "testcontainers";
import { TaskContext, test } from "vitest";
import {
createClickHouseContainer,
createElectricContainer,
createPostgresContainer,
createRedisContainer,
@@ -12,6 +13,8 @@ import {
withContainerSetup,
} from "./utils";
import { getTaskMetadata, logCleanup, logSetup } from "./logs";
import { StartedClickHouseContainer } from "./clickhouse";
import { ClickHouseClient, createClient } from "@clickhouse/client";
export { assertNonNullable } from "./utils";
export { StartedRedisContainer };
@@ -32,7 +35,8 @@ type ElectricContext = {
electricOrigin: string;
};
type ContainerContext = NetworkContext & PostgresContext & RedisContext;
type ContainerContext = NetworkContext & PostgresContext & RedisContext & ClickhouseContext;
type PostgresAndRedisContext = NetworkContext & PostgresContext & RedisContext;
type ContainerWithElectricAndRedisContext = ContainerContext & ElectricContext;
type ContainerWithElectricContext = NetworkContext & PostgresContext & ElectricContext;
@@ -170,12 +174,61 @@ const electricOrigin = async (
await useContainer("electricContainer", { container, task, use: () => use(origin) });
};
const clickhouseContainer = async (
{ network, task }: { network: StartedNetwork } & TaskContext,
use: Use<StartedClickHouseContainer>
) => {
const { container, metadata } = await withContainerSetup({
name: "clickhouseContainer",
task,
setup: createClickHouseContainer(network),
});
await useContainer("clickhouseContainer", { container, task, use: () => use(container) });
};
const clickhouseClient = async (
{ clickhouseContainer, task }: { clickhouseContainer: StartedClickHouseContainer } & TaskContext,
use: Use<ClickHouseClient>
) => {
const testName = task.name;
const client = createClient({ url: clickhouseContainer.getConnectionUrl() });
try {
await use(client);
} finally {
await logCleanup("clickhouseClient", client.close(), { testName });
}
};
type ClickhouseContext = {
network: StartedNetwork;
clickhouseContainer: StartedClickHouseContainer;
clickhouseClient: ClickHouseClient;
};
export const clickhouseTest = test.extend<ClickhouseContext>({
network,
clickhouseContainer,
clickhouseClient,
});
export const postgresAndRedisTest = test.extend<PostgresAndRedisContext>({
network,
postgresContainer,
prisma,
redisContainer,
redisOptions,
});
export const containerTest = test.extend<ContainerContext>({
network,
postgresContainer,
prisma,
redisContainer,
redisOptions,
clickhouseContainer,
clickhouseClient,
});
export const containerWithElectricTest = test.extend<ContainerWithElectricContext>({
@@ -192,4 +245,6 @@ export const containerWithElectricAndRedisTest = test.extend<ContainerWithElectr
redisContainer,
redisOptions,
electricOrigin,
clickhouseContainer,
clickhouseClient,
});
internal-packages/testcontainers/src/utils.ts
+24
View File
@@ -9,6 +9,9 @@ import { x } from "tinyexec";
import { expect, TaskContext } from "vitest";
import { getContainerMetadata, getTaskMetadata, logCleanup } from "./logs";
import { logSetup } from "./logs";
import { ClickHouseContainer, runClickhouseMigrations } from "./clickhouse";
import { createClient } from "@clickhouse/client";
import { readdir, readFile } from "node:fs/promises";
export async function createPostgresContainer(network: StartedNetwork) {
const container = await new PostgreSqlContainer("docker.io/postgres:14")
@@ -45,6 +48,27 @@ export async function createPostgresContainer(network: StartedNetwork) {
return { url: container.getConnectionUri(), container, network };
}
export async function createClickHouseContainer(network: StartedNetwork) {
const container = await new ClickHouseContainer().withNetwork(network).start();
const client = createClient({
url: container.getConnectionUrl(),
});
await client.ping();
// Now we run the migrations
const migrationsPath = path.resolve(__dirname, "../../clickhouse/schema");
await runClickhouseMigrations(client, migrationsPath);
return {
url: container.getConnectionUrl(),
container,
network,
};
}
export async function createRedisContainer({
port,
network,
internal-packages/tracing/src/index.ts
+8
View File
@@ -45,6 +45,14 @@ export async function startSpan<T>(
});
}
export function recordSpanError(span: Span, error: Error) {
span.recordException(error);
span.setStatus({
code: SpanStatusCode.ERROR,
message: error.message,
});
}
export async function emitDebugLog(
logger: Logger,
message: string,
package.json
+2 -2
View File
@@ -22,7 +22,7 @@
"lint": "turbo run lint",
"docker": "docker compose -p triggerdotdev-docker -f docker/docker-compose.yml up -d --build --remove-orphans",
"docker:stop": "docker compose -p triggerdotdev-docker -f docker/docker-compose.yml stop",
"dev:docker": "docker compose -p triggerdotdev-dev-docker -f docker/dev-compose.yml up -d",
"dev:docker": "docker compose -p triggerdotdev-dev-docker -f docker/dev-compose.yml up -d --build --remove-orphans",
"dev:docker:build": "docker compose -p triggerdotdev-dev-docker -f docker/dev-compose.yml up -d --build",
"dev:docker:stop": "docker compose -p triggerdotdev-dev-docker -f docker/dev-compose.yml stop",
"test": "turbo run test --concurrency=1 -- --run",
@@ -81,4 +81,4 @@
"@kubernetes/client-node@1.0.0": "patches/@kubernetes__client-node@1.0.0.patch"
}
}
}
}
packages/core/src/v3/streams/asyncIterableStream.ts
+9 -1
View File
@@ -51,7 +51,7 @@ export function createAsyncIterableReadable<S, T>(
export function createAsyncIterableStreamFromAsyncIterable<T>(
asyncIterable: AsyncIterable<T>,
transformer: Transformer<T, T>,
transformer?: Transformer<T, T>,
signal?: AbortSignal
): AsyncIterableStream<T> {
const stream = new ReadableStream<T>({
@@ -95,3 +95,11 @@ export function createAsyncIterableStreamFromAsyncIterable<T>(
return transformedStream as AsyncIterableStream<T>;
}
export function createAsyncIterableStreamFromAsyncGenerator<T>(
asyncGenerator: AsyncGenerator<T, void, unknown>,
transformer: Transformer<T, T>,
signal?: AbortSignal
): AsyncIterableStream<T> {
return createAsyncIterableStreamFromAsyncIterable(asyncGenerator, transformer, signal);
}
packages/core/src/v3/tryCatch.ts
+3 -3
View File
@@ -1,8 +1,8 @@
// Types for the result object with discriminated union
type Success<T> = [null, T];
type Failure<E> = [E, null];
export type Success<T> = [null, T];
export type Failure<E> = [E, null];
type Result<T, E = Error> = Success<T> | Failure<E>;
export type Result<T, E = Error> = Success<T> | Failure<E>;
// Main wrapper function
export async function tryCatch<T, E = Error>(promise: Promise<T>): Promise<Result<T, E>> {
pnpm-lock.yaml
Generated
+911 -548
View File
File diff suppressed because it is too large Load Diff
references/hello-world/package.json
+3 -2
View File
@@ -12,6 +12,7 @@
"zod": "3.23.8"
},
"scripts": {
"dev": "trigger dev"
"dev": "trigger dev",
"deploy": "trigger deploy"
}
}
}
scripts/build-dockerfile.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/bin/bash
set -e
docker build -t local-triggerdotdev:latest -f docker/Dockerfile .
image=local-triggerdotdev:latest
src=/triggerdotdev
dst=$(mktemp -d)
mkdir -p $dst
echo -e "Extracting image into $dst..."
container=$(docker create "$image")
docker cp "$container:$src" "$dst"
docker rm "$container"
/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code "$dst/triggerdotdev"
scripts/start-prometheus.sh
Executable
+5
View File
@@ -0,0 +1,5 @@
#!/bin/bash
set -e
prometheus --config.file=./.configs/prometheus.yml --storage.tsdb.path=/tmp/prom-data