From e29e1c86d9b9cdc01c36b5fbd083c857c5909180 Mon Sep 17 00:00:00 2001 From: Mihai Popescu Date: Thu, 29 Jan 2026 12:47:59 +0200 Subject: [PATCH 01/22] Fix/tri 7032 logs page feedback (#2947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ✨ Changes ### UI & UX - Normalized log level display across table and detail view - Fixed table header scroll behavior and sidebar positioning - Improved loading state with taller segment and disabled resizing - Added "no more logs" message with count - Enhanced keyboard shortcuts ### Filtering & Search - Streamlined filters: RunId and Task only (removed run filters) - Side panel closes when filters change - Fixed logs from previous search remaining in table - Fixed table scroll position when changing filters ### Backend - Added performance indexes on message and attributes (`014_add_task_runs_v2_search_indexes.sql`) - Added DEBUG level logging by default - Removed internal logs from display - Fixed ServiceValidationError forwarding to frontend - Removed v1 logs API support --- .../app/components/LogLevelTooltipInfo.tsx | 63 +++ apps/webapp/app/components/Shortcuts.tsx | 31 ++ .../app/components/logs/LogDetailView.tsx | 64 +-- .../app/components/logs/LogsLevelFilter.tsx | 125 ++---- .../app/components/logs/LogsRunIdFilter.tsx | 2 +- .../app/components/logs/LogsSearchInput.tsx | 14 +- apps/webapp/app/components/logs/LogsTable.tsx | 69 ++-- .../app/components/logs/LogsTaskFilter.tsx | 144 +++++++ .../app/components/primitives/Table.tsx | 19 +- apps/webapp/app/hooks/useCanViewLogsPage.ts | 16 + .../v3/LogDetailPresenter.server.ts | 5 +- .../presenters/v3/LogsListPresenter.server.ts | 276 ++++--------- .../route.tsx | 386 ++++++++++++------ .../route.tsx | 53 +++ ...ojects.$projectParam.env.$envParam.logs.ts | 34 +- .../route.tsx | 81 ++-- apps/webapp/app/utils/logUtils.ts | 50 +-- .../014_add_task_runs_v2_serch_indexes.sql | 20 + internal-packages/clickhouse/src/index.ts | 4 - .../clickhouse/src/taskEvents.ts | 54 +-- 20 files changed, 876 insertions(+), 634 deletions(-) create mode 100644 apps/webapp/app/components/LogLevelTooltipInfo.tsx create mode 100644 apps/webapp/app/components/logs/LogsTaskFilter.tsx create mode 100644 apps/webapp/app/hooks/useCanViewLogsPage.ts create mode 100644 apps/webapp/app/routes/resources.orgs.$organizationSlug.can-view-logs-page/route.tsx create mode 100644 internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql diff --git a/apps/webapp/app/components/LogLevelTooltipInfo.tsx b/apps/webapp/app/components/LogLevelTooltipInfo.tsx new file mode 100644 index 000000000..6f967af70 --- /dev/null +++ b/apps/webapp/app/components/LogLevelTooltipInfo.tsx @@ -0,0 +1,63 @@ +import { BookOpenIcon } from "@heroicons/react/20/solid"; +import { LinkButton } from "./primitives/Buttons"; +import { Header3 } from "./primitives/Headers"; +import { Paragraph } from "./primitives/Paragraph"; + +export function LogLevelTooltipInfo() { + return ( +
+
+ Log Levels + + Structured logging helps you debug and monitor your tasks. + +
+
+
+ Info +
+ + General informational messages about task execution. + +
+
+
+ Warn +
+ + Warning messages indicating potential issues that don't prevent execution. + +
+
+
+ Error +
+ + Error messages for failures and exceptions during task execution. + +
+
+
+ Debug +
+ + Detailed diagnostic information for development and debugging. + +
+
+ Tracing & Spans + + Automatically track the flow of your code through task triggers, attempts, and HTTP + requests. Create custom traces to monitor specific operations. + +
+ + Read docs + +
+ ); +} diff --git a/apps/webapp/app/components/Shortcuts.tsx b/apps/webapp/app/components/Shortcuts.tsx index cf1b01a70..e3e4d6fe9 100644 --- a/apps/webapp/app/components/Shortcuts.tsx +++ b/apps/webapp/app/components/Shortcuts.tsx @@ -161,6 +161,37 @@ function ShortcutContent() { +
+ Logs page + + + + + + + + + + + + + to + + + + + + + + + + + + + + + +
Schedules page diff --git a/apps/webapp/app/components/logs/LogDetailView.tsx b/apps/webapp/app/components/logs/LogDetailView.tsx index a367e7549..22e2e288a 100644 --- a/apps/webapp/app/components/logs/LogDetailView.tsx +++ b/apps/webapp/app/components/logs/LogDetailView.tsx @@ -21,7 +21,7 @@ import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; -import { getLevelColor, getKindColor, getKindLabel } from "~/utils/logUtils"; +import { getLevelColor } from "~/utils/logUtils"; import { v3RunSpanPath, v3RunsPath, v3DeploymentVersionPath } from "~/utils/pathBuilder"; import type { loader as logDetailLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId"; import { TaskRunStatusCombo, descriptionForTaskRunStatus } from "~/components/runs/v3/TaskRunStatus"; @@ -94,16 +94,34 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet const isLoading = fetcher.state === "loading"; const log = fetcher.data ?? initialLog; - // Handle Escape key to close panel + const runPath = v3RunSpanPath( + organization, + project, + environment, + { friendlyId: log?.runId ?? "" }, + { spanId: log?.spanId ?? "" } + ); + + // Handle keyboard shortcuts useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { + const target = e.target as HTMLElement; + if (target && ( + target.tagName === "INPUT" || + target.tagName === "TEXTAREA" || + target.tagName === "SELECT" || + target.contentEditable === "true" + )) { + return; + } + if (e.key === "Escape") { onClose(); } }; window.addEventListener("keydown", handleKeyDown); return () => window.removeEventListener("keydown", handleKeyDown); - }, [onClose]); + }, [onClose, log, runPath, isLoading]); if (isLoading && !log) { return ( @@ -129,36 +147,18 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet ); } - const runPath = v3RunSpanPath( - organization, - project, - environment, - { friendlyId: log.runId }, - { spanId: log.spanId } - ); - return (
{/* Header */} -
-
- - {getKindLabel(log.kind)} - - - {log.level} - -
+
+ + {log.level} + @@ -185,8 +185,8 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet -
diff --git a/apps/webapp/app/components/logs/LogsLevelFilter.tsx b/apps/webapp/app/components/logs/LogsLevelFilter.tsx index 4ec7d9573..8c2abf64f 100644 --- a/apps/webapp/app/components/logs/LogsLevelFilter.tsx +++ b/apps/webapp/app/components/logs/LogsLevelFilter.tsx @@ -1,9 +1,8 @@ import * as Ariakit from "@ariakit/react"; -import { ExclamationTriangleIcon } from "@heroicons/react/20/solid"; -import { type ReactNode, useMemo } from "react"; +import { IconListTree } from "@tabler/icons-react"; +import { type ReactNode } from "react"; import { AppliedFilter } from "~/components/primitives/AppliedFilter"; import { - ComboBox, SelectItem, SelectList, SelectPopover, @@ -12,24 +11,20 @@ import { shortcutFromIndex, } from "~/components/primitives/Select"; import { useSearchParams } from "~/hooks/useSearchParam"; -import { FilterMenuProvider, appliedSummary } from "~/components/runs/v3/SharedFilters"; +import { appliedSummary } from "~/components/runs/v3/SharedFilters"; import type { LogLevel } from "~/presenters/v3/LogsListPresenter.server"; import { cn } from "~/utils/cn"; const allLogLevels: { level: LogLevel; label: string; color: string }[] = [ - { level: "ERROR", label: "Error", color: "text-error" }, - { level: "WARN", label: "Warning", color: "text-warning" }, { level: "INFO", label: "Info", color: "text-blue-400" }, - { level: "CANCELLED", label: "Cancelled", color: "text-charcoal-400" }, + { level: "WARN", label: "Warning", color: "text-warning" }, + { level: "ERROR", label: "Error", color: "text-error" }, { level: "DEBUG", label: "Debug", color: "text-charcoal-400" }, - { level: "TRACE", label: "Trace", color: "text-charcoal-500" }, ]; -function getAvailableLevels(showDebug: boolean): typeof allLogLevels { - if (showDebug) { - return allLogLevels; - } - return allLogLevels.filter((level) => level.level !== "DEBUG"); +// In the future we might add other levels or change which are available +function getAvailableLevels(): typeof allLogLevels { + return allLogLevels; } function getLevelBadgeColor(level: LogLevel): string { @@ -42,10 +37,6 @@ function getLevelBadgeColor(level: LogLevel): string { return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; case "INFO": return "text-blue-400 bg-blue-500/10 border-blue-500/20"; - case "TRACE": - return "text-charcoal-500 bg-charcoal-800 border-charcoal-700"; - case "CANCELLED": - return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; default: return "text-text-dimmed bg-charcoal-750 border-charcoal-700"; } @@ -53,81 +44,50 @@ function getLevelBadgeColor(level: LogLevel): string { const shortcut = { key: "l" }; -export function LogsLevelFilter({ showDebug = false }: { showDebug?: boolean }) { +export function LogsLevelFilter() { const { values } = useSearchParams(); const selectedLevels = values("levels"); const hasLevels = selectedLevels.length > 0 && selectedLevels.some((v) => v !== ""); if (hasLevels) { - return ; + return ; } return ( - - {(search, setSearch) => ( - } - variant="secondary/small" - shortcut={shortcut} - tooltipTitle="Filter by level" - > - Level - - } - searchValue={search} - clearSearchValue={() => setSearch("")} - showDebug={showDebug} - /> - )} - + } + variant="secondary/small" + shortcut={shortcut} + tooltipTitle="Filter by level" + > + Level + + } + /> ); } function LevelDropdown({ trigger, - clearSearchValue, - searchValue, - onClose, - showDebug = false, }: { trigger: ReactNode; - clearSearchValue: () => void; - searchValue: string; - onClose?: () => void; - showDebug?: boolean; }) { const { values, replace } = useSearchParams(); const handleChange = (values: string[]) => { - clearSearchValue(); replace({ levels: values, cursor: undefined, direction: undefined }); }; - const availableLevels = getAvailableLevels(showDebug); - const filtered = useMemo(() => { - return availableLevels.filter((item) => - item.label.toLowerCase().includes(searchValue.toLowerCase()) - ); - }, [searchValue, availableLevels]); + const availableLevels = getAvailableLevels(); return ( {trigger} - { - if (onClose) { - onClose(); - return false; - } - return true; - }} - > - + - {filtered.map((item, index) => ( + {availableLevels.map((item, index) => ( - {(search, setSearch) => ( - }> - } - value={appliedSummary(levels)} - onRemove={() => del(["levels", "cursor", "direction"])} - variant="secondary/small" - /> - - } - searchValue={search} - clearSearchValue={() => setSearch("")} - showDebug={showDebug} - /> - )} - + }> + } + value={appliedSummary(levels)} + onRemove={() => del(["levels", "cursor", "direction"])} + variant="secondary/small" + /> + + } + /> ); } diff --git a/apps/webapp/app/components/logs/LogsRunIdFilter.tsx b/apps/webapp/app/components/logs/LogsRunIdFilter.tsx index 5c23d1a19..857e623d7 100644 --- a/apps/webapp/app/components/logs/LogsRunIdFilter.tsx +++ b/apps/webapp/app/components/logs/LogsRunIdFilter.tsx @@ -14,7 +14,7 @@ import { import { useSearchParams } from "~/hooks/useSearchParam"; import { FilterMenuProvider } from "~/components/runs/v3/SharedFilters"; -const shortcut = { key: "r" }; +const shortcut = { key: "i" }; export function LogsRunIdFilter() { const { value } = useSearchParams(); diff --git a/apps/webapp/app/components/logs/LogsSearchInput.tsx b/apps/webapp/app/components/logs/LogsSearchInput.tsx index 41871722b..fd539f66a 100644 --- a/apps/webapp/app/components/logs/LogsSearchInput.tsx +++ b/apps/webapp/app/components/logs/LogsSearchInput.tsx @@ -1,5 +1,6 @@ import { MagnifyingGlassIcon, XMarkIcon } from "@heroicons/react/20/solid"; import { useNavigate } from "@remix-run/react"; +import { motion } from "framer-motion"; import { useCallback, useEffect, useRef, useState } from "react"; import { Input } from "~/components/primitives/Input"; import { ShortcutKey } from "~/components/primitives/ShortcutKey"; @@ -52,7 +53,16 @@ export function LogsSearchInput() { return (
-
+ 0 ? "24rem" : "auto" }} + transition={{ + type: "spring", + stiffness: 300, + damping: 30, + }} + className="relative h-6 min-w-52" + > -
+ {text.length > 0 && (
+
+ {list?.retention?.wasClamped && ( + + )} + {isAdmin && ( + + )} +
+
+ ); +} + +function LogsList({ + list, }: { list: Exclude["data"]>, { error: string }>; //exclude error, it is handled - rootOnlyDefault: boolean; isAdmin: boolean; showDebug: boolean; defaultPeriod?: string; @@ -253,37 +382,50 @@ function LogsList({ // Selected log state - managed locally to avoid triggering navigation const [selectedLogId, setSelectedLogId] = useState(); - const handleDebugToggle = useCallback( - (checked: boolean) => { - const url = new URL(window.location.href); - if (checked) { - url.searchParams.set("showDebug", "true"); - } else { - url.searchParams.delete("showDebug"); - } - window.location.href = url.toString(); - }, - [] - ); + // Track which filter state (search params) the current fetcher request corresponds to + const fetcherFilterStateRef = useRef(location.search); + // Clear accumulated logs immediately when filters change (for instant visual feedback) + useEffect(() => { + setAccumulatedLogs([]); + setNextCursor(undefined); + // Close side panel when filters change to avoid showing a log that's no longer visible + setSelectedLogId(undefined); + }, [location.search]); - // Reset accumulated logs when the initial list changes (e.g., filters change) + // Populate accumulated logs when new data arrives useEffect(() => { setAccumulatedLogs(list.logs); setNextCursor(list.pagination.next); }, [list.logs, list.pagination.next]); + // Clear log parameter from URL when selectedLogId is cleared + useEffect(() => { + if (!selectedLogId) { + const url = new URL(window.location.href); + if (url.searchParams.has("log")) { + url.searchParams.delete("log"); + window.history.replaceState(null, "", url.toString()); + } + } + }, [selectedLogId]); + // Append new logs when fetcher completes (with deduplication) useEffect(() => { if (fetcher.data && fetcher.state === "idle") { + // Ignore fetcher data if it was loaded for a different filter state + if (fetcherFilterStateRef.current !== location.search) { + return; + } + const existingIds = new Set(accumulatedLogs.map((log) => log.id)); const newLogs = fetcher.data.logs.filter((log) => !existingIds.has(log.id)); if (newLogs.length > 0) { setAccumulatedLogs((prev) => [...prev, ...newLogs]); - setNextCursor(fetcher.data.pagination.next); } + setNextCursor(fetcher.data.pagination.next); } - }, [fetcher.data, fetcher.state, accumulatedLogs]); + }, [fetcher.data, fetcher.state, accumulatedLogs, location.search]); // Build resource URL for loading more const loadMoreUrl = useMemo(() => { @@ -297,27 +439,26 @@ function LogsList({ const handleLoadMore = useCallback(() => { if (loadMoreUrl && fetcher.state === "idle") { + // Store the current filter state before loading + fetcherFilterStateRef.current = location.search; fetcher.load(loadMoreUrl); } - }, [loadMoreUrl, fetcher]); + }, [loadMoreUrl, fetcher, location.search]); const selectedLog = useMemo(() => { if (!selectedLogId) return undefined; return accumulatedLogs.find((log) => log.id === selectedLogId); }, [selectedLogId, accumulatedLogs]); - const updateUrlWithLog = useCallback( - (logId: string | undefined) => { - const url = new URL(window.location.href); - if (logId) { - url.searchParams.set("log", logId); - } else { - url.searchParams.delete("log"); - } - window.history.replaceState(null, "", url.toString()); - }, - [] - ); + const updateUrlWithLog = useCallback((logId: string | undefined) => { + const url = new URL(window.location.href); + if (logId) { + url.searchParams.set("log", logId); + } else { + url.searchParams.delete("log"); + } + window.history.replaceState(null, "", url.toString()); + }, []); const handleLogSelect = useCallback( (logId: string) => { @@ -339,51 +480,30 @@ function LogsList({ return ( -
- {/* Filters */} -
-
- - - -
- {isAdmin && ( - - )} -
- - {/* Table */} - -
+ - {/* Side panel for log details */} {selectedLogId && ( <> -
}> + + +
+ } + > { + if (isAdmin || isImpersonating) { + return true; + } + + const organization = await prisma.organization.findFirst({ + where: { + slug: organizationSlug, + members: { some: { userId } }, + }, + select: { + featureFlags: true, + }, + }); + + if (!organization?.featureFlags) { + return false; + } + + const flags = organization.featureFlags as Record; + const hasLogsPageAccessResult = validateFeatureFlagValue( + FEATURE_FLAG.hasLogsPageAccess, + flags.hasLogsPageAccess + ); + + return hasLogsPageAccessResult.success && hasLogsPageAccessResult.data === true; +} + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const user = await requireUser(request); + const { organizationSlug } = OrganizationParamsSchema.parse(params); + + const canViewLogsPage = user.admin || user.isImpersonating || await hasLogsPageAccess( + user.id, + user.admin, + user.isImpersonating, + organizationSlug + ); + + return typedjson({ canViewLogsPage }); +}; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts index fd6f1c1a6..656e20472 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts @@ -4,13 +4,13 @@ import { requireUser, requireUserId } from "~/services/session.server"; import { EnvironmentParamSchema } from "~/utils/pathBuilder"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; -import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server"; import { $replica } from "~/db.server"; import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getCurrentPlan } from "~/services/platform.v3.server"; // Valid log levels for filtering -const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]; +const validLevels: LogLevel[] = ["DEBUG", "INFO", "WARN", "ERROR"]; function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { const levelParams = url.searchParams.getAll("levels").filter((v) => v.length > 0); @@ -19,7 +19,10 @@ function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { } export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); + const user = await requireUser(request); + const userId = user.id; + const isAdmin = user?.admin || user?.isImpersonating; + const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); const project = await findProjectBySlug(organizationSlug, projectParam, userId); @@ -32,28 +35,41 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Environment not found", { status: 404 }); } - const user = await requireUser(request); - const isAdmin = user?.admin || user?.isImpersonating; + // Get the user's plan to determine log retention limit + const plan = await getCurrentPlan(project.organizationId); + const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30; - const filters = await getRunFiltersFromRequest(request); - - // Get search term, cursor, levels, and showDebug from query params + // Get filters from query params const url = new URL(request.url); + const tasks = url.searchParams.getAll("tasks").filter((t) => t.length > 0); + const runId = url.searchParams.get("runId") ?? undefined; const search = url.searchParams.get("search") ?? undefined; const cursor = url.searchParams.get("cursor") ?? undefined; const levels = parseLevelsFromUrl(url); const showDebug = url.searchParams.get("showDebug") === "true"; + const period = url.searchParams.get("period") ?? undefined; + const fromStr = url.searchParams.get("from"); + const toStr = url.searchParams.get("to"); + let from = fromStr ? parseInt(fromStr, 10) : undefined; + let to = toStr ? parseInt(toStr, 10) : undefined; + if (Number.isNaN(from)) from = undefined; + if (Number.isNaN(to)) to = undefined; const options = LogsListOptionsSchema.parse({ userId, projectId: project.id, - ...filters, + tasks: tasks.length > 0 ? tasks : undefined, + runId, search, cursor, + period, + from, + to, levels, includeDebugLogs: isAdmin && showDebug, defaultPeriod: "1h", + retentionLimitDays, }) as any; // Validated by LogsListOptionsSchema at runtime const presenter = new LogsListPresenter($replica, clickhouseClient); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index 6c6222e7c..bd186dcea 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -73,6 +73,7 @@ import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import { useSearchParams } from "~/hooks/useSearchParam"; import { useHasAdminAccess } from "~/hooks/useUser"; +import { useCanViewLogsPage } from "~/hooks/useCanViewLogsPage"; import { redirectWithErrorMessage } from "~/models/message.server"; import { type Span, SpanPresenter, type SpanRun } from "~/presenters/v3/SpanPresenter.server"; import { logger } from "~/services/logger.server"; @@ -319,6 +320,7 @@ function RunBody({ const { value, replace } = useSearchParams(); const tab = value("tab"); const resetFetcher = useTypedFetcher(); + const canViewLogsPage = useCanViewLogsPage(); return (
@@ -1012,44 +1014,55 @@ function RunBody({
{run.logsDeletedAt === null ? ( -
+ canViewLogsPage ? ( +
+ + View logs + + + + + + + + + + +
+ ) : ( - View logs + Download logs - - - - - - - - - -
+ ) ) : null}
diff --git a/apps/webapp/app/utils/logUtils.ts b/apps/webapp/app/utils/logUtils.ts index b4a130b8e..cad9bbc90 100644 --- a/apps/webapp/app/utils/logUtils.ts +++ b/apps/webapp/app/utils/logUtils.ts @@ -1,10 +1,10 @@ import { createElement, Fragment, type ReactNode } from "react"; import { z } from "zod"; -export const LogLevelSchema = z.enum(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]); +export const LogLevelSchema = z.enum(["DEBUG", "INFO", "WARN", "ERROR"]); export type LogLevel = z.infer; -export const validLogLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "CANCELLED"]; +export const validLogLevels: LogLevel[] = ["DEBUG", "INFO", "WARN", "ERROR",]; // Default styles for search highlighting const DEFAULT_HIGHLIGHT_STYLES: React.CSSProperties = { @@ -71,10 +71,6 @@ export function highlightSearchText( // Convert ClickHouse kind to display level export function kindToLevel(kind: string, status: string): LogLevel { - if (status === "CANCELLED") { - return "CANCELLED"; - } - // ERROR can come from either kind or status if (kind === "LOG_ERROR" || status === "ERROR") { return "ERROR"; @@ -94,7 +90,7 @@ export function kindToLevel(kind: string, status: string): LogLevel { case "ANCESTOR_OVERRIDE": case "SPAN_EVENT": default: - return "TRACE"; + return "INFO"; } } @@ -109,47 +105,7 @@ export function getLevelColor(level: LogLevel): string { return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; case "INFO": return "text-blue-400 bg-blue-500/10 border-blue-500/20"; - case "TRACE": - return "text-charcoal-500 bg-charcoal-800 border-charcoal-700"; - case "CANCELLED": - return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; default: return "text-text-dimmed bg-charcoal-750 border-charcoal-700"; } } - -// Event kind badge color styles -export function getKindColor(kind: string): string { - if (kind === "SPAN") { - return "text-purple-400 bg-purple-500/10 border-purple-500/20"; - } - if (kind === "SPAN_EVENT") { - return "text-amber-400 bg-amber-500/10 border-amber-500/20"; - } - if (kind.startsWith("LOG_")) { - return "text-blue-400 bg-blue-500/10 border-blue-500/20"; - } - return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; -} - -// Get human readable kind label -export function getKindLabel(kind: string): string { - switch (kind) { - case "SPAN": - return "Span"; - case "SPAN_EVENT": - return "Event"; - case "LOG_DEBUG": - case "LOG_INFO": - case "LOG_WARN": - case "LOG_ERROR": - case "LOG_LOG": - return "Log"; - case "DEBUG_EVENT": - return "Debug"; - case "ANCESTOR_OVERRIDE": - return "Override"; - default: - return kind; - } -} diff --git a/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql b/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql new file mode 100644 index 000000000..1d6af3294 --- /dev/null +++ b/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql @@ -0,0 +1,20 @@ +-- +goose Up + +-- Add indexes for text search on task task_events_v2 tables for message and attributes fields +ALTER TABLE trigger_dev.task_events_v2 + ADD INDEX IF NOT EXISTS idx_attributes_text_search lower(attributes_text) + TYPE ngrambf_v1(3, 32768, 2, 0) + GRANULARITY 1; + +ALTER TABLE trigger_dev.task_events_v2 + ADD INDEX IF NOT EXISTS idx_message_text_search lower(message) + TYPE ngrambf_v1(3, 32768, 2, 0) + GRANULARITY 1; + +-- +goose Down + +ALTER TABLE trigger_dev.task_events_v2 +DROP INDEX idx_attributes_text_search; + +ALTER TABLE trigger_dev.task_events_v2 +DROP INDEX idx_message_text_search; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index acb3c56a8..4f4cb5e3b 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -25,8 +25,6 @@ import { insertTaskEventsV2, getLogsListQueryBuilderV2, getLogDetailQueryBuilderV2, - getLogsListQueryBuilderV1, - getLogDetailQueryBuilderV1, } from "./taskEvents.js"; import { Logger, type LogLevel } from "@trigger.dev/core/logger"; import type { Agent as HttpAgent } from "http"; @@ -213,8 +211,6 @@ export class ClickHouse { traceSummaryQueryBuilder: getTraceSummaryQueryBuilder(this.reader), traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilder(this.reader), spanDetailsQueryBuilder: getSpanDetailsQueryBuilder(this.reader), - logsListQueryBuilder: getLogsListQueryBuilderV1(this.reader, this.logsQuerySettings?.list), - logDetailQueryBuilder: getLogDetailQueryBuilderV1(this.reader, this.logsQuerySettings?.detail), }; } diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index fa64a908d..890eab9cc 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -320,56 +320,4 @@ export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: Clic ], settings, }); -} - -// ============================================================================ -// Logs List Query Builders for V1 (task_events_v1) -// ============================================================================ - -export function getLogsListQueryBuilderV1(ch: ClickhouseReader, settings?: ClickHouseSettings) { - return ch.queryBuilderFast({ - name: "getLogsListV1", - table: "trigger_dev.task_events_v1", - columns: [ - "environment_id", - "organization_id", - "project_id", - "task_identifier", - "run_id", - "start_time", - "trace_id", - "span_id", - "parent_span_id", - { name: "message", expression: "LEFT(message, 512)" }, - "kind", - "status", - "duration", - "attributes_text" - ], - settings, - }); -} - -export function getLogDetailQueryBuilderV1(ch: ClickhouseReader, settings?: ClickHouseSettings) { - return ch.queryBuilderFast({ - name: "getLogDetailV1", - table: "trigger_dev.task_events_v1", - columns: [ - "environment_id", - "organization_id", - "project_id", - "task_identifier", - "run_id", - "start_time", - "trace_id", - "span_id", - "parent_span_id", - "message", - "kind", - "status", - "duration", - "attributes_text", - ], - settings, - }); -} +} \ No newline at end of file From c0b86efbd3d9f0577c1b3a5803bce0f43c131c12 Mon Sep 17 00:00:00 2001 From: Oskar Otwinowski Date: Thu, 29 Jan 2026 12:51:58 +0100 Subject: [PATCH 02/22] feat(webapp): Add MiddleTruncate component for long task names (#2946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes # ## ✅ Checklist - [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [ ] The PR title follows the convention. - [ ] I ran and tested the code works --- ## Testing Tested the MiddleTruncate component in the TasksDropdown by: 1. Verifying that long task names (e.g., "namespace:category:subcategory:task-name") are truncated in the middle 2. Confirming the full text appears in a tooltip on hover 3. Testing responsive behavior - truncation adjusts when the container is resized 4. Verifying that short task names that fit within the container are displayed in full without truncation --- ## Changelog Added a new `MiddleTruncate` primitive component that intelligently truncates text in the middle while preserving the beginning and end portions. This is particularly useful for long hierarchical identifiers like task slugs. **Key features:** - Truncates text in the middle with an ellipsis (…) when it exceeds available width - Shows full text in a tooltip on hover when truncated - Responsive - recalculates truncation on container resize using ResizeObserver - Maintains minimum character visibility (4 chars minimum on each side for readability) - Integrated into TasksDropdown to handle long task names **Changes:** - Created new `MiddleTruncate.tsx` component with binary search algorithm for optimal character distribution - Updated TasksDropdown to use MiddleTruncate for task slug display - Increased TasksDropdown popover width from 240px to 360px to provide better space for truncated text --- ## Screenshots 💯 https://github.com/user-attachments/assets/a7a2191a-2e36-437e-ab3f-517fe7620b93 --- Open with Devin --------- Co-authored-by: Claude --- .../components/primitives/MiddleTruncate.tsx | 168 ++++++++++++++++++ .../app/components/runs/v3/RunFilters.tsx | 5 +- 2 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 apps/webapp/app/components/primitives/MiddleTruncate.tsx diff --git a/apps/webapp/app/components/primitives/MiddleTruncate.tsx b/apps/webapp/app/components/primitives/MiddleTruncate.tsx new file mode 100644 index 000000000..c116205ae --- /dev/null +++ b/apps/webapp/app/components/primitives/MiddleTruncate.tsx @@ -0,0 +1,168 @@ +import { useRef, useState, useLayoutEffect, useCallback } from "react"; +import { cn } from "~/utils/cn"; +import { SimpleTooltip } from "./Tooltip"; + +type MiddleTruncateProps = { + text: string; + className?: string; +}; + +/** + * A component that truncates text in the middle, showing the beginning and end. + * Shows the full text in a tooltip on hover when truncated. + * + * Example: "namespace:category:subcategory:task-name" becomes "namespace:cat…task-name" + */ +export function MiddleTruncate({ text, className }: MiddleTruncateProps) { + const containerRef = useRef(null); + const measureRef = useRef(null); + const [displayText, setDisplayText] = useState(text); + const [isTruncated, setIsTruncated] = useState(false); + + const calculateTruncation = useCallback(() => { + const container = containerRef.current; + const measure = measureRef.current; + if (!container || !measure) return; + + const parent = container.parentElement; + if (!parent) return; + + // Get the available width from the parent container + const parentStyle = getComputedStyle(parent); + const availableWidth = + parent.clientWidth - + parseFloat(parentStyle.paddingLeft) - + parseFloat(parentStyle.paddingRight); + + // Measure full text width + measure.textContent = text; + const fullTextWidth = measure.offsetWidth; + + // If text fits, no truncation needed + if (fullTextWidth <= availableWidth) { + setDisplayText(text); + setIsTruncated(false); + return; + } + + // Text needs truncation - find optimal split + const ellipsis = "…"; + measure.textContent = ellipsis; + const ellipsisWidth = measure.offsetWidth; + + const targetWidth = availableWidth - ellipsisWidth - 4; // small buffer + + if (targetWidth <= 0) { + setDisplayText(ellipsis); + setIsTruncated(true); + return; + } + + // Incrementally find the optimal character counts + let startChars = 0; + let endChars = 0; + + // Alternate adding characters from start and end + while (startChars + endChars < text.length) { + // Try adding to start + const testStart = text.slice(0, startChars + 1); + const testEnd = endChars > 0 ? text.slice(-endChars) : ""; + measure.textContent = testStart + ellipsis + testEnd; + + if (measure.offsetWidth > targetWidth) break; + startChars++; + + if (startChars + endChars >= text.length) break; + + // Try adding to end + const newTestEnd = text.slice(-(endChars + 1)); + measure.textContent = text.slice(0, startChars) + ellipsis + newTestEnd; + + if (measure.offsetWidth > targetWidth) break; + endChars++; + } + + // Ensure minimum characters on each side for readability + const minChars = 4; + const prevStartChars = startChars; + const prevEndChars = endChars; + + if (startChars < minChars && text.length > minChars * 2 + 1) { + startChars = minChars; + } + if (endChars < minChars && text.length > minChars * 2 + 1) { + endChars = minChars; + } + + // Re-measure after enforcing minChars to prevent overflow + if (startChars !== prevStartChars || endChars !== prevEndChars) { + measure.textContent = text.slice(0, startChars) + ellipsis + text.slice(-endChars); + if (measure.offsetWidth > targetWidth) { + // Revert to previous values if minChars enforcement causes overflow + startChars = prevStartChars; + endChars = prevEndChars; + } + } + + // If combined chars would exceed text length, show full text + if (startChars + endChars >= text.length) { + setDisplayText(text); + setIsTruncated(false); + return; + } + + const result = text.slice(0, startChars) + ellipsis + text.slice(-endChars); + setDisplayText(result); + setIsTruncated(true); + }, [text]); + + useLayoutEffect(() => { + calculateTruncation(); + + // Recalculate on resize (guard for jsdom/older browsers) + if (typeof ResizeObserver === "undefined") { + return; + } + + const resizeObserver = new ResizeObserver(() => { + calculateTruncation(); + }); + + const container = containerRef.current; + if (container?.parentElement) { + resizeObserver.observe(container.parentElement); + } + + return () => { + resizeObserver.disconnect(); + }; + }, [calculateTruncation]); + + const content = ( + + {/* Hidden span for measuring text width */} + + ); + + if (isTruncated) { + return ( + {text}} + side="top" + asChild + /> + ); + } + + return content; +} diff --git a/apps/webapp/app/components/runs/v3/RunFilters.tsx b/apps/webapp/app/components/runs/v3/RunFilters.tsx index 569508181..cff56573a 100644 --- a/apps/webapp/app/components/runs/v3/RunFilters.tsx +++ b/apps/webapp/app/components/runs/v3/RunFilters.tsx @@ -31,6 +31,7 @@ import { DateTime } from "~/components/primitives/DateTime"; import { FormError } from "~/components/primitives/FormError"; import { Input } from "~/components/primitives/Input"; import { Label } from "~/components/primitives/Label"; +import { MiddleTruncate } from "~/components/primitives/MiddleTruncate"; import { Paragraph } from "~/components/primitives/Paragraph"; import { ComboBox, @@ -634,7 +635,7 @@ function TasksDropdown({ {trigger} { if (onClose) { onClose(); @@ -654,7 +655,7 @@ function TasksDropdown({ } > - {item.slug} + ))} From f53db6fd164aa4c591d09523b8316786d20531fb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 29 Jan 2026 13:02:47 +0000 Subject: [PATCH 03/22] Query: time limits, performance improvements, styling (#2953) Summary - Query: add time limits, performance improvements, and styling updates Changes - Add ClickHouse output_text and error_text columns with indexes - Automatically use _text columns for JSON based on query pattern; support JSON column data prefixes - Add idempotency key and scope columns - Add enforcedWhereClause for tenant and time restrictions, instead of the old tenant stuff. - Implement basic time filter limiting and set default time period based on plan; show message when results are clipped - UX: resizable code area (including vertical splits), collapsible sidebar, fix table/chart vertical sizing, max height for chart legend in fullscreen - Styling and UI tweaks: improved chart legend styling, more chart colours, thinner line chart stroke, pricing callout color, improved layout for callouts - Features: generate and save AI titles --- Open with Devin --- .../app/components/code/QueryResultsChart.tsx | 32 +- .../app/components/code/TSQLResultsTable.tsx | 5 +- .../components/primitives/AnimatedNumber.tsx | 61 +- .../app/components/primitives/Callout.tsx | 9 +- .../app/components/primitives/Resizable.tsx | 35 +- .../app/components/primitives/charts/Card.tsx | 4 +- .../primitives/charts/ChartLegendCompound.tsx | 119 +-- .../primitives/charts/ChartLine.tsx | 4 +- .../primitives/charts/ChartRoot.tsx | 7 + .../app/components/runs/v3/SharedFilters.tsx | 156 ++-- apps/webapp/app/env.server.ts | 2 +- .../presenters/v3/QueryPresenter.server.ts | 4 + .../ExamplesContent.tsx | 3 +- .../QueryHelpSidebar.tsx | 89 ++- .../QueryHistoryPopover.tsx | 34 +- .../route.tsx | 542 +++++++++++--- .../utils.ts | 32 - ...jectParam.env.$envParam.query.ai-title.tsx | 81 ++ .../app/services/queryService.server.ts | 101 ++- apps/webapp/app/v3/querySchemas.ts | 14 +- .../v3/services/aiQueryTitleService.server.ts | 71 ++ apps/webapp/package.json | 2 +- ...date_output_error_text_to_extract_data.sql | 45 ++ .../clickhouse/src/client/tsql.ts | 86 ++- internal-packages/clickhouse/src/index.ts | 2 +- internal-packages/clickhouse/src/tsql.test.ts | 250 ++++--- .../migration.sql | 7 + .../database/prisma/schema.prisma | 4 +- internal-packages/tsql/src/index.test.ts | 373 ++++++++- internal-packages/tsql/src/index.ts | 105 +-- .../tsql/src/query/printer.test.ts | 707 +++++++++++++++--- internal-packages/tsql/src/query/printer.ts | 520 +++++++++++-- .../tsql/src/query/printer_context.ts | 86 ++- internal-packages/tsql/src/query/schema.ts | 36 + .../tsql/src/query/security.test.ts | 147 +++- pnpm-lock.yaml | 10 +- 36 files changed, 2987 insertions(+), 798 deletions(-) delete mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts create mode 100644 apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx create mode 100644 apps/webapp/app/v3/services/aiQueryTitleService.server.ts create mode 100644 internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql create mode 100644 internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql diff --git a/apps/webapp/app/components/code/QueryResultsChart.tsx b/apps/webapp/app/components/code/QueryResultsChart.tsx index bde823af1..26a34722d 100644 --- a/apps/webapp/app/components/code/QueryResultsChart.tsx +++ b/apps/webapp/app/components/code/QueryResultsChart.tsx @@ -5,9 +5,10 @@ import { Chart } from "~/components/primitives/charts/ChartCompound"; import { Paragraph } from "../primitives/Paragraph"; import type { AggregationType, ChartConfiguration } from "./ChartConfigPanel"; -// Color palette for chart series +// Color palette for chart series - 30 distinct colors for large datasets const CHART_COLORS = [ - "#7655fd", // Primary purple + // Primary colors + "#7655fd", // Purple "#22c55e", // Green "#f59e0b", // Amber "#ef4444", // Red @@ -17,6 +18,28 @@ const CHART_COLORS = [ "#14b8a6", // Teal "#f97316", // Orange "#6366f1", // Indigo + // Extended palette + "#84cc16", // Lime + "#0ea5e9", // Sky + "#f43f5e", // Rose + "#a855f7", // Fuchsia + "#eab308", // Yellow + "#10b981", // Emerald + "#3b82f6", // Blue + "#d946ef", // Magenta + "#78716c", // Stone + "#facc15", // Gold + // Additional distinct colors + "#2dd4bf", // Turquoise + "#fb923c", // Light orange + "#a3e635", // Yellow-green + "#38bdf8", // Light blue + "#c084fc", // Light purple + "#4ade80", // Light green + "#fbbf24", // Light amber + "#f472b6", // Light pink + "#67e8f9", // Light cyan + "#818cf8", // Light indigo ]; function getSeriesColor(index: number): string { @@ -30,6 +53,8 @@ interface QueryResultsChartProps { fullLegend?: boolean; /** Callback when "View all" legend button is clicked */ onViewAllLegendItems?: () => void; + /** When true, constrains legend to max 50% height with scrolling */ + legendScrollable?: boolean; } interface TransformedData { @@ -702,6 +727,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ config, fullLegend = false, onViewAllLegendItems, + legendScrollable = false, }: QueryResultsChartProps) { const { xAxisColumn, @@ -872,6 +898,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({ minHeight="300px" fillContainer onViewAllLegendItems={onViewAllLegendItems} + legendScrollable={legendScrollable} > MAX_STRING_DISPLAY_LENGTH; if (isTruncated) { @@ -1137,6 +1139,7 @@ export const TSQLResultsTable = memo(function TSQLResultsTable({ height: `${rowVirtualizer.getTotalSize()}px`, position: "relative", }} + className="bg-background-dimmed divide-y divide-charcoal-700" > {rowVirtualizer.getVirtualItems().map((virtualRow) => { const row = tableRows[virtualRow.index]; diff --git a/apps/webapp/app/components/primitives/AnimatedNumber.tsx b/apps/webapp/app/components/primitives/AnimatedNumber.tsx index 2d1ff7ea7..fea0f9d89 100644 --- a/apps/webapp/app/components/primitives/AnimatedNumber.tsx +++ b/apps/webapp/app/components/primitives/AnimatedNumber.tsx @@ -1,9 +1,64 @@ import { animate, motion, useMotionValue, useTransform } from "framer-motion"; -import { useEffect } from "react"; +import { useEffect, useMemo } from "react"; -export function AnimatedNumber({ value, duration = 0.5 }: { value: number; duration?: number }) { +/** + * Determines the number of decimal places to display based on the value. + * - For integers or large numbers (>=100), no decimals + * - For numbers >= 10, 1 decimal place + * - For numbers >= 1, 2 decimal places + * - For smaller numbers, up to 4 decimal places + */ +function getDecimalPlaces(value: number): number { + if (Number.isInteger(value)) return 0; + + const absValue = Math.abs(value); + if (absValue >= 100) return 0; + if (absValue >= 10) return 1; + if (absValue >= 1) return 2; + if (absValue >= 0.1) return 3; + return 4; +} + +/** + * Sanitizes a decimal places value to ensure it's valid for toLocaleString. + * - Coerces to a finite number (handles NaN, Infinity, -Infinity) + * - Rounds to an integer + * - Clamps to the valid 0-20 range for toLocaleString options + */ +function sanitizeDecimals(decimals: number): number { + if (!Number.isFinite(decimals)) { + return 0; + } + return Math.min(20, Math.max(0, Math.round(decimals))); +} + +export function AnimatedNumber({ + value, + duration = 0.5, + decimalPlaces, +}: { + value: number; + duration?: number; + /** Number of decimal places to display. If not provided, auto-detects based on value. */ + decimalPlaces?: number; +}) { const motionValue = useMotionValue(value); - let display = useTransform(motionValue, (current) => Math.round(current).toLocaleString()); + + // Determine decimal places - use provided value or auto-detect, then sanitize + const safeDecimals = useMemo(() => { + const rawDecimals = decimalPlaces !== undefined ? decimalPlaces : getDecimalPlaces(value); + return sanitizeDecimals(rawDecimals); + }, [decimalPlaces, value]); + + const display = useTransform(motionValue, (current) => { + if (safeDecimals === 0) { + return Math.round(current).toLocaleString(); + } + return current.toLocaleString(undefined, { + minimumFractionDigits: safeDecimals, + maximumFractionDigits: safeDecimals, + }); + }); useEffect(() => { animate(motionValue, value, { diff --git a/apps/webapp/app/components/primitives/Callout.tsx b/apps/webapp/app/components/primitives/Callout.tsx index 207ad134b..da2d2ea76 100644 --- a/apps/webapp/app/components/primitives/Callout.tsx +++ b/apps/webapp/app/components/primitives/Callout.tsx @@ -1,4 +1,5 @@ import { + CreditCardIcon, ExclamationCircleIcon, ExclamationTriangleIcon, InformationCircleIcon, @@ -60,10 +61,10 @@ export const variantClasses = { linkClassName: "transition hover:bg-blue-400/20", }, pricing: { - className: "border-charcoal-700 bg-charcoal-800", - icon: , - textColor: "text-text-bright", - linkClassName: "transition hover:bg-charcoal-750", + className: "border-indigo-400/20 bg-indigo-800/30", + icon: , + textColor: "text-indigo-300", + linkClassName: "transition hover:bg-indigo-400/20", }, } as const; diff --git a/apps/webapp/app/components/primitives/Resizable.tsx b/apps/webapp/app/components/primitives/Resizable.tsx index 22fb38d35..830cd0118 100644 --- a/apps/webapp/app/components/primitives/Resizable.tsx +++ b/apps/webapp/app/components/primitives/Resizable.tsx @@ -26,19 +26,40 @@ const ResizableHandle = ({ }) => ( div]:rotate-90", + // Base styles + "group relative flex items-center justify-center focus-custom", + // Horizontal orientation (default) + "w-0.75 after:absolute after:inset-y-0 after:left-1/2 after:w-1 after:-translate-x-1/2", + // Vertical orientation + "data-[handle-orientation=vertical]:h-0.75 data-[handle-orientation=vertical]:w-full", + "data-[handle-orientation=vertical]:after:inset-x-0 data-[handle-orientation=vertical]:after:inset-y-auto", + "data-[handle-orientation=vertical]:after:top-1/2 data-[handle-orientation=vertical]:after:left-0", + "data-[handle-orientation=vertical]:after:h-1 data-[handle-orientation=vertical]:after:w-full", + "data-[handle-orientation=vertical]:after:-translate-y-1/2 data-[handle-orientation=vertical]:after:translate-x-0", className )} size="3px" {...props} > -
+ {/* Horizontal orientation line indicator */} +
+ {/* Vertical orientation line indicator */} +
{withHandle && ( -
- {Array.from({ length: 3 }).map((_, index) => ( -
- ))} -
+ <> + {/* Horizontal orientation dots (vertical arrangement) */} +
+ {Array.from({ length: 3 }).map((_, index) => ( +
+ ))} +
+ {/* Vertical orientation dots (horizontal arrangement) */} +
+ {Array.from({ length: 3 }).map((_, index) => ( +
+ ))} +
+ )} ); diff --git a/apps/webapp/app/components/primitives/charts/Card.tsx b/apps/webapp/app/components/primitives/charts/Card.tsx index 429c51e33..c618b51d0 100644 --- a/apps/webapp/app/components/primitives/charts/Card.tsx +++ b/apps/webapp/app/components/primitives/charts/Card.tsx @@ -6,7 +6,7 @@ export const Card = ({ children, className }: { children: ReactNode; className?: return (
@@ -17,7 +17,7 @@ export const Card = ({ children, className }: { children: ReactNode; className?: const CardHeader = ({ children }: { children: ReactNode }) => { return ( - {children} + {children} ); }; diff --git a/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx b/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx index 08a5abc98..1ab1bb855 100644 --- a/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx @@ -17,6 +17,8 @@ export type ChartLegendCompoundProps = { totalLabel?: string; /** Callback when "View all" button is clicked */ onViewAllLegendItems?: () => void; + /** When true, constrains legend to max 50% height with scrolling */ + scrollable?: boolean; }; /** @@ -37,6 +39,7 @@ export function ChartLegendCompound({ className, totalLabel = "Total", onViewAllLegendItems, + scrollable = false, }: ChartLegendCompoundProps) { const { config, dataKey, dataKeys, highlight, labelFormatter } = useChartContext(); const totals = useSeriesTotal(); @@ -128,11 +131,17 @@ export function ChartLegendCompound({ const isHovering = (highlight.activePayload?.length ?? 0) > 0; return ( -
+
{/* Total row */}
@@ -143,62 +152,68 @@ export function ChartLegendCompound({
{/* Separator */} -
+
- {legendItems.visible.map((item) => { - const total = currentData[item.dataKey] ?? 0; - const isActive = highlight.activeBarKey === item.dataKey; + {/* Legend items - scrollable when scrollable prop is true */} +
+ {legendItems.visible.map((item) => { + const total = currentData[item.dataKey] ?? 0; + const isActive = highlight.activeBarKey === item.dataKey; - return ( -
highlight.setHoveredLegendItem(item.dataKey)} - onMouseLeave={() => highlight.reset()} - > - {/* Active highlight background */} - {isActive && item.color && ( -
- )} -
-
- {item.color && ( -
- )} - - {item.label} + return ( +
highlight.setHoveredLegendItem(item.dataKey)} + onMouseLeave={() => highlight.reset()} + > + {/* Active highlight background */} + {isActive && item.color && ( +
+ )} +
+
+ {item.color && ( +
+ )} + + {item.label} + +
+ +
- - -
-
- ); - })} + ); + })} - {/* View more row - replaced by hovered hidden item when applicable */} - {legendItems.remaining > 0 && - (legendItems.hoveredHiddenItem ? ( - - ) : ( - - ))} + {/* View more row - replaced by hovered hidden item when applicable */} + {legendItems.remaining > 0 && + (legendItems.hoveredHiddenItem ? ( + + ) : ( + + ))} +
); } diff --git a/apps/webapp/app/components/primitives/charts/ChartLine.tsx b/apps/webapp/app/components/primitives/charts/ChartLine.tsx index f6bc220af..9148727ca 100644 --- a/apps/webapp/app/components/primitives/charts/ChartLine.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartLine.tsx @@ -172,7 +172,7 @@ export function ChartLineRenderer({ stroke={config[key]?.color} fill={config[key]?.color} fillOpacity={0.6} - strokeWidth={2} + strokeWidth={1} stackId="stack" isAnimationActive={false} /> @@ -220,7 +220,7 @@ export function ChartLineRenderer({ dataKey={key} type={lineType} stroke={config[key]?.color} - strokeWidth={2} + strokeWidth={1} dot={false} activeDot={{ r: 4 }} isAnimationActive={false} diff --git a/apps/webapp/app/components/primitives/charts/ChartRoot.tsx b/apps/webapp/app/components/primitives/charts/ChartRoot.tsx index 8e3810adc..d1496a2ff 100644 --- a/apps/webapp/app/components/primitives/charts/ChartRoot.tsx +++ b/apps/webapp/app/components/primitives/charts/ChartRoot.tsx @@ -31,6 +31,8 @@ export type ChartRootProps = { legendTotalLabel?: string; /** Callback when "View all" legend button is clicked */ onViewAllLegendItems?: () => void; + /** When true, constrains legend to max 50% height with scrolling */ + legendScrollable?: boolean; /** When true, chart fills its parent container height and distributes space between chart and legend */ fillContainer?: boolean; children: React.ComponentProps["children"]; @@ -72,6 +74,7 @@ export function ChartRoot({ maxLegendItems = 5, legendTotalLabel, onViewAllLegendItems, + legendScrollable = false, fillContainer = false, children, }: ChartRootProps) { @@ -94,6 +97,7 @@ export function ChartRoot({ maxLegendItems={maxLegendItems} legendTotalLabel={legendTotalLabel} onViewAllLegendItems={onViewAllLegendItems} + legendScrollable={legendScrollable} fillContainer={fillContainer} > {children} @@ -109,6 +113,7 @@ type ChartRootInnerProps = { maxLegendItems?: number; legendTotalLabel?: string; onViewAllLegendItems?: () => void; + legendScrollable?: boolean; fillContainer?: boolean; children: React.ComponentProps["children"]; }; @@ -120,6 +125,7 @@ function ChartRootInner({ maxLegendItems = 5, legendTotalLabel, onViewAllLegendItems, + legendScrollable = false, fillContainer = false, children, }: ChartRootInnerProps) { @@ -160,6 +166,7 @@ function ChartRootInner({ maxItems={maxLegendItems} totalLabel={legendTotalLabel} onViewAllLegendItems={onViewAllLegendItems} + scrollable={legendScrollable} /> )}
diff --git a/apps/webapp/app/components/runs/v3/SharedFilters.tsx b/apps/webapp/app/components/runs/v3/SharedFilters.tsx index bf4497ca8..b7675eb83 100644 --- a/apps/webapp/app/components/runs/v3/SharedFilters.tsx +++ b/apps/webapp/app/components/runs/v3/SharedFilters.tsx @@ -4,36 +4,31 @@ import { endOfDay, endOfMonth, endOfWeek, - isSaturday, - isSunday, - previousSaturday, startOfDay, startOfMonth, startOfWeek, - startOfYear, subDays, - subMonths, - subWeeks, + subWeeks } from "date-fns"; import parse from "parse-duration"; import { startTransition, useCallback, useEffect, useState, type ReactNode } from "react"; +import simplur from "simplur"; import { AppliedFilter } from "~/components/primitives/AppliedFilter"; +import { Callout } from "~/components/primitives/Callout"; import { DateTime } from "~/components/primitives/DateTime"; import { DateTimePicker } from "~/components/primitives/DateTimePicker"; import { Label } from "~/components/primitives/Label"; import { Paragraph } from "~/components/primitives/Paragraph"; import { RadioButtonCircle } from "~/components/primitives/RadioButton"; import { ComboboxProvider, SelectPopover, SelectProvider } from "~/components/primitives/Select"; +import { useOptionalOrganization } from "~/hooks/useOrganizations"; import { useSearchParams } from "~/hooks/useSearchParam"; import { type ShortcutDefinition } from "~/hooks/useShortcutKeys"; import { cn } from "~/utils/cn"; -import { Button } from "../../primitives/Buttons"; +import { organizationBillingPath } from "~/utils/pathBuilder"; +import { Button, LinkButton } from "../../primitives/Buttons"; import { filterIcon } from "./RunFilters"; -export type DisplayableEnvironment = Pick & { - userName?: string; -}; - export function FilterMenuProvider({ children, onClose, @@ -95,6 +90,10 @@ const timePeriods = [ label: "3 days", value: "3d", }, + { + label: "5 days", + value: "5d", + }, { label: "7 days", value: "7d", @@ -106,11 +105,7 @@ const timePeriods = [ { label: "30 days", value: "30d", - }, - { - label: "90 days", - value: "90d", - }, + } ]; const timeUnits = [ @@ -128,6 +123,22 @@ function parsePeriodString(period: string): { value: number; unit: string } | nu return null; } +const MS_PER_DAY = 1000 * 60 * 60 * 24; + +// Convert a period string to days using parse-duration +function periodToDays(period: string): number { + const ms = parse(period); + if (!ms) return 0; + return ms / MS_PER_DAY; +} + +// Calculate the number of days a date range spans from now +function dateRangeToDays(from?: Date): number { + if (!from) return 0; + const now = new Date(); + return Math.ceil((now.getTime() - from.getTime()) / MS_PER_DAY); +} + const DEFAULT_PERIOD = "7d"; const defaultPeriodMs = parse(DEFAULT_PERIOD); if (!defaultPeriodMs) { @@ -292,6 +303,8 @@ export interface TimeFilterProps { applyShortcut?: ShortcutDefinition | undefined; /** Callback when the user applies a time filter selection, receives the applied values */ onValueChange?: (values: TimeFilterApplyValues) => void; + /** When set an upgrade message will be shown if you select a period further back than this number of days */ + maxPeriodDays?: number; } export function TimeFilter({ @@ -303,6 +316,7 @@ export function TimeFilter({ hideLabel = false, applyShortcut, onValueChange, + maxPeriodDays, }: TimeFilterProps = {}) { const { value } = useSearchParams(); const periodValue = period ?? value("period"); @@ -339,6 +353,7 @@ export function TimeFilter({ labelName={labelName} applyShortcut={applyShortcut} onValueChange={onValueChange} + maxPeriodDays={maxPeriodDays} /> )} @@ -356,6 +371,8 @@ function getInitialCustomDuration(period?: string): { value: string; unit: strin return { value: "", unit: "m" }; } +type SectionType = "duration" | "dateRange"; + export function TimeDropdown({ trigger, period, @@ -366,6 +383,7 @@ export function TimeDropdown({ applyShortcut, onApply, onValueChange, + maxPeriodDays, }: { trigger: ReactNode; period?: string; @@ -377,14 +395,16 @@ export function TimeDropdown({ onApply?: (values: TimeFilterApplyValues) => void; /** When provided, the component operates in controlled mode and skips URL navigation */ onValueChange?: (values: TimeFilterApplyValues) => void; + /** When set an upgrade message will be shown if you select a period further back than this number of days */ + maxPeriodDays?: number; }) { + const organization = useOptionalOrganization(); const [open, setOpen] = useState(); const { replace } = useSearchParams(); const [fromValue, setFromValue] = useState(from); const [toValue, setToValue] = useState(to); // Section selection state: "duration" or "dateRange" - type SectionType = "duration" | "dateRange"; const initialSection: SectionType = from || to ? "dateRange" : "duration"; const [activeSection, setActiveSection] = useState(initialSection); const [validationError, setValidationError] = useState(null); @@ -418,9 +438,28 @@ export function TimeDropdown({ return !isNaN(value) && value > 0; })(); + // Calculate if the current selection exceeds maxPeriodDays + const exceedsMaxPeriod = (() => { + if (!maxPeriodDays) return false; + + if (activeSection === "duration") { + const periodToCheck = selectedPeriod === "custom" ? `${customValue}${customUnit}` : selectedPeriod; + if (!periodToCheck) return false; + return periodToDays(periodToCheck) > maxPeriodDays; + } else { + // For date range, check if fromValue is further back than maxPeriodDays + return dateRangeToDays(fromValue) > maxPeriodDays; + } + })(); + const applySelection = useCallback(() => { setValidationError(null); + if (exceedsMaxPeriod) { + setValidationError(`Your plan allows a maximum of ${maxPeriodDays} days. Upgrade for longer retention.`); + return; + } + if (activeSection === "duration") { // Validate custom duration if (selectedPeriod === "custom" && !isCustomDurationValid) { @@ -498,6 +537,8 @@ export function TimeDropdown({ replace, onApply, onValueChange, + exceedsMaxPeriod, + maxPeriodDays ]); return ( @@ -683,7 +724,7 @@ export function TimeDropdown({ />
{/* Quick select date ranges */} -
e.stopPropagation()}> +
e.stopPropagation()}> { const today = new Date(); setFromValue(startOfDay(today)); - setToValue(today); + setToValue(endOfDay(today)); setActiveSection("dateRange"); setValidationError(null); setSelectedQuickDate("today"); }} /> +
+
e.stopPropagation()}> { const now = new Date(); setFromValue(startOfWeek(now, { weekStartsOn: 1 })); - setToValue(now); + setToValue(endOfWeek(now, { weekStartsOn: 1 })); setActiveSection("dateRange"); setValidationError(null); setSelectedQuickDate("thisWeek"); }} /> - { - const now = new Date(); - let saturday: Date; - if (isSaturday(now)) { - saturday = subDays(now, 7); - } else if (isSunday(now)) { - saturday = subDays(now, 8); - } else { - saturday = previousSaturday(now); - } - const sunday = endOfDay(subDays(saturday, -1)); - setFromValue(startOfDay(saturday)); - setToValue(sunday); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("lastWeekend"); - }} - /> - { - const lastWeek = subWeeks(new Date(), 1); - const monday = startOfWeek(lastWeek, { weekStartsOn: 1 }); - const friday = endOfDay(subDays(monday, -4)); // Monday + 4 days = Friday - setFromValue(startOfDay(monday)); - setToValue(friday); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("lastWeekdays"); - }} - /> - { - const lastMonth = subMonths(new Date(), 1); - setFromValue(startOfMonth(lastMonth)); - setToValue(endOfMonth(lastMonth)); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("lastMonth"); - }} - /> { const now = new Date(); setFromValue(startOfMonth(now)); - setToValue(now); + setToValue(endOfMonth(now)); setActiveSection("dateRange"); setValidationError(null); setSelectedQuickDate("thisMonth"); }} /> - { - const now = new Date(); - setFromValue(startOfYear(now)); - setToValue(now); - setActiveSection("dateRange"); - setValidationError(null); - setSelectedQuickDate("yearToDate"); - }} - />
{validationError && activeSection === "dateRange" && ( @@ -812,6 +796,17 @@ export function TimeDropdown({
+ {/* Upgrade callout when exceeding maxPeriodDays */} + {exceedsMaxPeriod && organization && ( + Upgrade} + className="items-center" + > + {simplur`Your plan allows a maximum of ${maxPeriodDays} day[|s].`} + + )} + {/* Action buttons */}
diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 1dc3091f1..98c04b6f9 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -521,7 +521,6 @@ const EnvironmentSchema = z PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(), CENTS_PER_RUN: z.coerce.number().default(0), - CENTS_PER_QUERY_BYTE_SECOND: z.coerce.number().default(0), EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"), RESOURCE_MONITOR_ENABLED: z.string().default("0"), @@ -1197,6 +1196,7 @@ const EnvironmentSchema = z QUERY_CLICKHOUSE_MAX_AST_ELEMENTS: z.coerce.number().int().default(4_000_000), QUERY_CLICKHOUSE_MAX_EXPANDED_AST_ELEMENTS: z.coerce.number().int().default(4_000_000), QUERY_CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY: z.coerce.number().int().default(0), + QUERY_CLICKHOUSE_MAX_RETURNED_ROWS: z.coerce.number().int().default(10_000), // Query page concurrency limits QUERY_DEFAULT_ORG_CONCURRENCY_LIMIT: z.coerce.number().int().default(3), diff --git a/apps/webapp/app/presenters/v3/QueryPresenter.server.ts b/apps/webapp/app/presenters/v3/QueryPresenter.server.ts index ef33e45fd..53ebb3ccd 100644 --- a/apps/webapp/app/presenters/v3/QueryPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/QueryPresenter.server.ts @@ -8,6 +8,8 @@ export type QueryHistoryItem = { scope: QueryScope; createdAt: Date; userName: string | null; + /** AI-generated title summarizing the query */ + title: string | null; /** Time filter settings */ filterPeriod: string | null; filterFrom: Date | null; @@ -24,6 +26,7 @@ export class QueryPresenter extends BasePresenter { id: true, query: true, scope: true, + title: true, createdAt: true, filterPeriod: true, filterFrom: true, @@ -43,6 +46,7 @@ export class QueryPresenter extends BasePresenter { scope: q.scope.toLowerCase() as QueryScope, createdAt: q.createdAt, userName: q.user?.displayName ?? q.user?.name ?? null, + title: q.title, filterPeriod: q.filterPeriod, filterFrom: q.filterFrom, filterTo: q.filterTo, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index b172e4d35..0238efd5b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -48,7 +48,7 @@ LIMIT 20`, total_cost, usage_duration, machine, - created_at + triggered_at FROM runs WHERE triggered_at > now() - INTERVAL 7 DAY ORDER BY total_cost DESC @@ -79,4 +79,3 @@ export function ExamplesContent({
); } - diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx index f7c94d1ef..daa7187ac 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx @@ -36,53 +36,82 @@ export function QueryHelpSidebar({ onValueChange={onTabChange} className="flex min-h-0 flex-col overflow-hidden pt-1" > - - -
- AI -
- - - Writing TRQL - - - Table schema - - - Examples - - +
+ + +
+ AI +
+ + + Writing TRQL + + + Table schema + + + Examples + + +
- +
+ +
- +
+ +
- +
+ +
- +
+ +
); } - diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx index b51217067..66492ca94 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx @@ -36,9 +36,14 @@ const SQL_KEYWORDS = [ ]; function highlightSQL(query: string): React.ReactNode[] { - // Normalize whitespace for display (let CSS line-clamp handle truncation) - const normalized = query.replace(/\s+/g, " ").slice(0, 200); - const suffix = ""; + // Normalize: collapse multiple spaces/tabs to single space, but preserve newlines + // Then trim each line and limit total length + const normalized = query + .split("\n") + .map((line) => line.replace(/[ \t]+/g, " ").trim()) + .filter((line) => line.length > 0) + .join("\n") + .slice(0, 500); // Create a regex pattern that matches keywords as whole words (case insensitive) const keywordPattern = new RegExp( @@ -69,10 +74,6 @@ function highlightSQL(query: string): React.ReactNode[] { parts.push(normalized.slice(lastIndex)); } - if (suffix) { - parts.push(suffix); - } - return parts; } @@ -118,10 +119,21 @@ export function QueryHistoryPopover({ }} className="flex w-full items-center gap-2 rounded-sm px-2 py-2 outline-none transition-colors focus-custom hover:bg-charcoal-900" > -
-

- {highlightSQL(item.query)} -

+
+ {item.title ? ( + <> +

+ {item.title} +

+

+ {highlightSQL(item.query)} +

+ + ) : ( +

+ {highlightSQL(item.query)} +

+ )}
{item.scope} {valueLabel && · {valueLabel}} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index 422d8cc2f..996149a46 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -1,13 +1,23 @@ -import { ArrowDownTrayIcon, ArrowsPointingInIcon, ArrowsPointingOutIcon, ArrowTrendingUpIcon, ClipboardIcon } from "@heroicons/react/20/solid"; -import type { OutputColumnMetadata, WhereClauseFallback } from "@internal/clickhouse"; +import { + ArrowDownTrayIcon, + ArrowsPointingOutIcon, + ArrowTrendingUpIcon, + ClipboardIcon, + TableCellsIcon, +} from "@heroicons/react/20/solid"; +import type { OutputColumnMetadata } from "@internal/clickhouse"; +import { type WhereClauseCondition } from "@internal/tsql"; +import { useFetcher } from "@remix-run/react"; import { redirect, type ActionFunctionArgs, type LoaderFunctionArgs, } from "@remix-run/server-runtime"; +import parse from "parse-duration"; import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from "react"; import { flushSync } from "react-dom"; import { typedjson, useTypedFetcher, useTypedLoaderData } from "remix-typedjson"; +import simplur from "simplur"; import { z } from "zod"; import { AISparkleIcon } from "~/assets/icons/AISparkleIcon"; import { AlphaTitle } from "~/components/AlphaBadge"; @@ -21,9 +31,7 @@ import { autoFormatSQL, TSQLEditor } from "~/components/code/TSQLEditor"; import { TSQLResultsTable } from "~/components/code/TSQLResultsTable"; import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; -import { TimeFilter, timeFilters } from "~/components/runs/v3/SharedFilters"; -import { useSearchParams } from "~/hooks/useSearchParam"; -import { Button } from "~/components/primitives/Buttons"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Card } from "~/components/primitives/charts/Card"; import { @@ -32,6 +40,7 @@ import { ClientTabsList, ClientTabsTrigger, } from "~/components/primitives/ClientTabs"; +import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog"; import { Header3 } from "~/components/primitives/Headers"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { Paragraph } from "~/components/primitives/Paragraph"; @@ -49,27 +58,30 @@ import { import { Select, SelectItem } from "~/components/primitives/Select"; import { Spinner } from "~/components/primitives/Spinner"; import { Switch } from "~/components/primitives/Switch"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; +import { TimeFilter, timeFilters } from "~/components/runs/v3/SharedFilters"; import { prisma } from "~/db.server"; +import { env } from "~/env.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; +import { useSearchParams } from "~/hooks/useSearchParam"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { QueryPresenter, type QueryHistoryItem } from "~/presenters/v3/QueryPresenter.server"; +import type { action as titleAction } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title"; +import { getLimit } from "~/services/platform.v3.server"; import { executeQuery, type QueryScope } from "~/services/queryService.server"; +import { requireUser } from "~/services/session.server"; import { downloadFile, rowsToCSV, rowsToJSON } from "~/utils/dataExport"; -import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { EnvironmentParamSchema, organizationBillingPath } from "~/utils/pathBuilder"; import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server"; import { querySchemas } from "~/v3/querySchemas"; +import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { QueryHelpSidebar } from "./QueryHelpSidebar"; import { QueryHistoryPopover } from "./QueryHistoryPopover"; import type { AITimeFilter } from "./types"; -import { formatQueryStats } from "./utils"; -import { requireUser } from "~/services/session.server"; -import parse from "parse-duration"; -import { SimpleTooltip } from "~/components/primitives/Tooltip"; -import { Dialog, DialogContent, DialogHeader, DialogPortal, DialogTrigger } from "~/components/primitives/Dialog"; -import { DialogOverlay } from "@radix-ui/react-dialog"; +import { formatDurationNanoseconds } from "@trigger.dev/core/v3"; /** Convert a Date or ISO string to ISO string format */ function toISOString(value: Date | string): string { @@ -159,12 +171,21 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typedjson({ defaultQuery, + defaultPeriod: await getDefaultPeriod(project.organizationId), history, isAdmin, + maxRows: env.QUERY_CLICKHOUSE_MAX_RETURNED_ROWS, }); }; -const DEFAULT_PERIOD = "7d"; +async function getDefaultPeriod(organizationId: string): Promise { + const idealDefaultPeriodDays = 7; + const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30); + if (maxQueryPeriod < idealDefaultPeriodDays) { + return `${maxQueryPeriod}d`; + } + return `${idealDefaultPeriodDays}d`; +} const ActionSchema = z.object({ query: z.string().min(1, "Query is required"), @@ -193,8 +214,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 403 } ); @@ -209,8 +232,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 404 } ); @@ -225,8 +250,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 404 } ); @@ -250,8 +277,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + periodClipped: null, }, { status: 400 } ); @@ -263,31 +292,54 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const explain = explainParam === "true" && isAdmin; // Build time filter fallback for triggered_at column + const defaultPeriod = await getDefaultPeriod(project.organizationId); const timeFilter = timeFilters({ period: period ?? undefined, from: from ?? undefined, to: to ?? undefined, - defaultPeriod: DEFAULT_PERIOD, + defaultPeriod, }); - let triggeredAtFallback: WhereClauseFallback; - if (timeFilter.from && timeFilter.to) { - // Both from and to specified - use BETWEEN - triggeredAtFallback = { op: "between", low: timeFilter.from, high: timeFilter.to }; - } else if (timeFilter.from) { - // Only from specified - triggeredAtFallback = { op: "gte", value: timeFilter.from }; - } else if (timeFilter.to) { - // Only to specified - triggeredAtFallback = { op: "lte", value: timeFilter.to }; - } else { + // Calculate the effective "from" date the user is requesting (for period clipping check) + // This is null only when the user specifies just a "to" date (rare case) + let requestedFromDate: Date | null = null; + if (timeFilter.from) { + requestedFromDate = new Date(timeFilter.from); + } else if (!timeFilter.to) { // Period specified (or default) - calculate from now - const periodMs = parse(timeFilter.period ?? DEFAULT_PERIOD) ?? 7 * 24 * 60 * 60 * 1000; - triggeredAtFallback = { op: "gte", value: new Date(Date.now() - periodMs) }; + const periodMs = parse(timeFilter.period ?? defaultPeriod) ?? 7 * 24 * 60 * 60 * 1000; + requestedFromDate = new Date(Date.now() - periodMs); } + // Build the fallback WHERE condition based on what the user specified + let triggeredAtFallback: WhereClauseCondition; + if (timeFilter.from && timeFilter.to) { + triggeredAtFallback = { op: "between", low: timeFilter.from, high: timeFilter.to }; + } else if (timeFilter.from) { + triggeredAtFallback = { op: "gte", value: timeFilter.from }; + } else if (timeFilter.to) { + triggeredAtFallback = { op: "lte", value: timeFilter.to }; + } else { + triggeredAtFallback = { op: "gte", value: requestedFromDate! }; + } + + const maxQueryPeriod = await getLimit(project.organizationId, "queryPeriodDays", 30); + const maxQueryPeriodDate = new Date(Date.now() - maxQueryPeriod * 24 * 60 * 60 * 1000); + + // Check if the requested time period exceeds the plan limit + const periodClipped = requestedFromDate !== null && requestedFromDate < maxQueryPeriodDate; + + // Force tenant isolation and time period limits + const enforcedWhereClause = { + organization_id: { op: "eq", value: project.organizationId }, + project_id: + scope === "project" || scope === "environment" ? { op: "eq", value: project.id } : undefined, + environment_id: scope === "environment" ? { op: "eq", value: environment.id } : undefined, + triggered_at: { op: "gte", value: maxQueryPeriodDate }, + } satisfies Record; + try { - const [error, result] = await executeQuery({ + const [error, result, queryId] = await executeQuery({ name: "query-page", query, schema: z.record(z.any()), @@ -298,6 +350,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { projectId: project.id, environmentId: environment.id, explain, + enforcedWhereClause, whereClauseFallback: { triggered_at: triggeredAtFallback, }, @@ -323,8 +376,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + queryId: null, + periodClipped: null, }, { status: 400 } ); @@ -336,8 +392,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: result.columns, stats: result.stats, hiddenColumns: result.hiddenColumns ?? null, + reachedMaxRows: result.reachedMaxRows, explainOutput: result.explainOutput ?? null, generatedSql: result.generatedSql ?? null, + queryId, + periodClipped: periodClipped ? maxQueryPeriod : null, }); } catch (err) { const errorMessage = err instanceof Error ? err.message : "Unknown error executing query"; @@ -348,8 +407,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { columns: null, stats: null, hiddenColumns: null, + reachedMaxRows: null, explainOutput: null, generatedSql: null, + queryId: null, + periodClipped: null, }, { status: 500 } ); @@ -368,18 +430,45 @@ interface QueryEditorFormHandle { const QueryEditorForm = forwardRef< QueryEditorFormHandle, { + defaultPeriod: string; defaultQuery: string; defaultScope: QueryScope; defaultTimeFilter?: { period?: string; from?: string; to?: string }; history: QueryHistoryItem[]; fetcher: ReturnType>; isAdmin: boolean; + onQuerySubmit?: () => void; + onHistorySelected?: (item: QueryHistoryItem) => void; } ->(function QueryEditorForm({ defaultQuery, defaultScope, defaultTimeFilter, history, fetcher, isAdmin }, ref) { +>(function QueryEditorForm( + { + defaultPeriod, + defaultQuery, + defaultScope, + defaultTimeFilter, + history, + fetcher, + isAdmin, + onQuerySubmit, + onHistorySelected, + }, + ref +) { const isLoading = fetcher.state === "submitting" || fetcher.state === "loading"; const [query, setQuery] = useState(defaultQuery); const [scope, setScope] = useState(defaultScope); const formRef = useRef(null); + const prevFetcherState = useRef(fetcher.state); + const plan = useCurrentPlan(); + const maxPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number; + + // Notify parent when query is submitted (for title generation) + useEffect(() => { + if (prevFetcherState.current !== "submitting" && fetcher.state === "submitting") { + onQuerySubmit?.(); + } + prevFetcherState.current = fetcher.state; + }, [fetcher.state, onQuerySubmit]); // Get time filter values - initialize from props (which may come from history) const [period, setPeriod] = useState(defaultTimeFilter?.period); @@ -406,18 +495,23 @@ const QueryEditorForm = forwardRef< [query] ); - const handleHistorySelected = useCallback((item: QueryHistoryItem) => { - setQuery(item.query); - setScope(item.scope); - // Apply time filter from history item - // Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization - setPeriod(item.filterPeriod ?? undefined); - setFrom(item.filterFrom ? toISOString(item.filterFrom) : undefined); - setTo(item.filterTo ? toISOString(item.filterTo) : undefined); - }, []); + const handleHistorySelected = useCallback( + (item: QueryHistoryItem) => { + setQuery(item.query); + setScope(item.scope); + // Apply time filter from history item + // Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization + setPeriod(item.filterPeriod ?? undefined); + setFrom(item.filterFrom ? toISOString(item.filterFrom) : undefined); + setTo(item.filterTo ? toISOString(item.filterTo) : undefined); + // Notify parent about history selection (for title) + onHistorySelected?.(item); + }, + [onHistorySelected] + ); return ( -
+
- + {/* Pass time filter values to action */} @@ -468,14 +565,16 @@ const QueryEditorForm = forwardRef< {queryHasTriggeredAt ? ( - Set in query - } + button={ + + } content="Your query includes a WHERE clause with triggered_at so this filter is disabled." /> ) : ( )}
- - - - - - + return ( + <> + + +
+ + +
+ + {titleContent} +
+ +
+ + + + + + - - Chart - -
- + {queryTitle ?? "Chart"} +
+
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts deleted file mode 100644 index 41fc6e31a..000000000 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts +++ /dev/null @@ -1,32 +0,0 @@ -export function formatQueryStats(stats: { - read_rows: string; - read_bytes: string; - elapsed_ns: string; - byte_seconds: string; -}): string { - const readRows = parseInt(stats.read_rows, 10); - const readBytes = parseInt(stats.read_bytes, 10); - const elapsedNs = parseInt(stats.elapsed_ns, 10); - const byteSeconds = parseFloat(stats.byte_seconds); - - const elapsedMs = elapsedNs / 1_000_000; - const formattedTime = - elapsedMs < 1000 ? `${elapsedMs.toFixed(1)}ms` : `${(elapsedMs / 1000).toFixed(2)}s`; - const formattedBytes = formatBytes(readBytes); - - return `${readRows.toLocaleString()} rows read · ${formattedBytes} · ${formattedTime} · ${formatBytes( - byteSeconds - )}s`; -} - -export function formatBytes(bytes: number): string { - if (bytes === 0) return "0 B"; - if (bytes < 0) return "-" + formatBytes(-bytes); - const k = 1024; - const sizes = ["B", "KB", "MB", "GB"]; - const i = Math.max( - 0, - Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1) - ); - return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`; -} diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx new file mode 100644 index 000000000..9fa57d7fb --- /dev/null +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx @@ -0,0 +1,81 @@ +import { openai } from "@ai-sdk/openai"; +import { json, type ActionFunctionArgs } from "@remix-run/server-runtime"; +import { tryCatch } from "@trigger.dev/core/utils"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { findProjectBySlug } from "~/models/project.server"; +import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; +import { requireUserId } from "~/services/session.server"; +import { EnvironmentParamSchema } from "~/utils/pathBuilder"; +import { AIQueryTitleService } from "~/v3/services/aiQueryTitleService.server"; + +const RequestSchema = z.object({ + query: z.string().min(1, "Query is required"), + queryId: z.string().optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params); + + // Parse the request body + const [error, data] = await tryCatch(request.json()); + if (error) { + return json({ success: false as const, error: error.message, title: null }, { status: 400 }); + } + const submission = RequestSchema.safeParse(data); + + if (!submission.success) { + return json( + { success: false as const, error: "Invalid request data", title: null }, + { status: 400 } + ); + } + + const project = await findProjectBySlug(organizationSlug, projectParam, userId); + if (!project) { + return json( + { success: false as const, error: "Project not found", title: null }, + { status: 404 } + ); + } + + const environment = await findEnvironmentBySlug(project.id, envParam, userId); + if (!environment) { + return json( + { success: false as const, error: "Environment not found", title: null }, + { status: 404 } + ); + } + + if (!env.OPENAI_API_KEY) { + return json( + { success: false as const, error: "OpenAI API key is not configured", title: null }, + { status: 400 } + ); + } + + const { query, queryId } = submission.data; + + const service = new AIQueryTitleService(openai(env.AI_RUN_FILTER_MODEL ?? "gpt-4o-mini")); + + const result = await service.generateTitle(query); + + if (!result.success) { + return json({ success: false as const, error: result.error, title: null }, { status: 500 }); + } + + // Strip leading/trailing quotes that AI sometimes adds + const title = result.title.replace(/^["']|["']$/g, ""); + + // If a queryId was provided, update the CustomerQuery record with the title + if (queryId) { + await prisma.customerQuery.update({ + where: { id: queryId, organizationId: project.organizationId }, + data: { title }, + }); + } + + return json({ success: true as const, title, error: null }); +} diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 6673caf44..1d5af9e00 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -7,7 +7,7 @@ import { type TSQLQueryResult, } from "@internal/clickhouse"; import type { CustomerQuerySource } from "@trigger.dev/database"; -import type { TableSchema } from "@internal/tsql"; +import type { TableSchema, WhereClauseCondition } from "@internal/tsql"; import { type z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; @@ -56,17 +56,14 @@ function getDefaultClickhouseSettings(): ClickHouseSettings { export type ExecuteQueryOptions = Omit< ExecuteTSQLOptions, - "tableSchema" | "organizationId" | "projectId" | "environmentId" | "fieldMappings" + "tableSchema" | "fieldMappings" > & { + organizationId: string; + projectId?: string; + environmentId?: string; tableSchema: TableSchema[]; /** The scope of the query - determines tenant isolation */ scope: QueryScope; - /** Organization ID (required) */ - organizationId: string; - /** Project ID (required for project/environment scope) */ - projectId: string; - /** Environment ID (required for environment scope) */ - environmentId: string; /** History options for saving query to billing/audit */ history?: { /** Where the query originated from */ @@ -89,18 +86,27 @@ export type ExecuteQueryOptions = Omit< customOrgConcurrencyLimit?: number; }; +/** + * Extended result type that includes the optional queryId when saved to history + */ +export type ExecuteQueryResult = + | [error: Error, result: null, queryId: null] + | [error: null, result: T, queryId: string | null]; + /** * Execute a TSQL query against ClickHouse with tenant isolation * Handles building tenant options, field mappings, and optionally saves to history + * Returns [error, result, queryId] where queryId is the CustomerQuery ID if saved to history */ export async function executeQuery( options: ExecuteQueryOptions -): Promise>> { +): Promise>[1], null>>> { const { scope, organizationId, projectId, environmentId, + enforcedWhereClause, history, customOrgConcurrencyLimit, whereClauseFallback, @@ -112,39 +118,22 @@ export async function executeQuery( const orgLimit = customOrgConcurrencyLimit ?? DEFAULT_ORG_CONCURRENCY_LIMIT; // Acquire concurrency slot - const acquireResult = await queryConcurrencyLimiter.acquire({ - key: organizationId, - requestId, - keyLimit: orgLimit, - globalLimit: GLOBAL_CONCURRENCY_LIMIT, - }); + const acquireResult = await queryConcurrencyLimiter.acquire({ + key: organizationId, + requestId, + keyLimit: orgLimit, + globalLimit: GLOBAL_CONCURRENCY_LIMIT, + }); - if (!acquireResult.success) { - const errorMessage = - acquireResult.reason === "key_limit" - ? `You've exceeded your query concurrency of ${orgLimit} for this organization. Please try again later.` - : "We're experiencing a lot of queries at the moment. Please try again later."; - return [new QueryError(errorMessage, { query: options.query }), null]; - } + if (!acquireResult.success) { + const errorMessage = + acquireResult.reason === "key_limit" + ? `You've exceeded your query concurrency of ${orgLimit} for this organization. Please try again later.` + : "We're experiencing a lot of queries at the moment. Please try again later."; + return [new QueryError(errorMessage, { query: options.query }), null, null]; + } try { - // Build tenant IDs based on scope - const tenantOptions: { - organizationId: string; - projectId?: string; - environmentId?: string; - } = { - organizationId, - }; - - if (scope === "project" || scope === "environment") { - tenantOptions.projectId = projectId; - } - - if (scope === "environment") { - tenantOptions.environmentId = environmentId; - } - // Build field mappings for project_ref → project_id and environment_id → slug translation const projects = await prisma.project.findMany({ where: { organizationId }, @@ -163,18 +152,29 @@ export async function executeQuery( const result = await executeTSQL(clickhouseClient.reader, { ...baseOptions, - ...tenantOptions, + enforcedWhereClause, fieldMappings, whereClauseFallback, clickhouseSettings: { ...getDefaultClickhouseSettings(), ...baseOptions.clickhouseSettings, // Allow caller overrides if needed }, + querySettings: { + maxRows: env.QUERY_CLICKHOUSE_MAX_RETURNED_ROWS, + ...baseOptions.querySettings, // Allow caller overrides if needed + }, }); + // If query failed, return early with no queryId + if (result[0] !== null) { + return [result[0], null, null]; + } + + let queryId: string | null = null; + // If query succeeded and history options provided, save to history // Skip history for EXPLAIN queries (admin debugging) and when explicitly skipped (e.g., impersonating) - if (result[0] === null && history && !history.skip && !baseOptions.explain) { + if (history && !history.skip && !baseOptions.explain) { // Check if this query is the same as the last one saved (avoid duplicate history entries) const lastQuery = await prisma.customerQuery.findFirst({ where: { @@ -183,7 +183,7 @@ export async function executeQuery( userId: history.userId ?? null, }, orderBy: { createdAt: "desc" }, - select: { query: true, scope: true, filterPeriod: true, filterFrom: true, filterTo: true }, + select: { id: true, query: true, scope: true, filterPeriod: true, filterFrom: true, filterTo: true }, }); const timeFilter = history.timeFilter; @@ -195,17 +195,15 @@ export async function executeQuery( lastQuery.filterFrom?.getTime() === (timeFilter?.from?.getTime() ?? undefined) && lastQuery.filterTo?.getTime() === (timeFilter?.to?.getTime() ?? undefined); - if (!isDuplicate) { - const stats = result[1].stats; - const byteSeconds = parseFloat(stats.byte_seconds) || 0; - const costInCents = byteSeconds * env.CENTS_PER_QUERY_BYTE_SECOND; - - await prisma.customerQuery.create({ + if (isDuplicate && lastQuery) { + // Return the existing query's ID for duplicate queries + queryId = lastQuery.id; + } else { + const created = await prisma.customerQuery.create({ data: { query: options.query, scope: scopeToEnum[scope], - stats: { ...stats }, - costInCents, + stats: { ...result[1].stats }, source: history.source, organizationId, projectId: scope === "project" || scope === "environment" ? projectId : null, @@ -216,10 +214,11 @@ export async function executeQuery( filterTo: history.timeFilter?.to ?? null, }, }); + queryId = created.id; } } - return result; + return [null, result[1], queryId]; } finally { // Always release the concurrency slot await queryConcurrencyLimiter.release({ diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index cb574cfd9..fe7005974 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -167,10 +167,14 @@ export const runsSchema: TableSchema = { expression: "if(depth > 0, true, false)", }, - // Useless until we show the user-provided key idempotency_key: { name: "idempotency_key", - ...column("String", { description: "Idempotency key", example: "user-123-action-456" }), + clickhouseName: "idempotency_key_user", + ...column("String", { description: "Idempotency key (available from 4.3.3)", example: "user-123-action-456" }), + }, + idempotency_key_scope: { + name: "idempotency_key_scope", + ...column("String", { description: "The idempotency key scope determines whether a task should be considered unique within a parent run, a specific attempt, or globally. An empty value means there's no idempotency key set (available from 4.3.3).", example: "run", allowedValues: ["global", "run", "attempt"], }), }, region: { name: "region", @@ -325,6 +329,8 @@ export const runsSchema: TableSchema = { // Output & error (JSON columns) // For JSON columns, NULL checks are transformed to check for empty object '{}' // So `error IS NULL` becomes `error = '{}'` and `error IS NOT NULL` becomes `error != '{}'` + // textColumn uses the pre-materialized text columns for better performance + // dataPrefix handles the internal {"data": ...} wrapper transparently output: { name: "output", ...column("JSON", { @@ -332,6 +338,8 @@ export const runsSchema: TableSchema = { example: '{"result": "success"}', }), nullValue: "'{}'", // Transform NULL checks to compare against empty object + textColumn: "output_text", // Use output_text for full JSON value queries + dataPrefix: "data", // Internal data is wrapped in {"data": ...} }, error: { name: "error", @@ -341,6 +349,8 @@ export const runsSchema: TableSchema = { example: '{"message": "Task failed"}', }), nullValue: "'{}'", // Transform NULL checks to compare against empty object + textColumn: "error_text", // Use error_text for full JSON value queries + dataPrefix: "data", // Internal data is wrapped in {"data": ...} }, // Tags & versions diff --git a/apps/webapp/app/v3/services/aiQueryTitleService.server.ts b/apps/webapp/app/v3/services/aiQueryTitleService.server.ts new file mode 100644 index 000000000..983f73245 --- /dev/null +++ b/apps/webapp/app/v3/services/aiQueryTitleService.server.ts @@ -0,0 +1,71 @@ +import { openai } from "@ai-sdk/openai"; +import { generateText, type LanguageModelV1 } from "ai"; +import { env } from "~/env.server"; + +/** + * Result type for title generation + */ +export type AIQueryTitleResult = + | { success: true; title: string } + | { success: false; error: string }; + +/** + * Service for generating concise titles for SQL queries using AI + */ +export class AIQueryTitleService { + constructor(private readonly model: LanguageModelV1 = openai("gpt-4o-mini")) {} + + /** + * Generate a concise title for a SQL query + */ + async generateTitle(query: string): Promise { + if (!env.OPENAI_API_KEY) { + return { success: false, error: "OpenAI API key is not configured" }; + } + + try { + const result = await generateText({ + model: this.model, + system: `You are a helpful assistant that generates concise titles for SQL queries. + +Your task is to create a short, descriptive title (5-10 words) that summarizes what the query does. + +Guidelines: +- Focus on the main purpose/intent of the query +- Use plain language, not technical SQL terms +- Start with an action verb when appropriate (e.g., "Count", "List", "Show", "Find") +- Be specific about what data is being retrieved +- Do not include quotes around the title +- Do not include punctuation at the end + +Examples: +- "Failed runs by hour over 7 days" +- "Top 50 most expensive task runs" +- "Run counts grouped by status" +- "Average execution time by task" +- "Recent runs with errors"`, + prompt: `Generate a concise title for this SQL query:\n\n${query}`, + maxTokens: 50, + experimental_telemetry: { + isEnabled: true, + metadata: { + feature: "ai-query-title", + }, + }, + }); + + const title = result.text.trim(); + + if (!title) { + return { success: false, error: "No title generated" }; + } + + return { success: true, title }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : "Failed to generate title", + }; + } + } +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 97f453334..51a468b50 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -121,7 +121,7 @@ "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*", "@trigger.dev/otlp-importer": "workspace:*", - "@trigger.dev/platform": "1.0.21", + "@trigger.dev/platform": "1.0.22", "@trigger.dev/redis-worker": "workspace:*", "@trigger.dev/sdk": "workspace:*", "@types/pg": "8.6.6", diff --git a/internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql b/internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql new file mode 100644 index 000000000..c66a1492a --- /dev/null +++ b/internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql @@ -0,0 +1,45 @@ +-- +goose Up +-- Update the materialized columns to extract the 'data' field if it exists +-- This avoids the {"data": ...} wrapper in the text representation +-- Note: Direct JSON path access (output.data) returns null for nested objects, +-- so we use JSONExtractRaw on the stringified JSON instead +ALTER TABLE trigger_dev.task_runs_v2 +ADD COLUMN output_text String MATERIALIZED if ( + toJSONString (output) = '{}', + '', + if ( + length (JSONExtractRaw (toJSONString (output), 'data')) > 0, + JSONExtractRaw (toJSONString (output), 'data'), + toJSONString (output) + ) +); + +-- For error: extract error.data if it exists +ALTER TABLE trigger_dev.task_runs_v2 +ADD COLUMN error_text String MATERIALIZED if ( + toJSONString (error) = '{}', + '', + if ( + length (JSONExtractRaw (toJSONString (error), 'data')) > 0, + JSONExtractRaw (toJSONString (error), 'data'), + toJSONString (error) + ) +); + +-- Add the indexes +ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_output_text output_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4; + +ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_error_text error_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4; + +-- +goose Down +ALTER TABLE trigger_dev.task_runs_v2 +DROP INDEX IF EXISTS idx_output_text; + +ALTER TABLE trigger_dev.task_runs_v2 +DROP INDEX IF EXISTS idx_error_text; + +ALTER TABLE trigger_dev.task_runs_v2 +DROP COLUMN IF EXISTS output_text; + +ALTER TABLE trigger_dev.task_runs_v2 +DROP COLUMN IF EXISTS error_text; \ No newline at end of file diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index c68923369..61868b610 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -2,7 +2,7 @@ * TSQL Query Execution for ClickHouse * * This module provides a safe interface for executing TSQL queries against ClickHouse - * with automatic tenant isolation and SQL injection protection. + * with enforced WHERE clause conditions (tenant isolation + plan limits) and SQL injection protection. */ import type { ClickHouseSettings } from "@clickhouse/client"; @@ -14,7 +14,7 @@ import { type TableSchema, type QuerySettings, type FieldMappings, - type WhereClauseFallback, + type WhereClauseCondition } from "@internal/tsql"; import type { ClickhouseReader, QueryStats } from "./types.js"; import { QueryError } from "./errors.js"; @@ -25,7 +25,7 @@ const logger = new Logger("tsql", "info"); export type { QueryStats }; -export type { TableSchema, QuerySettings, FieldMappings, WhereClauseFallback }; +export type { TableSchema, QuerySettings, FieldMappings, WhereClauseCondition }; /** * Options for executing a TSQL query @@ -37,14 +37,26 @@ export interface ExecuteTSQLOptions { query: string; /** The Zod schema for validating output rows */ schema: TOut; - /** The organization ID for tenant isolation (required) */ - organizationId: string; - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - projectId?: string; - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - environmentId?: string; /** Schema registry defining allowed tables and columns */ tableSchema: TableSchema[]; + /** + * REQUIRED: Conditions always applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + * Applied to every table reference including subqueries, CTEs, and JOINs. + * + * @example + * ```typescript + * { + * // Tenant isolation + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * // Plan-based time limit + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) } + * } + * ``` + */ + enforcedWhereClause: Record; /** Optional ClickHouse query settings */ clickhouseSettings?: ClickHouseSettings; /** Optional TSQL query settings (maxRows, timezone, etc.) */ @@ -78,6 +90,7 @@ export interface ExecuteTSQLOptions { /** * Fallback WHERE conditions to apply when the user hasn't filtered on a column. * Key is the column name, value is the fallback condition. + * These are applied at the AST level (top-level query only). * * @example * ```typescript @@ -87,7 +100,7 @@ export interface ExecuteTSQLOptions { * } * ``` */ - whereClauseFallback?: Record; + whereClauseFallback?: Record; } /** @@ -102,6 +115,11 @@ export interface TSQLQuerySuccess { * Only populated when SELECT * is transformed to core columns only. */ hiddenColumns?: string[]; + /** + * Whether the result count equals the maxRows limit. + * When true, the results may be truncated and more rows may exist. + */ + reachedMaxRows: boolean; /** * The raw EXPLAIN output from ClickHouse. * Only populated when `explain: true` is passed. @@ -123,7 +141,7 @@ export type TSQLQueryResult = [QueryError, null] | [null, TSQLQuerySuccess * Execute a TSQL query against ClickHouse * * This function: - * 1. Compiles the TSQL query to ClickHouse SQL (parse, validate, inject tenant guards) + * 1. Compiles the TSQL query to ClickHouse SQL (parse, validate, inject enforced WHERE clauses) * 2. Executes the query and returns validated results * * @example @@ -132,10 +150,12 @@ export type TSQLQueryResult = [QueryError, null] | [null, TSQLQuerySuccess * name: "get_task_runs", * query: "SELECT id, status FROM task_runs WHERE status = 'completed' ORDER BY created_at DESC LIMIT 100", * schema: z.object({ id: z.string(), status: z.string() }), - * organizationId: "org_123", - * projectId: "proj_456", - * environmentId: "env_789", * tableSchema: [taskRunsSchema], + * enforcedWhereClause: { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * }, * }); * ``` */ @@ -145,18 +165,22 @@ export async function executeTSQL( ): Promise>> { const shouldTransformValues = options.transformValues ?? true; const isExplain = options.explain ?? false; + const maxRows = options.querySettings?.maxRows; let generatedSql: string | undefined; let generatedParams: Record | undefined; try { // 1. Compile the TSQL query to ClickHouse SQL + // Pass maxRows + 1 to fetch one extra row for overflow detection + const compiledSettings = maxRows !== undefined + ? { ...options.querySettings, maxRows: maxRows + 1 } + : options.querySettings; + const { sql, params, columns, hiddenColumns } = compileTSQL(options.query, { - organizationId: options.organizationId, - projectId: options.projectId, - environmentId: options.environmentId, tableSchema: options.tableSchema, - settings: options.querySettings, + enforcedWhereClause: options.enforcedWhereClause, + settings: compiledSettings, fieldMappings: options.fieldMappings, whereClauseFallback: options.whereClauseFallback, }); @@ -231,26 +255,36 @@ export async function executeTSQL( columns: [], stats, hiddenColumns, + reachedMaxRows: false, explainOutput: combinedOutput, generatedSql, }, ]; } + // Determine if we exceeded maxRows (we fetched maxRows + 1 to detect overflow) + const reachedMaxRows = maxRows !== undefined && rows !== undefined && rows.length > maxRows; + + // Remove the overflow row if we got one (pop is O(1), slice would be O(n)) + const finalRows = rows ?? []; + if (reachedMaxRows) { + finalRows.pop(); + } + // Build the result, including hiddenColumns if present - const baseResult = { columns, stats, hiddenColumns }; + const baseResult = { columns, stats, hiddenColumns, reachedMaxRows }; // 3. Transform result values if enabled - if (shouldTransformValues && rows) { + if (shouldTransformValues && finalRows.length > 0) { const transformedRows = transformResults( - rows as Record[], + finalRows as Record[], options.tableSchema, { fieldMappings: options.fieldMappings } ); return [null, { rows: transformedRows as z.output[], ...baseResult }]; } - return [null, { rows: rows ?? [], ...baseResult }]; + return [null, { rows: finalRows as z.output[], ...baseResult }]; } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error"; @@ -284,9 +318,11 @@ export async function executeTSQL( * name: "get_task_runs", * query: "SELECT * FROM task_runs LIMIT 10", * schema: taskRunRowSchema, - * organizationId: "org_123", - * projectId: "proj_456", - * environmentId: "env_789", + * enforcedWhereClause: { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * }, * }); * ``` */ diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 4f4cb5e3b..50b39d35a 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -54,7 +54,7 @@ export { type TSQLQuerySuccess, type QueryStats, type FieldMappings, - type WhereClauseFallback, + type WhereClauseCondition, } from "./client/tsql.js"; export type { OutputColumnMetadata } from "@internal/tsql"; diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index fd33ed510..5c1d6ed80 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -106,9 +106,11 @@ describe("TSQL Integration Tests", () => { name: "test-simple-select", query: "SELECT run_id, status FROM task_runs", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -145,9 +147,11 @@ describe("TSQL Integration Tests", () => { name: "test-where-clause", query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED_SUCCESSFULLY'", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -197,9 +201,11 @@ describe("TSQL Integration Tests", () => { name: "test-tenant-isolation-1", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -212,9 +218,11 @@ describe("TSQL Integration Tests", () => { name: "test-tenant-isolation-2", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant2", - projectId: "proj_tenant2", - environmentId: "env_tenant2", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant2" }, + project_id: { op: "eq", value: "proj_tenant2" }, + environment_id: { op: "eq", value: "env_tenant2" }, + }, tableSchema: [taskRunsSchema], }); @@ -254,9 +262,11 @@ describe("TSQL Integration Tests", () => { name: "test-cross-tenant-attack", query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED' OR 1=1", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_attacker", - projectId: "proj_attacker", - environmentId: "env_attacker", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_attacker" }, + project_id: { op: "eq", value: "proj_attacker" }, + environment_id: { op: "eq", value: "env_attacker" }, + }, tableSchema: [taskRunsSchema], }); @@ -288,9 +298,11 @@ describe("TSQL Integration Tests", () => { query: "SELECT status, count(*) as cnt FROM task_runs GROUP BY status ORDER BY cnt DESC, status ASC", schema: z.object({ status: z.string(), cnt: z.coerce.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -325,9 +337,11 @@ describe("TSQL Integration Tests", () => { name: "test-order-limit", query: "SELECT run_id FROM task_runs ORDER BY created_at DESC LIMIT 2", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -347,9 +361,11 @@ describe("TSQL Integration Tests", () => { name: "test-unknown-table", query: "SELECT * FROM unknown_table", schema: z.object({ id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -378,9 +394,11 @@ describe("TSQL Integration Tests", () => { name: "test-executor", query: "SELECT run_id, status FROM task_runs WHERE status = 'PENDING'", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, }); expect(error).toBeNull(); @@ -406,9 +424,11 @@ describe("TSQL Integration Tests", () => { name: "test-injection", query: "SELECT run_id, status FROM task_runs WHERE status = 'DROP TABLE task_runs'", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -438,9 +458,11 @@ describe("TSQL Integration Tests", () => { query: "SELECT run_id, status FROM task_runs WHERE status IN ('COMPLETED_SUCCESSFULLY', 'FAILED')", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -467,9 +489,11 @@ describe("TSQL Integration Tests", () => { name: "test-like-query", query: "SELECT run_id, task_identifier FROM task_runs WHERE task_identifier LIKE 'email%'", schema: z.object({ run_id: z.string(), task_identifier: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [taskRunsSchema], }); @@ -530,8 +554,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-cross-project-query", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_multi", - // projectId and environmentId omitted - query across all + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_multi" }, + // project_id and environment_id omitted - query across all + }, tableSchema: [taskRunsSchema], }); @@ -590,9 +616,11 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-cross-env-query", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_envtest", - projectId: "proj_envtest", - // environmentId omitted - query across all environments + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_envtest" }, + project_id: { op: "eq", value: "proj_envtest" }, + // environment_id omitted - query across all environments + }, tableSchema: [taskRunsSchema], }); @@ -649,8 +677,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-org-isolation-1", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_isolation_1", - // projectId and environmentId omitted + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_isolation_1" }, + // project_id and environment_id omitted + }, tableSchema: [taskRunsSchema], }); @@ -663,8 +693,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-org-isolation-2", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_isolation_2", - // projectId and environmentId omitted + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_isolation_2" }, + // project_id and environment_id omitted + }, tableSchema: [taskRunsSchema], }); @@ -706,8 +738,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-or-bypass-attempt", query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED' OR 1=1", schema: z.object({ run_id: z.string(), status: z.string() }), - organizationId: "org_attacker", - // No project/env filter - but org filter should still protect + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_attacker" }, + // No project/env filter - but org filter should still protect + }, tableSchema: [taskRunsSchema], }); @@ -751,8 +785,10 @@ describe("TSQL Optional Tenant Filter Tests", () => { name: "test-executor-optional", query: "SELECT run_id FROM task_runs", schema: z.object({ run_id: z.string() }), - organizationId: "org_executor_test", - // projectId and environmentId omitted + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_executor_test" }, + // project_id and environment_id omitted + }, }); expect(error).toBeNull(); @@ -839,9 +875,11 @@ describe("TSQL Virtual Column Tests", () => { execution_duration: z.number().nullable(), usage_duration_seconds: z.number(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -889,9 +927,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-virtual-column-where", query: "SELECT run_id FROM task_runs WHERE execution_duration > 5000", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -935,9 +975,11 @@ describe("TSQL Virtual Column Tests", () => { run_id: z.string(), usage_duration_seconds: z.number(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -977,9 +1019,11 @@ describe("TSQL Virtual Column Tests", () => { run_id: z.string(), dur_sec: z.number(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -1013,9 +1057,11 @@ describe("TSQL Virtual Column Tests", () => { run_id: z.string(), execution_duration: z.number().nullable(), }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [virtualColumnSchema], }); @@ -1110,9 +1156,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-division-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost > 1.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1153,9 +1201,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-gte-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost >= 1.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1192,9 +1242,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-lt-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost < 1.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1236,9 +1288,11 @@ describe("TSQL Virtual Column Tests", () => { query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost BETWEEN 1.0 AND 2.0", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1282,9 +1336,11 @@ describe("TSQL Virtual Column Tests", () => { query: "SELECT run_id FROM task_runs WHERE status = 'COMPLETED_SUCCESSFULLY' AND invocation_cost > 2.0", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1328,9 +1384,11 @@ describe("TSQL Virtual Column Tests", () => { name: "test-expression-large-integer-where", query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost > 100", schema: z.object({ run_id: z.string(), invocation_cost: z.number() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [costExpressionSchema], }); @@ -1393,9 +1451,11 @@ describe("Field Mapping Tests", () => { name: "test-field-mapping-select", query: "SELECT run_id, project_ref FROM task_runs", schema: z.object({ run_id: z.string(), project_ref: z.string().nullable() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { @@ -1434,9 +1494,11 @@ describe("Field Mapping Tests", () => { name: "test-field-mapping-unmapped", query: "SELECT run_id, project_ref FROM task_runs WHERE run_id = 'run_fm_unmapped'", schema: z.object({ run_id: z.string(), project_ref: z.string().nullable() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { @@ -1481,9 +1543,11 @@ describe("Field Mapping Tests", () => { name: "test-field-mapping-where", query: "SELECT run_id FROM task_runs WHERE project_ref = 'my-project-ref'", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { @@ -1530,7 +1594,9 @@ describe("Field Mapping Tests", () => { query: "SELECT run_id FROM task_runs WHERE project_ref IN ('my-project-ref', 'other-project')", schema: z.object({ run_id: z.string() }), - organizationId: "org_tenant1", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + }, tableSchema: [fieldMappingSchema], fieldMappings: { project: { diff --git a/internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql b/internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql new file mode 100644 index 000000000..ab542e609 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql @@ -0,0 +1,7 @@ +-- AlterTable +ALTER TABLE "CustomerQuery" +ADD COLUMN IF NOT EXISTS "title" TEXT; + +-- AlterTable +ALTER TABLE "CustomerQuery" +DROP COLUMN IF EXISTS "costInCents"; \ No newline at end of file diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 163445e3b..c76b41141 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2452,8 +2452,8 @@ model CustomerQuery { /// Query execution statistics from ClickHouse stats Json - /// Cost of the query in cents (for Stripe metering) - costInCents Float @default(0) + /// AI-generated title summarizing the query + title String? /// Where the query originated from source CustomerQuerySource @default(DASHBOARD) diff --git a/internal-packages/tsql/src/index.test.ts b/internal-packages/tsql/src/index.test.ts index 8621325c8..7a5182668 100644 --- a/internal-packages/tsql/src/index.test.ts +++ b/internal-packages/tsql/src/index.test.ts @@ -5,12 +5,12 @@ import { isColumnReferencedInExpression, createFallbackExpression, injectFallbackConditions, - type WhereClauseFallback, + type WhereClauseCondition, } from "./index.js"; import { column, type TableSchema } from "./query/schema.js"; /** - * Test table schema for whereClauseFallback tests + * Test table schema for enforcedWhereClause tests */ const taskRunsSchema: TableSchema = { name: "task_runs", @@ -21,6 +21,7 @@ const taskRunsSchema: TableSchema = { created_at: { name: "created_at", ...column("DateTime64") }, updated_at: { name: "updated_at", ...column("DateTime64") }, time: { name: "time", ...column("DateTime64") }, + triggered_at: { name: "triggered_at", ...column("DateTime64") }, organization_id: { name: "organization_id", ...column("String") }, project_id: { name: "project_id", ...column("String") }, environment_id: { name: "environment_id", ...column("String") }, @@ -32,6 +33,46 @@ const taskRunsSchema: TableSchema = { }, }; +/** + * Test table schema with tenant columns (lookup table with tenant isolation) + */ +const lookupTableSchema: TableSchema = { + name: "lookup_table", + clickhouseName: "trigger_dev.lookup_table", + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + columns: { + id: { name: "id", ...column("String") }, + name: { name: "name", ...column("String") }, + }, +}; + +/** + * Test table schema WITHOUT tenant columns (e.g., global reference data) + */ +// @ts-expect-error - tenant columns are required but not set +const nonTenantTableSchema: TableSchema = { + name: "reference_data", + clickhouseName: "trigger_dev.reference_data", + // No tenantColumns - this is a global table + columns: { + id: { name: "id", ...column("String") }, + value: { name: "value", ...column("String") }, + }, +}; + +/** + * Base options with tenant isolation for tests + */ +const baseEnforcedWhereClause: Record = { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, +}; + describe("isColumnReferencedInExpression", () => { it("should detect column in simple WHERE clause", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-01-01'"); @@ -126,7 +167,7 @@ describe("createFallbackExpression", () => { describe("injectFallbackConditions", () => { it("should inject fallback when column is not in WHERE", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE status = 'completed'"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, }; @@ -140,7 +181,7 @@ describe("injectFallbackConditions", () => { it("should NOT inject fallback when column is already in WHERE", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, }; @@ -154,7 +195,7 @@ describe("injectFallbackConditions", () => { it("should inject fallback when query has no WHERE clause", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, }; @@ -167,7 +208,7 @@ describe("injectFallbackConditions", () => { it("should inject multiple fallbacks", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, status: { op: "eq", value: "completed" }, }; @@ -182,7 +223,7 @@ describe("injectFallbackConditions", () => { it("should only inject fallbacks for unreferenced columns", () => { const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'"); - const fallbacks: Record = { + const fallbacks: Record = { time: { op: "gte", value: "2024-01-01" }, // Should NOT be injected status: { op: "eq", value: "completed" }, // Should be injected }; @@ -197,10 +238,8 @@ describe("injectFallbackConditions", () => { describe("compileTSQL with whereClauseFallback", () => { const baseOptions = { - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", tableSchema: [taskRunsSchema], + enforcedWhereClause: baseEnforcedWhereClause, }; describe("simple comparison fallbacks", () => { @@ -474,3 +513,317 @@ describe("compileTSQL with whereClauseFallback", () => { }); }); +describe("compileTSQL with enforcedWhereClause", () => { + describe("validation tests", () => { + it("should throw error when required tenant column is missing", () => { + expect(() => + compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: {}, // Missing organization_id + }) + ).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause"); + }); + + it("should throw error when organization_id is missing but other tenant columns are present", () => { + expect(() => + compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + project_id: { op: "eq", value: "proj_123" }, + environment_id: { op: "eq", value: "env_456" }, + }, + }) + ).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause"); + }); + + it("should work with non-tenant table and empty enforcedWhereClause", () => { + const { sql } = compileTSQL("SELECT id FROM reference_data", { + tableSchema: [nonTenantTableSchema], + enforcedWhereClause: {}, + }); + + expect(sql).toContain("SELECT"); + expect(sql).toContain("FROM"); + }); + + it("should work with only organization_id (project and env are optional)", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + }, + }); + + expect(sql).toContain("organization_id"); + expect(sql).not.toContain("project_id"); + expect(sql).not.toContain("environment_id"); + }); + }); + + describe("basic functionality", () => { + it("should apply single enforced condition", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + }, + }); + + expect(sql).toContain("equals("); + expect(sql).toContain("organization_id"); + }); + + it("should apply multiple enforced conditions", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + project_id: { op: "eq", value: "proj_456" }, + environment_id: { op: "eq", value: "env_789" }, + }, + }); + + expect(sql).toContain("organization_id"); + expect(sql).toContain("project_id"); + expect(sql).toContain("environment_id"); + }); + + it("should apply enforced condition even when user filters on same field", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + } + ); + + // Should have BOTH the user's condition AND the enforced condition + // User's condition: greater(triggered_at, '2025-01-01') + // Enforced condition: greaterOrEquals(triggered_at, '2024-01-01') + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2); + }); + + it("should apply different comparison operators", () => { + const { sql: sqlGt } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + time: { op: "gt", value: "2024-01-01" }, + }, + }); + expect(sqlGt).toContain("greater("); + + const { sql: sqlLt } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + time: { op: "lt", value: "2024-12-31" }, + }, + }); + expect(sqlLt).toContain("less("); + + const { sql: sqlNeq } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + status: { op: "neq", value: "deleted" }, + }, + }); + expect(sqlNeq).toContain("notEquals("); + }); + + it("should apply BETWEEN condition", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + time: { op: "between", low: "2024-01-01", high: "2024-12-31" }, + }, + }); + + expect(sql).toContain("time BETWEEN"); + }); + + it("should handle Date values in enforced conditions", () => { + const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + const { sql, params } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: sevenDaysAgo }, + }, + }); + + expect(sql).toContain("triggered_at"); + expect(sql).toContain("toDateTime64"); + }); + }); + + describe("enforcedWhereClause + whereClauseFallback interaction", () => { + it("should apply both enforced and fallback conditions when user doesn't filter", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + whereClauseFallback: { + status: { op: "eq", value: "completed" }, + }, + }); + + // Should have both enforced (triggered_at) and fallback (status) + expect(sql).toContain("triggered_at"); + expect(sql).toContain("status"); + }); + + it("should apply enforced but not fallback when user filters on fallback column", () => { + const { sql, params } = compileTSQL( + "SELECT id FROM task_runs WHERE status = 'failed'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + whereClauseFallback: { + status: { op: "eq", value: "completed" }, + }, + } + ); + + // Enforced triggered_at should be applied + expect(sql).toContain("triggered_at"); + // User's status = 'failed' should be there (as a parameter) + expect(Object.values(params)).toContain("failed"); + // The fallback 'completed' should NOT be applied since user filtered on status + expect(Object.values(params)).not.toContain("completed"); + }); + + it("should apply both enforced and fallback on same field (enforced always, fallback only if not filtered)", () => { + // User doesn't filter on triggered_at, so BOTH enforced AND fallback apply + const { sql } = compileTSQL("SELECT id FROM task_runs WHERE status = 'completed'", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: last 6 months + }, + whereClauseFallback: { + triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: last year + }, + }); + + // Both should be applied (enforced at printer level, fallback at AST level) + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2); + }); + + it("should skip fallback but keep enforced when user filters on same field", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: always applied + }, + whereClauseFallback: { + triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: skipped since user filtered + }, + } + ); + + // User's condition + enforced should be present + // Fallback should NOT be applied since user filtered on triggered_at + // Count distinct triggered_at conditions + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + // Should be 2: user's condition + enforced condition (NOT 3, no fallback) + expect(triggeredAtMatches.length).toBe(2); + }); + }); + + describe("security tests", () => { + it("should apply enforced conditions to UNION queries", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE status = 'completed' UNION ALL SELECT id FROM task_runs WHERE status = 'failed'", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + } + ); + + // Both parts of the UNION should have the enforced conditions + const orgMatches = sql.match(/organization_id/g) || []; + expect(orgMatches.length).toBe(2); + + const triggeredAtMatches = sql.match(/triggered_at/g) || []; + expect(triggeredAtMatches.length).toBe(2); + }); + + it("should NOT be bypassable via OR clause", () => { + const { sql } = compileTSQL( + "SELECT id FROM task_runs WHERE status = 'completed' OR 1=1", + { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + triggered_at: { op: "gte", value: "2024-01-01" }, + }, + } + ); + + // The enforced conditions should be ANDed with the entire user WHERE clause + // So the structure should be: (enforced AND enforced AND ...) AND (user_where) + expect(sql).toContain("organization_id"); + expect(sql).toContain("triggered_at"); + // The 1=1 should be within the user's OR clause, not affecting enforced conditions + }); + + it("should skip enforced conditions for columns that don't exist in table", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + nonexistent_column: { op: "eq", value: "test" }, + }, + }); + + // Should not contain nonexistent_column + expect(sql).not.toContain("nonexistent_column"); + // Should still have organization_id + expect(sql).toContain("organization_id"); + }); + }); + + describe("edge cases", () => { + it("should handle empty enforced conditions for non-tenant table", () => { + const { sql } = compileTSQL("SELECT id FROM reference_data", { + tableSchema: [nonTenantTableSchema], + enforcedWhereClause: {}, + }); + + expect(sql).toContain("SELECT"); + expect(sql).not.toContain("WHERE"); // No WHERE clause needed + }); + + it("should properly format numeric values", () => { + const { sql } = compileTSQL("SELECT id FROM task_runs", { + tableSchema: [taskRunsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_123" }, + }, + }); + + // org_123 should be parameterized, not inlined + expect(sql).toContain("tsql_val_"); + }); + }); +}); + diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index 9a7b3ddb2..e0f061eef 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -23,7 +23,13 @@ import { CompareOperationOp } from "./query/ast.js"; import { SyntaxError as TSQLSyntaxError } from "./query/errors.js"; import { TSQLParseTreeConverter } from "./query/parser.js"; import { printToClickHouse, type PrintResult } from "./query/printer.js"; -import { createPrinterContext, type QuerySettings } from "./query/printer_context.js"; +import { + createPrinterContext, + type BetweenCondition, + type QuerySettings, + type SimpleComparisonCondition, + type WhereClauseCondition, +} from "./query/printer_context.js"; import { createSchemaRegistry, type FieldMappings, type TableSchema } from "./query/schema.js"; /** @@ -113,9 +119,12 @@ export { createPrinterContext, DEFAULT_QUERY_SETTINGS, PrinterContext, + type BetweenCondition, type PrinterContextOptions, type QueryNotice, type QuerySettings, + type SimpleComparisonCondition, + type WhereClauseCondition, } from "./query/printer_context.js"; // Re-export printer @@ -304,7 +313,7 @@ function createValueExpression(value: Date | string | number): Expression { /** * Map fallback operator to CompareOperationOp */ -function mapFallbackOpToCompareOp(op: SimpleComparisonFallback["op"]): CompareOperationOp { +function mapFallbackOpToCompareOp(op: SimpleComparisonCondition["op"]): CompareOperationOp { switch (op) { case "eq": return CompareOperationOp.Eq; @@ -330,7 +339,7 @@ function mapFallbackOpToCompareOp(op: SimpleComparisonFallback["op"]): CompareOp */ export function createFallbackExpression( column: string, - fallback: WhereClauseFallback + fallback: WhereClauseCondition ): Expression { const fieldExpr: Field = { expression_type: "field", @@ -367,7 +376,7 @@ export function createFallbackExpression( */ export function injectFallbackConditions( ast: SelectQuery | SelectSetQuery, - fallbacks: Record + fallbacks: Record ): SelectQuery | SelectSetQuery { // Handle SelectSetQuery (UNION, etc.) - apply to each query in the set if (ast.expression_type === "select_set_query") { @@ -434,46 +443,31 @@ export function injectFallbackConditions( }; } -/** - * A simple comparison fallback condition (e.g., column > value) - */ -export interface SimpleComparisonFallback { - /** The comparison operator */ - op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte"; - /** The value to compare against */ - value: Date | string | number; -} - -/** - * A between fallback condition (e.g., column BETWEEN low AND high) - */ -export interface BetweenFallback { - /** The between operator */ - op: "between"; - /** The low bound of the range */ - low: Date | string | number; - /** The high bound of the range */ - high: Date | string | number; -} - -/** - * A WHERE clause fallback condition. - * Used to apply default filters when the user hasn't specified one for a column. - */ -export type WhereClauseFallback = SimpleComparisonFallback | BetweenFallback; /** * Options for compiling a TSQL query to ClickHouse SQL */ export interface CompileTSQLOptions { - /** The organization ID for tenant isolation (required) */ - organizationId: string; - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - projectId?: string; - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - environmentId?: string; /** Schema definitions for allowed tables and columns */ tableSchema: TableSchema[]; + /** + * REQUIRED: Conditions always applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + * Applied to every table reference including subqueries, CTEs, and JOINs. + * + * @example + * ```typescript + * { + * // Tenant isolation + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * // Plan-based time limit + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) } + * } + * ``` + */ + enforcedWhereClause: Record; /** Optional query settings */ settings?: Partial; /** @@ -491,6 +485,7 @@ export interface CompileTSQLOptions { /** * Fallback WHERE conditions to apply when the user hasn't filtered on a column. * Key is the column name, value is the fallback condition. + * These are applied at the AST level (top-level query only). * * @example * ```typescript @@ -505,7 +500,7 @@ export interface CompileTSQLOptions { * } * ``` */ - whereClauseFallback?: Record; + whereClauseFallback?: Record; } /** @@ -514,24 +509,28 @@ export interface CompileTSQLOptions { * This function: * 1. Parses the TSQL query into an AST * 2. Validates tables and columns against the schema - * 3. Injects tenant isolation WHERE clauses - * 4. Generates parameterized ClickHouse SQL + * 3. Injects enforced WHERE clauses (tenant isolation + plan limits) at printer level + * 4. Optionally injects fallback WHERE conditions at AST level + * 5. Generates parameterized ClickHouse SQL * * @param query - The TSQL query string to compile - * @param options - Compilation options including tenant IDs and schema + * @param options - Compilation options including enforcedWhereClause and schema * @returns The compiled SQL and parameters * @throws TSQLSyntaxError if the query is invalid - * @throws QueryError if tables/columns are not allowed + * @throws QueryError if tables/columns are not allowed or required tenant columns are missing * * @example * ```typescript * const { sql, params } = compileTSQL( * "SELECT * FROM task_runs WHERE status = 'completed' LIMIT 100", * { - * organizationId: "org_123", - * projectId: "proj_456", - * environmentId: "env_789", * tableSchema: [taskRunsSchema], + * enforcedWhereClause: { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * environment_id: { op: "eq", value: "env_789" }, + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }, + * }, * } * ); * ``` @@ -540,7 +539,7 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe // 1. Parse the TSQL query let ast = parseTSQLSelect(query); - // 2. Inject fallback WHERE conditions if provided + // 2. Inject fallback WHERE conditions if provided (applied at AST level - top-level query only) if (options.whereClauseFallback && Object.keys(options.whereClauseFallback).length > 0) { ast = injectFallbackConditions(ast, options.whereClauseFallback); } @@ -548,16 +547,20 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe // 3. Create schema registry from table schemas const schemaRegistry = createSchemaRegistry(options.tableSchema); - // 4. Create printer context with tenant IDs and field mappings + + // 4. Strip undefined values from enforcedWhereClause + const enforcedWhereClause = Object.fromEntries( + Object.entries(options.enforcedWhereClause).filter(([_, value]) => value !== undefined) + ) as Record; + + // 5. Create printer context with enforced WHERE clause and field mappings const context = createPrinterContext({ - organizationId: options.organizationId, - projectId: options.projectId, - environmentId: options.environmentId, schema: schemaRegistry, settings: options.settings, fieldMappings: options.fieldMappings, + enforcedWhereClause, }); - // 5. Print the AST to ClickHouse SQL + // 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level) return printToClickHouse(ast, context); } diff --git a/internal-packages/tsql/src/query/printer.test.ts b/internal-packages/tsql/src/query/printer.test.ts index dcbb79b2d..6c5cda6e6 100644 --- a/internal-packages/tsql/src/query/printer.test.ts +++ b/internal-packages/tsql/src/query/printer.test.ts @@ -85,10 +85,12 @@ function createTestContext( ): PrinterContext { const schema = createSchemaRegistry([taskRunsSchema, taskEventsSchema]); return createPrinterContext({ - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, + }, ...overrides, }); } @@ -153,10 +155,12 @@ describe("ClickHousePrinter", () => { it("should expand SELECT * with column name mapping", () => { const schema = createSchemaRegistry([runsSchema]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); const { sql, columns } = printQuery("SELECT * FROM runs", ctx); @@ -216,10 +220,12 @@ describe("ClickHousePrinter", () => { const schema = createSchemaRegistry([schemaWithVirtual]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); const { sql, columns } = printQuery("SELECT * FROM runs", ctx); @@ -241,15 +247,17 @@ describe("ClickHousePrinter", () => { describe("Table and column name mapping", () => { function createMappedContext() { const schema = createSchemaRegistry([runsSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } - it("should map user-friendly table name to ClickHouse name", () => { + it("should map user-friendly table name to ClickHouse name", () => { const ctx = createMappedContext(); const { sql } = printQuery("SELECT * FROM runs", ctx); @@ -472,15 +480,17 @@ describe("ClickHousePrinter", () => { function createJsonContext() { const schema = createSchemaRegistry([jsonSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } - it("should transform IS NULL to equals empty object for JSON columns with nullValue", () => { + it("should transform IS NULL to equals empty object for JSON columns with nullValue", () => { const ctx = createJsonContext(); const { sql } = printQuery("SELECT * FROM runs WHERE error IS NULL", ctx); @@ -597,6 +607,501 @@ describe("ClickHousePrinter", () => { expect(sql).toContain("GROUP BY status"); expect(sql).not.toContain(".:String"); }); + + it("should NOT add .:String type hint for JSON subfield in WHERE comparison", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE error.data.name = 'test'", + ctx + ); + + // WHERE clause should NOT have .:String type hint (it breaks the query) + expect(sql).toContain("equals(error.data.name,"); + expect(sql).not.toContain("error.data.name.:String"); + }); + + it("should NOT add .:String for JSON subfield in WHERE with LIKE", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE error.message LIKE '%error%'", + ctx + ); + + // WHERE clause should NOT have .:String type hint + expect(sql).toContain("like(error.message,"); + expect(sql).not.toContain("error.message.:String"); + }); + + it("should NOT add .:String in SELECT or WHERE when no GROUP BY", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT error.data.name FROM runs WHERE error.data.name = 'test'", + ctx + ); + + // SELECT should NOT have .:String (no GROUP BY, so no need for type hint) + expect(sql).toContain("error.data.name AS error_data_name"); + expect(sql).not.toContain(".:String"); + // WHERE should NOT have .:String + expect(sql).toContain("equals(error.data.name,"); + }); + + it("should add .:String in GROUP BY but not in WHERE for same query", () => { + const ctx = createJsonContext(); + const { sql } = printQuery( + "SELECT error.data.name, count() AS cnt FROM runs WHERE error.data.name = 'test' GROUP BY error.data.name", + ctx + ); + + // SELECT should have .:String + expect(sql).toContain("error.data.name.:String AS error_data_name"); + // GROUP BY should have .:String + expect(sql).toContain("GROUP BY error.data.name.:String"); + // WHERE should NOT have .:String + expect(sql).toContain("equals(error.data.name,"); + expect(sql).not.toMatch(/equals\(error\.data\.name\.:String/); + }); + }); + + describe("textColumn optimization for JSON columns", () => { + // Create a schema with JSON columns that have textColumn set + const textColumnSchema: TableSchema = { + name: "runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + id: { name: "id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "output_text", + }, + error: { + name: "error", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "error_text", + }, + status: { name: "status", ...column("String") }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function createTextColumnContext() { + const schema = createSchemaRegistry([textColumnSchema]); + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } + + describe("SELECT clause", () => { + it("should use text column when selecting bare JSON column", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output FROM runs", ctx); + + // Should use the text column with an alias to preserve the column name + expect(sql).toContain("output_text AS output"); + }); + + it("should use text column for multiple JSON columns", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output, error FROM runs", ctx); + + expect(sql).toContain("output_text AS output"); + expect(sql).toContain("error_text AS error"); + }); + + it("should use JSON column for subfield access without .:String when no GROUP BY", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output.data.name FROM runs", ctx); + + // Should use the original JSON column without .:String (no GROUP BY) + expect(sql).toContain("output.data.name AS output_data_name"); + expect(sql).not.toContain("output_text"); + expect(sql).not.toContain(".:String"); + }); + }); + + describe("SELECT * expansion", () => { + it("should use text columns when expanding SELECT *", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT * FROM runs", ctx); + + // Should use text columns for JSON columns + expect(sql).toContain("output_text AS output"); + expect(sql).toContain("error_text AS error"); + }); + }); + + describe("WHERE clause", () => { + it("should use text column for exact equality comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output = '{}'", ctx); + + expect(sql).toContain("equals(output_text,"); + expect(sql).not.toMatch(/equals\(output,/); + }); + + it("should use text column for inequality comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output != '{}'", ctx); + + expect(sql).toContain("notEquals(output_text,"); + }); + + it("should use text column for LIKE comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output LIKE '%error%'", ctx); + + expect(sql).toContain("like(output_text,"); + expect(sql).not.toMatch(/like\(output,/); + }); + + it("should use text column for ILIKE comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE error ILIKE '%failed%'", ctx); + + expect(sql).toContain("ilike(error_text,"); + }); + + it("should use text column for NOT LIKE comparison", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output NOT LIKE '%test%'", ctx); + + expect(sql).toContain("notLike(output_text,"); + }); + + it("should use JSON column for subfield comparison without .:String", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE output.data.name = 'test'", + ctx + ); + + // Should use the original JSON column, not the text column + // And should NOT have .:String in WHERE (breaks the query) + expect(sql).toContain("equals(output.data.name,"); + expect(sql).not.toContain("output_text"); + expect(sql).not.toContain("output.data.name.:String"); + }); + + it("should still use nullValue transformation for IS NULL", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE output IS NULL", ctx); + + // NULL check should use the text column with nullValue + expect(sql).toContain("equals(output_text, '{}')"); + }); + + it("should still use nullValue transformation for IS NOT NULL", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT id FROM runs WHERE error IS NOT NULL", ctx); + + expect(sql).toContain("notEquals(error_text, '{}')"); + }); + }); + + describe("edge cases", () => { + it("should work with columns without textColumn defined", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT status FROM runs WHERE status = 'completed'", ctx); + + // Regular column should work as before + expect(sql).toContain("status"); + expect(sql).not.toContain("status_text"); + }); + + it("should use text column for aliased JSON columns in SELECT", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT output AS result FROM runs", ctx); + + // Should use text column with user's alias + expect(sql).toContain("output_text AS result"); + }); + + it("should use text column for table-qualified JSON columns in SELECT", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery("SELECT runs.output FROM runs", ctx); + + // Should use text column + expect(sql).toContain("output_text AS output"); + }); + + it("should use text column in both SELECT and WHERE for same query", () => { + const ctx = createTextColumnContext(); + const { sql } = printQuery( + "SELECT output FROM runs WHERE output LIKE '%test%'", + ctx + ); + + // SELECT should use text column + expect(sql).toContain("output_text AS output"); + // WHERE should use text column + expect(sql).toContain("like(output_text,"); + }); + }); + + describe("JOINs with textColumn", () => { + // Create a second schema with the same JSON column names to test JOIN ambiguity + const runsSchemaWithTextColumn: TableSchema = { + name: "runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + id: { name: "id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "output_text", + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + const eventsSchemaWithTextColumn: TableSchema = { + name: "events", + clickhouseName: "trigger_dev.task_events_v2", + columns: { + id: { name: "id", ...column("String") }, + run_id: { name: "run_id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + textColumn: "output_text", + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function createJoinTextColumnContext() { + const schema = createSchemaRegistry([runsSchemaWithTextColumn, eventsSchemaWithTextColumn]); + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } + + it("should qualify text column with table alias in JOIN WHERE clause to avoid ambiguity", () => { + const ctx = createJoinTextColumnContext(); + const { sql } = printQuery( + `SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE r.output = '{}'`, + ctx + ); + + // The text column should be table-qualified to avoid ambiguity + // since both tables have an output_text column + expect(sql).toContain("equals(r.output_text,"); + // Should NOT have unqualified output_text in the comparison + expect(sql).not.toMatch(/equals\(output_text,/); + }); + + it("should qualify text column with table alias for LIKE in JOIN", () => { + const ctx = createJoinTextColumnContext(); + const { sql } = printQuery( + `SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE e.output LIKE '%error%'`, + ctx + ); + + // Should use table-qualified text column + expect(sql).toContain("like(e.output_text,"); + expect(sql).not.toMatch(/like\(output_text,/); + }); + + it("should handle multiple qualified text column comparisons in JOIN", () => { + const ctx = createJoinTextColumnContext(); + const { sql } = printQuery( + `SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE r.output = '{}' AND e.output != '{}'`, + ctx + ); + + // Both comparisons should be table-qualified + expect(sql).toContain("equals(r.output_text,"); + expect(sql).toContain("notEquals(e.output_text,"); + }); + }); + }); + + describe("dataPrefix for JSON columns", () => { + // Create a schema with JSON columns that have dataPrefix set + const dataPrefixSchema: TableSchema = { + name: "runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + id: { name: "id", ...column("String") }, + output: { + name: "output", + ...column("JSON"), + nullValue: "'{}'", + dataPrefix: "data", + }, + error: { + name: "error", + ...column("JSON"), + nullValue: "'{}'", + dataPrefix: "data", + }, + status: { name: "status", ...column("String") }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function createDataPrefixContext() { + const schema = createSchemaRegistry([dataPrefixSchema]); + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); + } + + describe("SELECT clause", () => { + it("should inject dataPrefix into JSON subfield path", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.message FROM runs", ctx); + + // Should transform output.message to output.data.message + expect(sql).toContain("output.data.message"); + }); + + it("should generate clean alias without dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql, columns } = printQuery("SELECT output.message FROM runs", ctx); + + // Alias should be output_message, not output_data_message + expect(sql).toContain("AS output_message"); + expect(sql).not.toContain("AS output_data_message"); + expect(columns).toContainEqual( + expect.objectContaining({ name: "output_message" }) + ); + }); + + it("should handle nested paths with dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.user.name FROM runs", ctx); + + // Should transform output.user.name to output.data.user.name + expect(sql).toContain("output.data.user.name"); + // Alias should be output_user_name + expect(sql).toContain("AS output_user_name"); + }); + + it("should work with multiple JSON columns with dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.msg, error.code FROM runs", ctx); + + expect(sql).toContain("output.data.msg"); + expect(sql).toContain("error.data.code"); + expect(sql).toContain("AS output_msg"); + expect(sql).toContain("AS error_code"); + }); + + it("should not affect bare JSON column selection", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output FROM runs", ctx); + + // Bare column should not have dataPrefix injected + expect(sql).not.toContain("output.data"); + expect(sql).toMatch(/SELECT\s+output[\s,]/); + }); + }); + + describe("WHERE clause", () => { + it("should inject dataPrefix into WHERE comparison", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE output.status = 'success'", + ctx + ); + + // Should transform output.status to output.data.status + expect(sql).toContain("output.data.status"); + }); + + it("should inject dataPrefix into LIKE comparison", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery( + "SELECT id FROM runs WHERE error.message LIKE '%failed%'", + ctx + ); + + expect(sql).toContain("error.data.message"); + }); + }); + + describe("GROUP BY clause", () => { + it("should inject dataPrefix into GROUP BY", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery( + "SELECT output.type, count() AS cnt FROM runs GROUP BY output.type", + ctx + ); + + // Should inject dataPrefix in both SELECT and GROUP BY + expect(sql).toContain("output.data.type"); + expect(sql).toContain("GROUP BY output.data.type"); + }); + }); + + describe("edge cases", () => { + it("should not affect columns without dataPrefix", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT status FROM runs", ctx); + + // Regular column should not be affected + expect(sql).toContain("status"); + expect(sql).not.toContain("status.data"); + }); + + it("should work with explicit alias on JSON subfield", () => { + const ctx = createDataPrefixContext(); + const { sql } = printQuery("SELECT output.message AS msg FROM runs", ctx); + + // Should inject dataPrefix but use user's alias + expect(sql).toContain("output.data.message"); + expect(sql).toContain("AS msg"); + }); + }); }); describe("ORDER BY clauses", () => { @@ -742,9 +1247,11 @@ describe("ClickHousePrinter", () => { describe("Tenant isolation", () => { it("should inject tenant guards for single table", () => { const context = createTestContext({ - organizationId: "org_abc", - projectId: "proj_def", - environmentId: "env_ghi", + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_abc" }, + project_id: { op: "eq", value: "proj_def" }, + environment_id: { op: "eq", value: "env_ghi" }, + }, }); const { sql, params } = printQuery("SELECT * FROM task_runs", context); @@ -1057,15 +1564,17 @@ describe("Value mapping (valueMap)", () => { function createValueMapContext() { const schema = createSchemaRegistry([statusMappedSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - it("should transform user-friendly value to internal value in equality comparison", () => { +it("should transform user-friendly value to internal value in equality comparison", () => { const ctx = createValueMapContext(); const { sql, params } = printQuery("SELECT * FROM runs WHERE status = 'Completed'", ctx); @@ -1173,15 +1682,17 @@ describe("WHERE transform (whereTransform)", () => { function createPrefixedContext() { const schema = createSchemaRegistry([prefixedIdSchema]); - return createPrinterContext({ - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, + }, + }); +} - it("should strip prefix from value in equality comparison", () => { +it("should strip prefix from value in equality comparison", () => { const ctx = createPrefixedContext(); const { params } = printQuery("SELECT * FROM runs WHERE batch_id = 'batch_abc123'", ctx); @@ -1397,16 +1908,18 @@ describe("Virtual columns", () => { function createVirtualColumnContext() { const schema = createSchemaRegistry([virtualColumnSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - describe("SELECT clause", () => { - it("should expand bare virtual column to expression with alias", () => { +describe("SELECT clause", () => { + it("should expand bare virtual column to expression with alias", () => { const ctx = createVirtualColumnContext(); const { sql } = printQuery("SELECT execution_duration FROM runs", ctx); @@ -1638,16 +2151,18 @@ describe("Expression columns with division (cost/invocation_cost pattern)", () = function createCostExpressionContext() { const schema = createSchemaRegistry([costExpressionSchema]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - describe("WHERE clause with division expression columns", () => { - it("should expand invocation_cost > 100 to (base_cost_in_cents / 100.0) > 100", () => { +describe("WHERE clause with division expression columns", () => { + it("should expand invocation_cost > 100 to (base_cost_in_cents / 100.0) > 100", () => { const ctx = createCostExpressionContext(); const { sql } = printQuery("SELECT * FROM runs WHERE invocation_cost > 100", ctx); @@ -1782,16 +2297,18 @@ describe("Column metadata", () => { function createMetadataTestContext() { const schema = createSchemaRegistry([schemaWithRenderTypes]); - return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", - schema, - }); - } + return createPrinterContext({ + schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, + }); +} - describe("Basic column metadata", () => { - it("should return column metadata for simple field references", () => { +describe("Basic column metadata", () => { + it("should return column metadata for simple field references", () => { const ctx = createMetadataTestContext(); const { columns } = printQuery("SELECT run_id, created_at FROM runs", ctx); @@ -2193,10 +2710,12 @@ describe("Unknown column blocking", () => { // Using the internal name directly should be blocked const schema = createSchemaRegistry([runsSchema]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); // 'created_at' is not in runsSchema - only 'created' which maps to 'created_at' @@ -2210,10 +2729,12 @@ describe("Unknown column blocking", () => { // When user types 'created_at', we should suggest 'created' const schema = createSchemaRegistry([runsSchema]); const ctx = createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); expect(() => { @@ -2346,9 +2867,6 @@ describe("Field Mapping Value Transformation", () => { function createFieldMappingContext(): PrinterContext { const schemaRegistry = createSchemaRegistry([fieldMappingSchema]); return new PrinterContext( - "org_123", - "proj_456", - "env_789", schemaRegistry, {}, { @@ -2356,6 +2874,11 @@ describe("Field Mapping Value Transformation", () => { proj_tenant1: "my-project-ref", proj_other: "other-project", }, + }, + { + organization_id: { op: "eq", value: "org_123" }, + project_id: { op: "eq", value: "proj_456" }, + environment_id: { op: "eq", value: "env_789" }, } ); } @@ -2474,20 +2997,24 @@ describe("Internal-only column blocking", () => { function createHiddenTenantContext(): PrinterContext { const schema = createSchemaRegistry([hiddenTenantSchema]); return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); } function createHiddenFilterContext(): PrinterContext { const schema = createSchemaRegistry([hiddenFilterSchema]); return createPrinterContext({ - organizationId: "org_test", - projectId: "proj_test", - environmentId: "env_test", schema, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test" }, + project_id: { op: "eq", value: "proj_test" }, + environment_id: { op: "eq", value: "env_test" }, + }, }); } @@ -2655,10 +3182,12 @@ describe("Required Filters", () => { function createRequiredFiltersContext(): PrinterContext { const schemaRegistry = createSchemaRegistry([schemaWithRequiredFilters]); return createPrinterContext({ - organizationId: "org_test123", - projectId: "proj_test456", - environmentId: "env_test789", schema: schemaRegistry, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_test123" }, + project_id: { op: "eq", value: "proj_test456" }, + environment_id: { op: "eq", value: "env_test789" }, + }, }); } diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index 75fcd5662..789f1836e 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -46,7 +46,7 @@ import { findTSQLFunction, validateFunctionArgs, } from "./functions"; -import { PrinterContext } from "./printer_context"; +import { PrinterContext, WhereClauseCondition } from "./printer_context"; import { findTable, validateTable, @@ -114,6 +114,8 @@ export class ClickHousePrinter { private outputColumns: OutputColumnMetadata[] = []; /** Whether we're currently processing GROUP BY expressions */ private inGroupByContext = false; + /** Whether the current query has a GROUP BY clause (used for JSON subfield type hints) */ + private queryHasGroupBy = false; /** Columns hidden when SELECT * is expanded to core columns only */ private hiddenColumns: string[] = []; /** @@ -392,6 +394,11 @@ export class ClickHousePrinter { } } + // Track if query has GROUP BY for JSON subfield type hint decisions + // (ClickHouse requires .:String for Dynamic types in GROUP BY, and SELECT must match) + const savedQueryHasGroupBy = this.queryHasGroupBy; + this.queryHasGroupBy = !!node.group_by; + // Process SELECT columns and collect metadata // Using flatMap because asterisk expansion can return multiple columns // Set inProjectionContext to block internal-only columns in user projections @@ -543,6 +550,7 @@ export class ClickHousePrinter { // Restore saved contexts (for nested queries) this.selectAliases = savedAliases; + this.queryHasGroupBy = savedQueryHasGroupBy; this.tableContexts = savedTableContexts; this.allowedInternalColumns = savedInternalColumns; this.internalOnlyColumns = savedInternalOnlyColumns; @@ -627,37 +635,47 @@ export class ClickHousePrinter { let sqlResult: string; if ((col as Field).expression_type === "field") { const field = col as Field; - const virtualColumnName = this.getVirtualColumnNameForField(field.chain); - if (virtualColumnName !== null) { - // Visit the field (which will return the expression) - const visited = this.visit(col); - // Add the alias to preserve the column name - sqlResult = `${visited} AS ${this.printIdentifier(virtualColumnName)}`; + // Check if this is a bare JSON field that should use a text column + const textColumn = this.getTextColumnForField(field.chain); + if (textColumn !== null && outputName) { + // Use the text column instead of the JSON column, with alias to preserve name + sqlResult = `${this.printIdentifier(textColumn)} AS ${this.printIdentifier(outputName)}`; } else { - // Visit the field to get the ClickHouse SQL - const visited = this.visit(col); + const virtualColumnName = this.getVirtualColumnNameForField(field.chain); - // Check if this is a JSON subfield access (will have .:String type hint) - // If so, add an alias to preserve the nice column name (dots → underscores) - const isJsonSubfield = this.isJsonSubfieldAccess(field.chain); - if (isJsonSubfield) { - // Build the alias using underscores (e.g., "error_data_name") - const aliasName = field.chain.filter((p): p is string => typeof p === "string").join("_"); - sqlResult = `${visited} AS ${this.printIdentifier(aliasName)}`; - // Override output name for metadata - effectiveOutputName = aliasName; - } - // Check if the column has a different clickhouseName - if so, add an alias - // to ensure results come back with the user-facing name - else if ( - outputName && - sourceColumn?.clickhouseName && - sourceColumn.clickhouseName !== outputName - ) { - sqlResult = `${visited} AS ${this.printIdentifier(outputName)}`; + if (virtualColumnName !== null) { + // Visit the field (which will return the expression) + const visited = this.visit(col); + // Add the alias to preserve the column name + sqlResult = `${visited} AS ${this.printIdentifier(virtualColumnName)}`; } else { - sqlResult = visited; + // Visit the field to get the ClickHouse SQL + const visited = this.visit(col); + + // Check if this is a JSON subfield access (will have .:String type hint) + // If so, add an alias to preserve the nice column name (dots → underscores) + const isJsonSubfield = this.isJsonSubfieldAccess(field.chain); + if (isJsonSubfield) { + // Build the alias using underscores, excluding any dataPrefix + // e.g., output.message -> "output_message" (not "output_data_message") + const dataPrefix = this.getDataPrefixForField(field.chain); + const aliasName = this.buildAliasWithoutDataPrefix(field.chain, dataPrefix); + sqlResult = `${visited} AS ${this.printIdentifier(aliasName)}`; + // Override output name for metadata + effectiveOutputName = aliasName; + } + // Check if the column has a different clickhouseName - if so, add an alias + // to ensure results come back with the user-facing name + else if ( + outputName && + sourceColumn?.clickhouseName && + sourceColumn.clickhouseName !== outputName + ) { + sqlResult = `${visited} AS ${this.printIdentifier(outputName)}`; + } else { + sqlResult = visited; + } } } } else if ( @@ -675,8 +693,23 @@ export class ClickHousePrinter { } else { sqlResult = visited; } + } else if ((col as Alias).expression_type === "alias") { + // Handle Alias expressions - check if inner expression is a bare JSON field with textColumn + const alias = col as Alias; + if ((alias.expr as Field).expression_type === "field") { + const innerField = alias.expr as Field; + const textColumn = this.getTextColumnForField(innerField.chain); + if (textColumn !== null) { + // Use the text column with the user's explicit alias + sqlResult = `${this.printIdentifier(textColumn)} AS ${this.printIdentifier(alias.alias)}`; + } else { + sqlResult = this.visit(col); + } + } else { + sqlResult = this.visit(col); + } } else { - // For Alias expressions or other types, visit normally + // For other types, visit normally sqlResult = this.visit(col); } @@ -817,6 +850,11 @@ export class ClickHousePrinter { if (isVirtualColumn(columnSchema)) { // Virtual column: use the expression with an alias sqlResult = `(${columnSchema.expression}) AS ${this.printIdentifier(columnName)}`; + } else if (columnSchema.textColumn) { + // JSON column with text column optimization: use the text column with alias + sqlResult = `${this.printIdentifier(columnSchema.textColumn)} AS ${this.printIdentifier( + columnName + )}`; } else { // Regular column: use the actual ClickHouse column name const clickhouseName = columnSchema.clickhouseName ?? columnName; @@ -1438,6 +1476,9 @@ export class ClickHousePrinter { // Look up table schema and get ClickHouse table name const tableSchema = this.lookupTable(tableName); + // Validate that required tenant columns are present in enforcedWhereClause + this.validateRequiredTenantColumns(tableSchema); + // Always add the TSQL table name as an alias if no explicit alias is provided // This ensures table-qualified column references work in WHERE clauses // (needed to avoid alias conflicts when columns have expressions) @@ -1486,8 +1527,8 @@ export class ClickHousePrinter { } } - // Add tenant isolation guard - extraWhere = this.createTenantGuard(tableSchema, effectiveAlias); + // Add enforced WHERE clause guard (tenant isolation + plan limits) + extraWhere = this.createEnforcedGuard(tableSchema, effectiveAlias); } else if ( (tableExpr as SelectQuery).expression_type === "select_query" || (tableExpr as SelectSetQuery).expression_type === "select_set_query" @@ -1534,77 +1575,202 @@ export class ClickHousePrinter { } // ============================================================ - // Tenant Isolation + // Enforced WHERE Clause // ============================================================ /** - * Create a WHERE clause expression for tenant isolation and required filters - * Note: We use just the column name without table prefix since ClickHouse - * requires the actual table name (task_runs_v2), not the TSQL alias (task_runs) + * Validate that required tenant columns are present in enforcedWhereClause. * - * Organization ID is always required. Project ID and Environment ID are optional - - * if not provided, the query will return results across all projects/environments. + * If a table defines `tenantColumns.organizationId`, the `enforcedWhereClause` + * MUST include that column to ensure tenant isolation. This prevents accidental + * data leaks when the caller forgets to include tenant isolation conditions. * - * Required filters from the table schema are also always included. + * @throws QueryError if a required tenant column is missing */ - private createTenantGuard(tableSchema: TableSchema, _tableAlias: string): And | CompareOperation { - const { tenantColumns, requiredFilters } = tableSchema; + private validateRequiredTenantColumns(tableSchema: TableSchema): void { + const { tenantColumns } = tableSchema; + if (!tenantColumns) return; - // Organization guard is always required - const orgGuard: CompareOperation = { - expression_type: "compare_operation", - op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [tenantColumns.organizationId] } as Field, - right: { expression_type: "constant", value: this.context.organizationId } as Constant, + // Organization ID is always required if the table defines it + if (tenantColumns.organizationId) { + const orgColumn = tenantColumns.organizationId; + if (!this.context.enforcedWhereClause[orgColumn]) { + throw new QueryError( + `Table '${tableSchema.name}' requires '${orgColumn}' in enforcedWhereClause for tenant isolation` + ); + } + } + // Note: projectId and environmentId are optional - no validation needed + } + + /** + * Format a Date as a ClickHouse-compatible DateTime64 string. + * ClickHouse expects format: 'YYYY-MM-DD HH:MM:SS.mmm' (in UTC) + */ + private formatDateForClickHouse(date: Date): string { + const year = date.getUTCFullYear(); + const month = String(date.getUTCMonth() + 1).padStart(2, "0"); + const day = String(date.getUTCDate()).padStart(2, "0"); + const hours = String(date.getUTCHours()).padStart(2, "0"); + const minutes = String(date.getUTCMinutes()).padStart(2, "0"); + const seconds = String(date.getUTCSeconds()).padStart(2, "0"); + const ms = String(date.getUTCMilliseconds()).padStart(3, "0"); + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}`; + } + + /** + * Create an AST expression for a value. + * Date values are wrapped in toDateTime64() for ClickHouse compatibility. + */ + private createValueExpression(value: Date | string | number): Expression { + if (value instanceof Date) { + // Wrap Date in toDateTime64(formatted_string, 3) for ClickHouse DateTime64(3) columns + return { + expression_type: "call", + name: "toDateTime64", + args: [ + { expression_type: "constant", value: this.formatDateForClickHouse(value) } as Constant, + { expression_type: "constant", value: 3 } as Constant, + ], + } as Call; + } + return { expression_type: "constant", value } as Constant; + } + + /** + * Map condition operator to CompareOperationOp + */ + private mapConditionOpToCompareOp( + op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" + ): CompareOperationOp { + switch (op) { + case "eq": + return CompareOperationOp.Eq; + case "neq": + return CompareOperationOp.NotEq; + case "gt": + return CompareOperationOp.Gt; + case "gte": + return CompareOperationOp.GtEq; + case "lt": + return CompareOperationOp.Lt; + case "lte": + return CompareOperationOp.LtEq; + } + } + + /** + * Create an AST expression from a WhereClauseCondition + * + * @param column - The column name + * @param condition - The condition to apply + * @param tableAlias - Optional table alias to qualify the column reference. + * When provided, constructs the field chain as [tableAlias, column] + * so resolveFieldChain will resolve to the correct table in multi-join queries. + * @returns The AST expression for the condition + */ + private createConditionExpression( + column: string, + condition: WhereClauseCondition, + tableAlias?: string + ): Expression { + // When tableAlias is provided, qualify the field chain to ensure it binds + // to the correct table in multi-join queries + const fieldExpr: Field = { + expression_type: "field", + chain: tableAlias ? [tableAlias, column] : [column], }; - // Collect all guards - org is always included - const guards: CompareOperation[] = [orgGuard]; - - // Only add project guard if projectId is provided - if (this.context.projectId !== undefined) { - const projectGuard: CompareOperation = { - expression_type: "compare_operation", - op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [tenantColumns.projectId] } as Field, - right: { expression_type: "constant", value: this.context.projectId } as Constant, + if (condition.op === "between") { + const betweenExpr: BetweenExpr = { + expression_type: "between_expr", + expr: fieldExpr, + low: this.createValueExpression(condition.low), + high: this.createValueExpression(condition.high), }; - guards.push(projectGuard); + return betweenExpr; } - // Only add environment guard if environmentId is provided - if (this.context.environmentId !== undefined) { - const envGuard: CompareOperation = { - expression_type: "compare_operation", - op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [tenantColumns.environmentId] } as Field, - right: { expression_type: "constant", value: this.context.environmentId } as Constant, - }; - guards.push(envGuard); + // Simple comparison + const compareExpr: CompareOperation = { + expression_type: "compare_operation", + left: fieldExpr, + right: this.createValueExpression(condition.value), + op: this.mapConditionOpToCompareOp(condition.op), + }; + return compareExpr; + } + + /** + * Create a WHERE clause expression for enforced conditions and required filters. + * + * This method applies: + * 1. All conditions from enforcedWhereClause (tenant isolation + plan limits) + * 2. Required filters from the table schema (e.g., engine = 'V2') + * + * Conditions are applied if the column exists in either: + * - The exposed columns (tableSchema.columns) + * - The tenant columns (tableSchema.tenantColumns) + * + * This ensures the same enforcedWhereClause can be used across different tables. + * + * All guard expressions are qualified with the table alias to ensure they bind + * to the correct table in multi-join queries, preventing potential security + * issues where an unqualified column reference could bind to the wrong table. + */ + private createEnforcedGuard(tableSchema: TableSchema, tableAlias: string): Expression | null { + const { requiredFilters, tenantColumns } = tableSchema; + const guards: Expression[] = []; + + // Build a set of valid columns for this table (exposed + tenant columns) + const validColumns = new Set(Object.keys(tableSchema.columns)); + if (tenantColumns) { + if (tenantColumns.organizationId) validColumns.add(tenantColumns.organizationId); + if (tenantColumns.projectId) validColumns.add(tenantColumns.projectId); + if (tenantColumns.environmentId) validColumns.add(tenantColumns.environmentId); } - // Add required filters from the table schema + // Apply all enforced conditions for columns that exist in this table + // Pass tableAlias to ensure guards are qualified and bind to the correct table + for (const [column, condition] of Object.entries(this.context.enforcedWhereClause)) { + // Skip undefined/null conditions (allows conditional inclusion like project_id?: condition) + if (condition === undefined || condition === null) { + continue; + } + // Only apply if column exists in this table's schema or is a tenant column + if (validColumns.has(column)) { + guards.push(this.createConditionExpression(column, condition, tableAlias)); + } + } + + // Add required filters from the table schema (e.g., engine = 'V2') + // Also qualified with table alias to ensure correct binding in multi-join queries if (requiredFilters && requiredFilters.length > 0) { for (const filter of requiredFilters) { const filterGuard: CompareOperation = { expression_type: "compare_operation", op: CompareOperationOp.Eq, - left: { expression_type: "field", chain: [filter.column] } as Field, + left: { expression_type: "field", chain: [tableAlias, filter.column] } as Field, right: { expression_type: "constant", value: filter.value } as Constant, }; guards.push(filterGuard); } } - // If only org guard, return it directly (no need for AND wrapper) + // Return null if no guards (empty enforcedWhereClause and no requiredFilters) + if (guards.length === 0) { + return null; + } + + // If only one guard, return it directly (no need for AND wrapper) if (guards.length === 1) { - return orgGuard; + return guards[0]; } return { expression_type: "and", exprs: guards, - }; + } as And; } // ============================================================ @@ -1711,7 +1877,39 @@ export class ClickHousePrinter { // Transform the right side if it contains user-friendly values const transformedRight = this.transformValueMapExpression(node.right, columnSchema); - const left = this.visit(node.left); + // Check if we should use a text column for bare JSON field comparisons + // This applies to: Eq, NotEq, Like, ILike, NotLike, NotILike + const textColumnOps = [ + CompareOperationOp.Eq, + CompareOperationOp.NotEq, + CompareOperationOp.Like, + CompareOperationOp.ILike, + CompareOperationOp.NotLike, + CompareOperationOp.NotILike, + ]; + const useTextColumn = textColumnOps.includes(node.op); + const leftTextColumn = useTextColumn ? this.getTextColumnForExpression(node.left) : null; + + // Build the left side, qualifying the text column with table alias if present + let left: string; + if (leftTextColumn) { + // Check if the field is qualified with a table alias (e.g., r.output) + // and prepend that alias to the text column to avoid ambiguity in JOINs + const fieldNode = node.left as Field; + if (fieldNode.expression_type === "field" && fieldNode.chain.length >= 2) { + const firstPart = fieldNode.chain[0]; + if (typeof firstPart === "string" && this.tableContexts.has(firstPart)) { + // The field is qualified with a table alias, prepend it to the text column + left = this.printIdentifier(firstPart) + "." + this.printIdentifier(leftTextColumn); + } else { + left = this.printIdentifier(leftTextColumn); + } + } else { + left = this.printIdentifier(leftTextColumn); + } + } else { + left = this.visit(node.left); + } const right = this.visit(transformedRight); switch (node.op) { @@ -2074,19 +2272,31 @@ export class ClickHousePrinter { return `(${virtualExpression})`; } + // Inject dataPrefix for JSON columns if needed (e.g., output.message -> output.data.message) + const chainWithPrefix = this.injectDataPrefix(node.chain); + // Try to resolve column names through table context - const resolvedChain = this.resolveFieldChain(node.chain); + const resolvedChain = this.resolveFieldChain(chainWithPrefix); // Print each chain element let result = resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join("."); // For JSON column subfield access (e.g., error.data.name), add .:String type hint - // This is required because ClickHouse's Dynamic/Variant types are not allowed in - // GROUP BY without type casting, and SELECT/GROUP BY expressions must match + // This is ONLY required when the query has GROUP BY, because: + // 1. ClickHouse's Dynamic/Variant types are not allowed in GROUP BY without type casting + // 2. SELECT/GROUP BY expressions must match + // For queries without GROUP BY, the .:String type hint actually breaks the query + // (returns NULL instead of the actual value) + // We also skip this in WHERE comparisons where it breaks the query if (resolvedChain.length > 1) { // Check if the root column (first part) is a JSON column const rootColumnSchema = this.resolveFieldToColumnSchema([node.chain[0]]); - if (rootColumnSchema?.type === "JSON") { + // Add .:String ONLY for GROUP BY queries, and NOT in WHERE comparisons + if ( + rootColumnSchema?.type === "JSON" && + this.queryHasGroupBy && + !this.isInWhereComparisonContext() + ) { // Add .:String type hint for JSON subfield access result = `${result}.:String`; } @@ -2114,6 +2324,20 @@ export class ClickHousePrinter { return false; } + /** + * Check if we're inside a WHERE/HAVING comparison operation. + * Unlike isInComparisonContext(), this does NOT include GROUP BY context. + * Used to skip .:String type hints in WHERE clauses where they break queries. + */ + private isInWhereComparisonContext(): boolean { + for (const node of this.stack) { + if ((node as CompareOperation).expression_type === "compare_operation") { + return true; + } + } + return false; + } + /** * Resolve field chain with table alias prefix to avoid alias conflicts. * This is used in WHERE clauses when a column has whereTransform to ensure @@ -2155,6 +2379,125 @@ export class ClickHousePrinter { return rootColumnSchema?.type === "JSON"; } + /** + * Check if a field should use a text column instead of the JSON column. + * Returns the text column name if the field is a bare JSON field with textColumn defined, + * or null if the original column should be used. + * + * A "bare" JSON field means selecting the entire column (e.g., SELECT output) + * rather than accessing a subfield (e.g., SELECT output.data.name). + */ + private getTextColumnForField(chain: Array): string | null { + if (chain.length === 0) return null; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return null; + + let columnSchema: ColumnSchema | null = null; + + if (chain.length === 1) { + // Unqualified: just column name + columnSchema = this.resolveFieldToColumnSchema(chain); + } else if (chain.length === 2) { + // Could be table.column (qualified) - check if first part is a table alias + const tableSchema = this.tableContexts.get(firstPart); + if (tableSchema) { + const columnName = chain[1]; + if (typeof columnName === "string") { + columnSchema = tableSchema.columns[columnName] || null; + } + } + // If not a table alias, it's JSON path access (e.g., output.data) - return null + } + // chain.length > 2 means JSON path access - return null + + return columnSchema?.textColumn ?? null; + } + + /** + * Get the text column for an expression if it's a bare JSON field. + * Returns null if the expression is not a field or doesn't have a textColumn. + */ + private getTextColumnForExpression(expr: Expression): string | null { + if ((expr as Field).expression_type !== "field") return null; + return this.getTextColumnForField((expr as Field).chain); + } + + /** + * Get the dataPrefix for a field chain if the root column has one defined. + * Returns null if the column doesn't have a dataPrefix or if this isn't a subfield access. + */ + private getDataPrefixForField(chain: Array): string | null { + if (chain.length < 2) return null; // Need at least column.subfield + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return null; + + // Check if first part is a table alias (table.column.subfield) + const tableSchema = this.tableContexts.get(firstPart); + if (tableSchema) { + // Qualified: table.column.subfield - need at least 3 parts + if (chain.length < 3) return null; + const columnName = chain[1]; + if (typeof columnName !== "string") return null; + const columnSchema = tableSchema.columns[columnName]; + return columnSchema?.dataPrefix ?? null; + } + + // Unqualified: column.subfield + const columnSchema = this.resolveFieldToColumnSchema([firstPart]); + return columnSchema?.dataPrefix ?? null; + } + + /** + * Inject dataPrefix into a field chain if the root column has one defined. + * e.g., [output, message] -> [output, data, message] when dataPrefix is "data" + * Returns the original chain if no dataPrefix applies. + */ + private injectDataPrefix(chain: Array): Array { + const dataPrefix = this.getDataPrefixForField(chain); + if (!dataPrefix) return chain; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return chain; + + // Check if first part is a table alias + const tableSchema = this.tableContexts.get(firstPart); + if (tableSchema) { + // Qualified: table.column.subfield -> table.column.dataPrefix.subfield + // [table, column, subfield] -> [table, column, dataPrefix, subfield] + return [chain[0], chain[1], dataPrefix, ...chain.slice(2)]; + } + + // Unqualified: column.subfield -> column.dataPrefix.subfield + // [column, subfield] -> [column, dataPrefix, subfield] + return [chain[0], dataPrefix, ...chain.slice(1)]; + } + + /** + * Build an alias name for a field chain, excluding the dataPrefix if present. + * e.g., [output, message] with dataPrefix "data" -> "output_message" + * This gives users clean column names without the internal data wrapper. + */ + private buildAliasWithoutDataPrefix( + chain: Array, + dataPrefix: string | null + ): string { + // Filter to just string parts and join with underscores + const parts = chain.filter((p): p is string => typeof p === "string"); + + if (dataPrefix) { + // Remove the dataPrefix from the parts (it's an implementation detail) + const prefixIndex = parts.indexOf(dataPrefix); + if (prefixIndex > 0) { + // Only remove if it's not the first element (column name) + parts.splice(prefixIndex, 1); + } + } + + return parts.join("_"); + } + /** * Resolve a field chain to its column schema (if it references a known column) */ @@ -2380,6 +2723,31 @@ export class ClickHousePrinter { return columnSchema.clickhouseName || columnSchema.name; } + // Check if this is a tenant column that's not exposed in the schema's columns + // These are internal columns used for tenant isolation guards + const { tenantColumns, requiredFilters } = tableSchema; + if (tenantColumns) { + if ( + columnName === tenantColumns.organizationId || + columnName === tenantColumns.projectId || + columnName === tenantColumns.environmentId + ) { + // Tenant columns are already ClickHouse column names, return as-is + return columnName; + } + } + + // Check if this is a required filter column (e.g., engine = 'V2') + // These are internal columns used for enforced filters + if (requiredFilters) { + for (const filter of requiredFilters) { + if (columnName === filter.column) { + // Required filter columns are already ClickHouse column names, return as-is + return columnName; + } + } + } + // Column not in schema - this is a security issue, block access // Check if the user typed a ClickHouse column name instead of the TSQL name for (const [tsqlName, colSchema] of Object.entries(tableSchema.columns)) { diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts index 15089e01f..2956e660d 100644 --- a/internal-packages/tsql/src/query/printer_context.ts +++ b/internal-packages/tsql/src/query/printer_context.ts @@ -18,6 +18,34 @@ export interface QuerySettings { timeoutSeconds?: number; } +/** + * A simple comparison condition (e.g., column > value) + */ +export interface SimpleComparisonCondition { + /** The comparison operator */ + op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte"; + /** The value to compare against */ + value: Date | string | number; +} + +/** + * A between condition (e.g., column BETWEEN low AND high) + */ +export interface BetweenCondition { + /** The between operator */ + op: "between"; + /** The low bound of the range */ + low: Date | string | number; + /** The high bound of the range */ + high: Date | string | number; +} + +/** + * A WHERE clause condition that can be either a simple comparison or a BETWEEN. + * Used for both enforcedWhereClause (always applied) and whereClauseFallback (default when user doesn't filter). + */ +export type WhereClauseCondition = SimpleComparisonCondition | BetweenCondition; + /** * Default query settings */ @@ -42,7 +70,7 @@ export interface QueryNotice { * Context for the TSQL to ClickHouse printer * * Holds: - * - Tenant IDs for automatic WHERE clause injection + * - Enforced WHERE conditions for tenant isolation and plan limits * - Schema registry for table/column validation * - Parameter accumulator for SQL injection safety * - Query settings and execution options @@ -64,23 +92,30 @@ export class PrinterContext { /** Runtime field mappings for dynamic value translation */ readonly fieldMappings: FieldMappings; + /** + * Enforced WHERE conditions that are ALWAYS applied at the table level. + * Used for tenant isolation (org_id, project_id, env_id) and plan-based limits. + * Applied to every table reference including subqueries, CTEs, and JOINs. + */ + readonly enforcedWhereClause: Record; + constructor( - /** The organization ID for tenant isolation (required) */ - public readonly organizationId: string, - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - public readonly projectId: string | undefined, - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - public readonly environmentId: string | undefined, /** Schema registry containing allowed tables and columns */ public readonly schema: SchemaRegistry, /** Query execution settings */ public readonly settings: QuerySettings = {}, /** Runtime field mappings for dynamic value translation */ - fieldMappings: FieldMappings = {} + fieldMappings: FieldMappings = {}, + /** + * Enforced WHERE conditions that are ALWAYS applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + */ + enforcedWhereClause: Record = {} ) { // Initialize with default settings this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings }; this.fieldMappings = fieldMappings; + this.enforcedWhereClause = enforcedWhereClause; } /** @@ -157,12 +192,10 @@ export class PrinterContext { */ createChildContext(): PrinterContext { const child = new PrinterContext( - this.organizationId, - this.projectId, - this.environmentId, this.schema, this.settings, - this.fieldMappings + this.fieldMappings, + this.enforcedWhereClause ); // Share the same values map so parameters are unified child.values = this.values; @@ -184,19 +217,30 @@ export class PrinterContext { * Options for creating a printer context */ export interface PrinterContextOptions { - /** The organization ID for tenant isolation (required) */ - organizationId: string; - /** The project ID for tenant isolation (optional - omit to query across all projects) */ - projectId?: string; - /** The environment ID for tenant isolation (optional - omit to query across all environments) */ - environmentId?: string; + /** Schema registry containing allowed tables and columns */ schema: SchemaRegistry; + /** Query execution settings */ settings?: QuerySettings; /** * Runtime field mappings for dynamic value translation. * Maps internal ClickHouse values to external user-facing values. */ fieldMappings?: FieldMappings; + /** + * REQUIRED: Conditions always applied at the table level. + * Must include tenant columns (e.g., organization_id) for multi-tenant tables. + * Applied to every table reference including subqueries, CTEs, and JOINs. + * + * @example + * ```typescript + * { + * organization_id: { op: "eq", value: "org_123" }, + * project_id: { op: "eq", value: "proj_456" }, + * triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) } + * } + * ``` + */ + enforcedWhereClause: Record; } /** @@ -204,12 +248,10 @@ export interface PrinterContextOptions { */ export function createPrinterContext(options: PrinterContextOptions): PrinterContext { return new PrinterContext( - options.organizationId, - options.projectId, - options.environmentId, options.schema, options.settings, - options.fieldMappings + options.fieldMappings, + options.enforcedWhereClause ); } diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts index a5153597a..21bc9ec95 100644 --- a/internal-packages/tsql/src/query/schema.ts +++ b/internal-packages/tsql/src/query/schema.ts @@ -214,6 +214,42 @@ export interface ColumnSchema { * ``` */ nullValue?: string; + /** + * Alternative text column to use when selecting or comparing the full JSON value. + * + * For JSON columns, this allows using a pre-materialized string column + * which is more efficient than reading from the JSON column directly. + * + * @example + * ```typescript + * { + * name: "output", + * type: "JSON", + * textColumn: "output_text", + * } + * ``` + */ + textColumn?: string; + /** + * Prefix path for JSON column data access. + * + * When set, user paths like `output.message` are automatically transformed + * to `output.data.message` in the actual query, and result aliases exclude + * the prefix (e.g., `output_message` instead of `output_data_message`). + * + * This is useful when JSON data is stored wrapped in a container object + * (e.g., `{"data": actualData}`) to handle arrays and primitives. + * + * @example + * ```typescript + * { + * name: "output", + * type: "JSON", + * dataPrefix: "data", // output.message → output.data.message + * } + * ``` + */ + dataPrefix?: string; } /** diff --git a/internal-packages/tsql/src/query/security.test.ts b/internal-packages/tsql/src/query/security.test.ts index f576c5e93..2fcca4777 100644 --- a/internal-packages/tsql/src/query/security.test.ts +++ b/internal-packages/tsql/src/query/security.test.ts @@ -53,10 +53,12 @@ const taskEventsSchema: TableSchema = { }; const defaultOptions: CompileTSQLOptions = { - organizationId: "org_tenant1", - projectId: "proj_tenant1", - environmentId: "env_tenant1", tableSchema: [taskRunsSchema, taskEventsSchema], + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + environment_id: { op: "eq", value: "env_tenant1" }, + }, }; function compile(query: string, options: Partial = {}) { @@ -412,8 +414,10 @@ describe("Optional Tenant Filters", () => { describe("Organization ID is always required", () => { it("should always inject organization guard even with optional project/env", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -431,8 +435,11 @@ describe("Optional Tenant Filters", () => { describe("Project ID is optional", () => { it("should inject org and project guards when project is provided", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: "proj_tenant1", - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + // environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -449,8 +456,10 @@ describe("Optional Tenant Filters", () => { it("should allow querying across all projects when projectId is omitted", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -479,8 +488,11 @@ describe("Optional Tenant Filters", () => { it("should allow querying across all environments when environmentId is omitted", () => { const { sql, params } = compile("SELECT * FROM task_runs", { - projectId: "proj_tenant1", - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + project_id: { op: "eq", value: "proj_tenant1" }, + // environment_id omitted + }, }); const whereClause = getWhereClause(sql); @@ -501,8 +513,10 @@ describe("Optional Tenant Filters", () => { const { sql, params } = compile( "SELECT * FROM task_runs WHERE organization_id = 'org_other'", { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -518,8 +532,10 @@ describe("Optional Tenant Filters", () => { JOIN task_events e ON r.id = e.run_id `, { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -540,8 +556,10 @@ describe("Optional Tenant Filters", () => { SELECT id, status FROM task_runs WHERE status = 'failed' `, { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -557,8 +575,10 @@ describe("Optional Tenant Filters", () => { WHERE id IN (SELECT run_id FROM task_events) `, { - projectId: undefined, - environmentId: undefined, + enforcedWhereClause: { + organization_id: { op: "eq", value: "org_tenant1" }, + // project_id and environment_id omitted + }, } ); @@ -570,6 +590,95 @@ describe("Optional Tenant Filters", () => { }); }); +describe("Multi-join Tenant Guard Qualification", () => { + /** + * Security Test: Verifies that tenant guards are properly table-qualified in multi-join queries. + * + * The bug: createEnforcedGuard was building unqualified guard expressions like: + * organization_id = 'org_tenant1' + * + * In a multi-table join where both tables have the same column (organization_id), + * an unqualified reference could potentially bind to the wrong table during resolution, + * or be ambiguous. The guards should be qualified like: + * r.organization_id = 'org_tenant1' AND e.organization_id = 'org_tenant1' + * + * This ensures each table's guard binds to the correct table, not just any matching column. + */ + it("should qualify tenant guards with table alias in JOIN queries", () => { + const { sql } = compile(` + SELECT r.id, e.event_type + FROM task_runs r + JOIN task_events e ON r.id = e.run_id + `); + + // The guards should be table-qualified to prevent binding to the wrong table + // Look for pattern like: r.organization_id and e.organization_id (with table alias prefix) + // The exact format in ClickHouse SQL is just "alias.column" after resolution + + // Count qualified organization_id references (should have table prefixes) + // In the WHERE clause, we should see both r.organization_id and e.organization_id + const whereClause = sql.substring(sql.indexOf("WHERE")); + + // Both tables should have their own qualified tenant guards + // The pattern should be: table_alias.organization_id for each table + expect(whereClause).toMatch(/\br\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be\b[^,]*organization_id/); + }); + + it("should qualify tenant guards with table alias in LEFT JOIN queries", () => { + const { sql } = compile(` + SELECT r.id, e.event_type + FROM task_runs r + LEFT JOIN task_events e ON r.id = e.run_id + `); + + const whereClause = sql.substring(sql.indexOf("WHERE")); + + // Both tables should have qualified guards + expect(whereClause).toMatch(/\br\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be\b[^,]*organization_id/); + }); + + it("should qualify tenant guards in multi-way JOIN queries", () => { + const { sql } = compile(` + SELECT r.id, e1.event_type, e2.event_type + FROM task_runs r + JOIN task_events e1 ON r.id = e1.run_id + JOIN task_events e2 ON r.id = e2.run_id + `); + + const whereClause = sql.substring(sql.indexOf("WHERE")); + + // All three table aliases should have qualified guards + expect(whereClause).toMatch(/\br\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be1\b[^,]*organization_id/); + expect(whereClause).toMatch(/\be2\b[^,]*organization_id/); + }); + + it("should ensure guards cannot bind to wrong table by verifying separate qualifications", () => { + const { sql, params } = compile(` + SELECT r.id, e.event_type + FROM task_runs r + JOIN task_events e ON r.id = e.run_id + WHERE r.status = 'completed' + `); + + // Count organization_id occurrences with different table prefixes + // This ensures each table gets its own guard, not shared/ambiguous references + const orgIdPattern = /(\w+)\.organization_id/g; + const matches = [...sql.matchAll(orgIdPattern)]; + const tableAliases = matches.map(m => m[1]); + + // Should have at least 2 different table aliases for organization_id + // (one for task_runs alias 'r' and one for task_events alias 'e') + expect(tableAliases).toContain("r"); + expect(tableAliases).toContain("e"); + + // Both should use the same tenant value (parameterized) + expect(Object.values(params)).toContain("org_tenant1"); + }); +}); + describe("Edge Cases", () => { it("should handle empty string values", () => { const { params } = compile("SELECT * FROM task_runs WHERE status = ''"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d73bef99f..525beac47 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -497,8 +497,8 @@ importers: specifier: workspace:* version: link:../../internal-packages/otlp-importer '@trigger.dev/platform': - specifier: 1.0.21 - version: 1.0.21 + specifier: 1.0.22 + version: 1.0.22 '@trigger.dev/redis-worker': specifier: workspace:* version: link:../../packages/redis-worker @@ -10301,8 +10301,8 @@ packages: react: ^18.2.0 react-dom: 18.2.0 - '@trigger.dev/platform@1.0.21': - resolution: {integrity: sha512-D1p+Y5pj21Un8hhN7oS/X7c+mhHKL58w1nwI9XYxbKUK1cNIIVhEMNZ0IyYmYuLelSARUXYePlKSl0v4hlusZg==} + '@trigger.dev/platform@1.0.22': + resolution: {integrity: sha512-tvPf40wqEDcQCZsHt/9A+WoQ08z+uObSWQ+oahqCgp3dSgKOUH8NdzZ/2ISSRiCkN2jURixNiUyDJmgsZipExg==} '@types/acorn@4.0.6': resolution: {integrity: sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==} @@ -30255,7 +30255,7 @@ snapshots: react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - '@trigger.dev/platform@1.0.21': + '@trigger.dev/platform@1.0.22': dependencies: zod: 3.23.8 From 9e087127495bbc3b5fa4730f89f5889ae1a26ba5 Mon Sep 17 00:00:00 2001 From: DKP <8297864+D-K-P@users.noreply.github.com> Date: Thu, 29 Jan 2026 13:55:43 +0000 Subject: [PATCH 04/22] Add building with ai/skills pages and updated intro (#2962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes # ## ✅ Checklist - [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [ ] The PR title follows the convention. - [ ] I ran and tested the code works --- ## Testing _[Describe the steps you took to test this change]_ --- ## Changelog _[Short description of what has changed]_ --- ## Screenshots _[Screenshots]_ 💯 --- Open with Devin --- docs/building-with-ai.mdx | 24 ++ docs/docs.json | 15 +- docs/images/intro-ai.jpg | Bin 0 -> 5495 bytes docs/introduction.mdx | 4 +- docs/mcp-agent-rules.mdx | 1 - docs/mcp-introduction.mdx | 399 +++++++++++++++++------- docs/mcp-tools.mdx | 520 ++++---------------------------- docs/quick-start.mdx | 28 +- docs/skills.mdx | 83 +++++ docs/snippets/step-cli-init.mdx | 15 +- 10 files changed, 492 insertions(+), 597 deletions(-) create mode 100644 docs/building-with-ai.mdx create mode 100644 docs/images/intro-ai.jpg create mode 100644 docs/skills.mdx diff --git a/docs/building-with-ai.mdx b/docs/building-with-ai.mdx new file mode 100644 index 000000000..ba8cd5bb4 --- /dev/null +++ b/docs/building-with-ai.mdx @@ -0,0 +1,24 @@ +--- +title: "Overview" +sidebarTitle: "Overview" +description: "Tools and resources for building Trigger.dev projects with AI coding assistants." +--- + +We provide tools to help you build Trigger.dev projects with AI coding assistants. We recommend using them for the best developer experience. + + + + Give your AI assistant direct access to Trigger.dev tools - search docs, trigger tasks, deploy projects, and monitor runs. + + ```bash + npx trigger.dev@latest install-mcp + ``` + + + Portable instruction sets that teach any AI coding assistant Trigger.dev best practices for writing tasks, configs, and more. + + ```bash + npx skills add triggerdotdev/skills + ``` + + diff --git a/docs/docs.json b/docs/docs.json index cff62e6d0..405f4eb0a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -43,6 +43,17 @@ "apikeys" ] }, + { + "group": "Building with AI", + "pages": [ + "building-with-ai", + { + "group": "MCP Server", + "pages": ["mcp-introduction", "mcp-tools", "mcp-agent-rules"] + }, + "skills" + ] + }, { "group": "Writing tasks", "pages": [ @@ -166,10 +177,6 @@ } ] }, - { - "group": "MCP Server", - "pages": ["mcp-introduction", "mcp-tools", "mcp-agent-rules"] - }, { "group": "Using the Dashboard", "pages": ["run-tests", "troubleshooting-alerts", "replaying", "bulk-actions"] diff --git a/docs/images/intro-ai.jpg b/docs/images/intro-ai.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9905922a22b3b586e618754f9d975831b7858f57 GIT binary patch literal 5495 zcmbuDbwCtRx4>tY1(s%kMY=;uT1q7Z0TG0SC8ZlA1?i9y7NtQ_y1N@`6c9xbBvlZQ z5@8Vug*R)w?|py2voW*xo;mm2bI+Xp-8p}M{t=+Oc|-jM0D%Ai1bhMK^FRau`y=@H zFan$q!3hcBL}bJ`A)_IqAOqhtRJ2qG@WV!b;Q~DyH!BM(w-6tnkj!Nn4GkN}|JT5I zKY)Nk=pY~QARGV`0l`B+&IbWW&`k&c#RJ{^ClDwBAs!5l4|YHRJlrh)2^4|{!zUm- zp94t23!s#Elwi-%l&9Js1xVg&iADv04;fveVnzDzBJfBeSyid+I;Aban4gEIKmd>u z@Qd?v>2r&-^@6?7L}eWYdI!4=lPvGs)Xoc+rx|t0nxp2DHEr|Lvf3SNOYH218Li3c zA0}Co6-Uixyw%>joW_`u;rOTw!vR_##&x?yHM$|9WSUy5bb#awwRPyMtw^5spzaNmVB5_(z%E+${l6*BxUhQb;b zFI2Div~!2`*$SvTTa9|k7x_PIeWtR1Vo)7$n%869E;m-#T~qxF^~G%Z^n+9qg-Pgg zx4{Y)%aVQ=6!PjS-IXPUNDZQ{O*%xXAX_&16*$boLHK&I@yiBUl9Ihni-{o)aotO-;rU#HPIrEFb z`=2|lYxjnYFwNS8kudpL(lu<7y#74Lhaa9)(^zR?D5M?`xJ3TyfOFrk)Vk*!c=_|g zo2zj^7v8&1U3=3#37^bS?!wt$y;_3g_7;yCBPg9^SH`1zBd2}nR}lFl5=umbEVMTS z+rpET4vU+FwwW@Qr0&wbHqmjj682*i-+WR&bP&RQEY%oXSh`TSd2+qBV`kKsSI75E zd~$WcWN61XCtg3tScy2HKn#vEOF{Ze*Z9;d&M{Vj2-(cmGhE%Z&-p+ znMGsL*hsVjTRpuc9UW~82_a8aQ>@^Uly^!Crij+CSi@LI-fbf!S5fav*x#Hd%1BZpjldw^T z5>yE&9SD^R@n9=0O!ZKsMQFA>r$RJCnM}y*XFNedKbR?A9%FOd(o(^0{P)L<3>P+v*>x=z3K@mpx0F-_TGSuq2KW6@T*;X#Qe4_JeD-7upwSv~lzLALOUd12UCgRK5> zvoN*8_{$gLO=)^62yid}aID($;V(kw=M~?mAbzZe#8;pLM5suSGE%V@#la!|NJ9c7 zZ;_0otXhHKiwH^pVTkFYD;p;jEc&PL=SUC=0KNa32*H|kg%W^3@n8gSd>91w+Zy;2 zfFbay*zyS|MMP1agzRDpx|R>>sW}vNJZO{>C&iJR+67i&{a|T=gB1%R5A3F3ROFW@ zWK|?*&>qtQ$hc}fOh_N|x~b$$moBh-b4LJM(R~_-A^?|XiLczWIQYH<#(5JnH^}PMWJBEjjDE z7al_I#ST1>8T!=xYr3FmX6X{*8^Z{8uxg%%Co3 zWcTo|Q{tF7NVc9k>2hI&J(7t7oRh(GmB0x9yBm=~%~)oyU9aB9>)(Q!XWDcbPMz0e zE9;AN(URWIMJPU+IZnUTYl~|M&-R7WSHNa(18X5ZN!c$>#-=0L#jcl=3R7-r+`1`S zQbOnPgy2AkoBxc|0L_sFbGuS;IHH;GzP6>uIx4dgId|a0mf=kDog4adzQT03dN3=` z#=0m2)@?hc6+yDU3oAD_iJdX^8jPDuFbGE@aNBwtwD!$U4jYj0hu zYnZRDZhpL>%@%(>tW9SQ4Bgb_0&@P3_rT!Ql`vB$oCDf#NF)m~^T`9z`>j~}%A$6P za7XnCF7I1n+BJ5s4!jTCV??bwTEc>5f)uf*&-Pd!1@Um){&h{!sq>^ZUfF~8i<`fG zdLZ$Hd=Vlt=EBiVme3lJ!fC<< z*0Px*V-xnmkC*ezGqo!W)r$!iRljfq+&+l9TPA_^ER(;HAn>!IUQ$X@cH_RC?_&%q(p4#nOMvX~W&#jIvfv(&y$8?EV zuKrs3jy(fA+{+qw3yJ-N>A1DnDQsdA>}))3wks>n*QBI=rwS&9Mvr!sJU?0F z`6e@>PTd#FVx}jsmJz5I^!kI*M0y<8{doTF&h$Zc+_G^(xmQbgmadC!3#}t(|KuWh zeD!e$-)G5-;miCLXViuXU*^WFHnKDQZSU|EI*gV|C)96u3Zk@B(lq&eDbmmM{3rO@ z`P2`4Xr>z73L_nf=?BjN)ily~XyLtL8egQJe19eNFUEONz0q6Z61jQj00VY&f_HmN zY1~~(=HeEyTy0$8k=!G-Io(6+;#rwiIwKlxLSHPaVRmkIlbP_#@wU^H;?>O8K@Z1hK)t5_SNhsn)JHJUJur!C31X|bQ;fAm%9MJ zYJW8ConuezbHaL8)kze7ujSc`AvH=Obr=JJiovLe)*DYhxl-X*99Sgfcc_?oIxc3M zO7fppN9vwcoGp*N*&?1snb_UEyxA)8rLFMjgh{`++vfd?s_kV)9VA(NhW%^hb3nb} zMOOx+j4NxwlD66zzEy~pM4W%&_h+Swzp6+wzE?_nubUaCZN+-n`=_Y$PHoAt6Ya3* z?nG}B3~W>>;p$~i=I~XDw^HeTS+Z|@t5dV8uQg7G9}r)WmUyL+Y8}qWm47l+gsd%# zN(wm<*{Zp=yx=_~#L%sb6zksbqt!e+X$s8h?7>yzRF-$xmLhp zJL{wW_9b5{r9;t0?eI174i%4daMW6tw%PW1b5YE;Y(eIMU7%6RPL_x!B@5h~{|t5S z-5YSFMw(YwA;-2xNLJhSY0r{M=!j{%lPK4dYYb?#kXQ9oY1L3pzVTIiQ|%;ba*Czn zXKCB{d^f46QbK_|yfU(T)95V6g6+@Aw}UXrSc<+S&YMjPHq;*b?|;sG70r*q^3Vmvq$O-ouuo6%r{+a+ zoxx`u9l2v|QhA($n%W4DlCiZ%h++n>SYypudOq)#v4}ubrs7pDhS!izq@;l{u4*r3 zgGF(x%KNS}5*KpA*Yh4*Jkw7c+CntNKDvAk;;+f8)iic?>9jWdT-Vu1O>(4JxtZ8~ zld>2|RZOKW$+gRCc?Z73f)4OO#C3pZ!EqeNFfxz698jq0a^EG3U*Galu~yWZCvPZ> z_3_E9Z|_7^3mBEqc}b5K5IZGn-_faPp27E!>e{cIqwq6#2^?Xz>evWtQ{-q8)zphC zt1_?%$KtHZuHZWD4XgA8n2kFME74ZFykXscgO z;Jwz$k5iRD#77Sf@)xomfsh4G1#=9qO^DZQ_7-lceer%;AS&Izd0L=B3rC!(a|UYy zAIhuceo^@AV{xG%^p~5E{kVy-DydKr(HE#=-Sgi40MalU0^Djt!5#Yl?!+Mg0!9VJ z#{*fIP2o53LLm?+@Ux=Q)VreM9Kc8Gcf3QRovYEop-40uUDJ$4tHm4v_yChrMMXuW z!DBQB?>I;66M2w%80Fc zk6$4VDV_#idzByfC|EQe?6ZU`vBH%oJx*O)XL-!@Z#VdP(hAUxtk4aM;H*(9EBoKg zbr^kSFmT8ui3aoLk@Yv_Fb)n*pRMkmVDnl(RnAEcP6Prd!! zR;N~{b3lpnzGr9W+^%zHYtkGT4l7Qd0GF0|@NK$SQM=;vR~kO-`1rKU;9Adtw6DA0 z!B~qS04TWEz{e-V|NGbjw-G@AREqf2?4k;k9F`u5G@LppEBAacaV5_O^?##}Cl8!b z8CYX>_Lf`+vpU_M`&U#Ky@AsN)RE9$W^$jDD~uHr)n|o@r~^>-As3tSu9)ShANH2k zZh!})lBK}cQJb-Yx#=Nqn)tK}X#zh?@sE5TjL~u-tfyS>-~SdTF5~vioo>NJMvaLZ zs(AKI+B8SMvB_u{)iPzSgJJjqk3(I5!DRS!l$cPov+=o10ZC%y95P8y(4?-H#CbsX z?W-mpB8KeN-5!ic!gU%o!LoeSgaL!zc*?GNvw=0jTcCF+_+_?OMAQv^)O9`>BA>41 zxn$_0VbX!qM7Pth!6aApcHR=>(dTTAVua4L!{cFt{kHP-Waj`=U1FJ@3YsaDmbs}} zz8fkZ!xZG9LMDM7+@V(yO35n!Hkj4DwAWyE?CKJ7sn7v6@1u9fIuvJl+?%NqRgENJ z6pS4`yP7cBi@i0ivw6GMssW|UZfUOnUAnx&%}+2X2g3E;uF_stM#@4=D)Za0f0<+7 zCk8vtP1j+ywF)n`?o{FZID0Y;vwc|8J)e|~AXT)caw_H2SfV<4)ZpG{Rnh=cTjdkE zq@Fubmz4^Mb6%IdS`_naeQ-qt4x_x0HYBS&oL`98dO8~-J^RwgpHj?*z9IIK8+l#B zE4AH|fW0uUDbfJ5$y78}g~CJ7*If579?TUM1OS2k*Ml=W2o%f}1xhv%?caF;-m*~O zpeD84{C<~!3)kI*@;)2G&t5eflUKMq1Rja0_Ki@UTuLNfzaeH-Koa@(7Uv?LK@_D} zW#p9ATxOwfbMb?rwZY{TTwf z!;ec{JSdC++$#SO2owR}u~R8fit1QWbBZatdnlo-9_0UZ4NTc1-idE3)YKgsI%U{A z1-6ghT0#d1uHw_@`emx(v9Ub7CuRK|S@C=aBc)v?Y9JKDp{mor+Q+7_=`gR*DCgbO zaQk(^xFMD&vV^@`NFnrw3Y2Xo_tQBbjCFOHw}V=s*M1gm#)#e8^EzlpdgP!(#jG4| z^*-&4z5Np35kDb$KKzbFrGA>E27@x4c4zm6J0Vv2^c_#B(2&(+#hnK-cU&_+>Fe+& pR?|-U6Zfc1%39BdQ%Q^;R_0$B$XK$yLq6POO!wl&QT+Mr{{UzMte*e? literal 0 HcmV?d00001 diff --git a/docs/introduction.mdx b/docs/introduction.mdx index fa57a4586..383040de1 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -16,8 +16,8 @@ mode: "center" > Browse our wide range of guides, frameworks and example projects - - Learn how to install and configure the Trigger.dev MCP Server + + Learn how to build Trigger.dev projects using AI coding assistants Watch an end-to-end demo of Trigger.dev in 10 minutes diff --git a/docs/mcp-agent-rules.mdx b/docs/mcp-agent-rules.mdx index 664d8bcf2..321f312a8 100644 --- a/docs/mcp-agent-rules.mdx +++ b/docs/mcp-agent-rules.mdx @@ -2,7 +2,6 @@ title: "Agent rules" sidebarTitle: "Agent rules" description: "Learn how to use the Trigger.dev agent rules with the MCP server" -tag: "new" --- ## What are Trigger.dev agent rules? diff --git a/docs/mcp-introduction.mdx b/docs/mcp-introduction.mdx index d9dc3474e..257522d57 100644 --- a/docs/mcp-introduction.mdx +++ b/docs/mcp-introduction.mdx @@ -2,7 +2,6 @@ title: "MCP Introduction" sidebarTitle: "Introduction" description: "Learn how to install and configure the Trigger.dev MCP Server" -tag: "new" --- ## What is the Trigger.dev MCP Server? @@ -18,44 +17,306 @@ The Trigger.dev MCP (Model Context Protocol) Server enables AI assistants to int ## Installation -### Automatic Installation (Recommended) - -The easiest way to install the Trigger.dev MCP Server is using the interactive installation wizard: +The quickest way to get set up is the interactive installer: ```bash npx trigger.dev@latest install-mcp ``` -This command will guide you through: +It will detect your installed clients and configure them automatically. You can also copy-paste the config for your client below. -1. Selecting which MCP clients to configure -2. Choosing installation scope (user, project, or local) -3. Automatically configuring the selected clients +## Client Configuration -## Command Line Options +Each client has a slightly different config format. Copy the snippet for your client into the appropriate file. -The `install-mcp` command supports the following options: + + + Install using the command line: -### Core Options + ```bash + npx trigger.dev@latest install-mcp --client claude-code + ``` -- `-p, --project-ref ` - Scope the MCP server to a specific Trigger.dev project by providing its project ref -- `-t, --tag ` - The version of the trigger.dev CLI package to use for the MCP server (default: latest or v4-beta) -- `--dev-only` - Restrict the MCP server to the dev environment only -- `--yolo` - Install the MCP server into all supported clients automatically -- `--scope ` - Choose the scope of the MCP server: `user`, `project`, or `local` -- `--client ` - Choose specific client(s) to install into + Or add this configuration to `~/.claude.json` (user) or `.mcp.json` (project): -### Configuration Options + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` -- `--log-file ` - Configure the MCP server to write logs to a file -- `-a, --api-url ` - Configure a custom Trigger.dev API URL -- `-l, --log-level ` - Set CLI log level (debug, info, log, warn, error, none) + [View Claude Code MCP docs ↗](https://code.claude.com/docs/en/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client cursor + ``` + + Or add this configuration to `~/.cursor/mcp.json` (user) or `.cursor/mcp.json` (project): + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Cursor MCP docs ↗](https://cursor.com/docs/context/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client windsurf + ``` + + Or add this configuration to `~/.codeium/windsurf/mcp_config.json`: + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Windsurf MCP docs ↗](https://docs.windsurf.com/windsurf/cascade/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client vscode + ``` + + Or add this configuration to `.vscode/mcp.json` (project) or `~/Library/Application Support/Code/User/mcp.json` (user, macOS): + + ```json + { + "servers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + VS Code uses `servers` instead of `mcpServers`. + + [View VS Code MCP docs ↗](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client zed + ``` + + Or add this configuration to `~/.config/zed/settings.json`: + + ```json + { + "context_servers": { + "trigger": { + "source": "custom", + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Zed context servers docs ↗](https://zed.dev/docs/ai/mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client cline + ``` + + Or add this configuration to `~/Library/Application Support/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json`: + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Cline MCP docs ↗](https://docs.cline.bot/mcp/configuring-mcp-servers) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client gemini-cli + ``` + + Or add this configuration to `~/.gemini/settings.json` (user) or `.gemini/settings.json` (project): + + ```json + { + "mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client amp + ``` + + Or add this configuration to `~/.config/amp/settings.json`: + + ```json + { + "amp.mcpServers": { + "trigger": { + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Sourcegraph AMP MCP docs ↗](https://ampcode.com/manual#mcp) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client openai-codex + ``` + + Or add this configuration to `~/.codex/config.toml`: + + ```toml + [mcp_servers.trigger] + command = "npx" + args = ["trigger.dev@latest", "mcp"] + ``` + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client crush + ``` + + Or add this configuration to `.crush.json` (project), `crush.json`, or `~/.config/crush/crush.json` (user). Files are loaded in priority order: `.crush.json` → `crush.json` → `$HOME/.config/crush/crush.json`. + + ```json + { + "mcp": { + "trigger": { + "type": "stdio", + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + [View Charm MCP docs ↗](https://github.com/charmbracelet/crush) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client opencode + ``` + + Or add this configuration to `~/.config/opencode/opencode.json` (user) or `./opencode.json` (project): + + ```json + { + "mcp": { + "trigger": { + "type": "local", + "command": ["npx", "trigger.dev@latest", "mcp"], + "enabled": true + } + } + } + ``` + + [View opencode MCP docs ↗](https://opencode.ai/docs/mcp-servers/) + + + Install using the command line: + + ```bash + npx trigger.dev@latest install-mcp --client ruler + ``` + + Or add this configuration to `.ruler/mcp.json`: + + ```json + { + "mcpServers": { + "trigger": { + "type": "stdio", + "command": "npx", + "args": ["trigger.dev@latest", "mcp"] + } + } + } + ``` + + + +After adding the config, restart your client. You should see a server named **trigger** connect automatically. ## Authentication -You can use the MCP server without authentication with the `search_docs` tool, but for any other tool call you will need to authenticate the MCP server via the same method as the [Trigger.dev CLI](/cli-login-commands).The first time you attempt to use a tool that requires authentication, you will be prompted to authenticate the MCP server via the MCP client. +The `search_docs` tool works without authentication. All other tools require you to be logged in via the [Trigger.dev CLI](/cli-login-commands). The first time you use an authenticated tool, your MCP client will prompt you to log in. -### Examples + + +The `install-mcp` command supports these options: + +**Core Options** + +- `-p, --project-ref ` — Scope the MCP server to a specific project +- `-t, --tag ` — CLI package version to use (default: latest) +- `--dev-only` — Restrict to the dev environment only +- `--yolo` — Install into all supported clients automatically +- `--scope ` — `user`, `project`, or `local` +- `--client ` — Install into specific client(s) + +**Configuration Options** + +- `--log-file ` — Write logs to a file +- `-a, --api-url ` — Custom Trigger.dev API URL +- `-l, --log-level ` — Log level (debug, info, log, warn, error, none) + +**Examples** Install for all supported clients: @@ -69,105 +330,21 @@ Install for specific clients: npx trigger.dev@latest install-mcp --client claude-code cursor --scope user ``` -Install with development environment restriction: +Restrict to dev environment for a specific project: ```bash npx trigger.dev@latest install-mcp --dev-only --project-ref proj_abc123 ``` -## Supported MCP Clients - -The Trigger.dev MCP Server supports the following clients: - -| Client | Scope Options | Configuration File | Documentation | -| -------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Claude Code** | user, project, local | `~/.claude.json` or `./.mcp.json` (project/local scope) | [Claude Code MCP Docs](https://docs.anthropic.com/en/docs/claude-code/mcp) | -| **Cursor** | user, project | `~/.cursor/mcp.json` (user) or `./.cursor/mcp.json` (project) | [Cursor MCP Docs](https://docs.cursor.com/features/mcp) | -| **VSCode** | user, project | `~/Library/Application Support/Code/User/mcp.json` (user) or `./.vscode/mcp.json` (project) | [VSCode MCP Docs](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) | -| **Zed** | user | `~/.config/zed/settings.json` | [Zed Context Servers Docs](https://zed.dev/docs/context-servers) | -| **Windsurf** | user | `~/.codeium/windsurf/mcp_config.json` | [Windsurf MCP Docs](https://docs.codeium.com/windsurf/mcp) | -| **Gemini CLI** | user, project | `~/.gemini/settings.json` (user) or `./.gemini/settings.json` (project) | [Gemini CLI MCP Tutorial](https://medium.com/@joe.njenga/gemini-cli-mcp-tutorial-setup-commands-practical-use-step-by-step-example-b57f55db5f4a) | -| **Charm Crush** | user, project, local | `~/.config/crush/crush.json` (user), `./crush.json` (project), or `./.crush.json` (local) | [Charm MCP Docs](https://github.com/charmbracelet/mcp) | -| **Cline** | user | `~/Library/Application Support/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json` | [Cline MCP Docs](https://github.com/saoudrizwan/claude-dev#mcp) | -| **OpenAI Codex CLI** | user | `~/.codex/config.toml` | See OpenAI Codex CLI documentation for MCP configuration | -| **Sourcegraph AMP** | user | `~/.config/amp/settings.json` | [Sourcegraph AMP MCP Docs](https://docs.sourcegraph.com/amp/mcp) | -| **opencode** | user, project | `~/.config/opencode/opencode.json` (user) or `./opencode.json` (project) | [opencode MCP Docs](https://opencode.ai/docs/mcp-servers/) | - -## Manual Configuration - -If your client isn't directly supported by the installer, you can configure it manually. The MCP server uses the following configuration: - -**Server Name:** `trigger` - -**Command:** `npx` - -**Arguments:** `["trigger.dev@latest", "mcp"]` - -### Example JSON Configuration +To add these options to a manual config, append them to the `args` array: ```json { - "mcpServers": { - "trigger": { - "command": "npx", - "args": ["trigger.dev@latest", "mcp"] - } - } + "args": ["trigger.dev@latest", "mcp", "--dev-only", "--project-ref", "proj_abc123"] } ``` -### Example TOML Configuration (for Codex CLI) - -```toml -[mcp_servers.trigger] -command = "npx" -args = ["trigger.dev@latest", "mcp"] -``` - -### Additional Options - -You can add these optional arguments to customize the server behavior: - -- `--log-file ` - Log to a specific file -- `--api-url ` - Use a custom Trigger.dev API URL -- `--dev-only` - Restrict to dev environment only -- `--project-ref ` - Scope to a specific project - -## Environment-Specific Configuration - -### Development Only - -To restrict the MCP server to only work with the development environment: - -```json -{ - "mcpServers": { - "trigger": { - "command": "npx", - "args": ["trigger.dev@latest", "mcp", "--dev-only"] - } - } -} -``` - -### Project-Scoped - -To scope the server to a specific project: - -```json -{ - "mcpServers": { - "trigger": { - "command": "npx", - "args": ["trigger.dev@latest", "mcp", "--project-ref", "proj_your_project_ref"] - } - } -} -``` - -## Verification - -After installation, restart your MCP client and look for a server named "trigger". The server should connect automatically and provide access to all Trigger.dev tools. + ## Getting Started diff --git a/docs/mcp-tools.mdx b/docs/mcp-tools.mdx index 0163de97a..058a3671a 100644 --- a/docs/mcp-tools.mdx +++ b/docs/mcp-tools.mdx @@ -1,527 +1,133 @@ --- title: "MCP Tools" sidebarTitle: "Tools" -description: "Learn about the tools available in the Trigger.dev MCP Server" -tag: "new" +description: "Learn about how to use the tools available in the Trigger.dev MCP Server" --- -The Trigger.dev MCP Server provides a comprehensive set of tools that enable AI assistants to interact with your Trigger.dev projects. These tools cover everything from project management to task execution and monitoring. - ## Documentation and Search Tools ### search_docs -Search across the Trigger.dev documentation to find relevant information, code examples, API references, and guides. +Search the Trigger.dev documentation for guides, examples, and API references. - - The search query to find information in the Trigger.dev documentation - - -**Usage Examples:** - -- "How do I create a scheduled task?" -- "webhook examples" -- "deployment configuration" -- "error handling patterns" - - -```json Example Usage -{ - "tool": "search_docs", - "arguments": { - "query": "webhook examples" - } -} -``` - +**Example usage:** +- _"How do I create a scheduled task?"_ +- _"Show me webhook examples"_ +- _"What are the deployment options?"_ ## Project Management Tools -### list_projects - -List all projects in your Trigger.dev account. - -**No parameters required** - - - Array of project objects containing project details, IDs, and metadata - - - -```json Example Response -{ - "projects": [ - { - "id": "proj_abc123", - "name": "My App", - "slug": "my-app", - "organizationId": "org_xyz789" - } - ] -} -``` - - ### list_orgs List all organizations you have access to. -**No parameters required** +**Example usage:** +- _"What organizations do I have?"_ +- _"Show me my orgs"_ - - Array of organization objects containing organization details and metadata - +### list_projects + +List all projects in your Trigger.dev account. + +**Example usage:** +- _"What projects do I have?"_ +- _"List my Trigger.dev projects"_ ### create_project_in_org Create a new project in an organization. - - The organization to create the project in, can either be the organization slug or the ID. Use the - `list_orgs` tool to get a list of organizations and ask the user to select one. - - - - The name of the project to create - - - -```json Example Usage -{ - "tool": "create_project_in_org", - "arguments": { - "orgParam": "my-org", - "name": "New Project" - } -} -``` - +**Example usage:** +- _"Create a new project called 'my-app'"_ +- _"Set up a new Trigger.dev project"_ ### initialize_project Initialize Trigger.dev in your project with automatic setup and configuration. - - The organization to create the project in, can either be the organization slug or the ID. Use the - `list_orgs` tool to get a list of organizations and ask the user to select one. - - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The name of the project to create. If projectRef is not provided, we will use this name to create - a new project in the organization you select. - - - - The current working directory of the project - +**Example usage:** +- _"Set up Trigger.dev in this project"_ +- _"Add Trigger.dev to my app"_ ## Task Management Tools -### get_tasks +### get_current_worker -Get all tasks in a project. +Get the current worker for a project, including the worker version, SDK version, and registered tasks with their payload schemas. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). If not provided, we will try to find the config file in the - current working directory. - - - - The environment to get tasks for. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to get tasks for, only used for preview environments - - - -```json Example Usage -{ - "tool": "get_tasks", - "arguments": { - "projectRef": "proj_abc123", - "environment": "dev" - } -} -``` - +**Example usage:** +- _"What tasks are available?"_ +- _"Show me the tasks in dev"_ ### trigger_task -Trigger a task to run. +Trigger a task to run with a specific payload. You can add a delay, set tags, configure retries, choose a machine size, set a TTL, or use an idempotency key. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to trigger the task in. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to trigger the task in, only used for preview environments - - - - The ID/slug of the task to trigger. Use the `get_tasks` tool to get a list of tasks and ask the - user to select one if it's not clear which one to use. - - - - The payload to trigger the task with, must be a valid JSON string - - - - Additional options for the task run - - - The name of the queue to trigger the task in, by default will use the queue configured in the - task - - - The delay before the task run is executed - - - The idempotency key to use for the task run - - - The machine preset to use for the task run. Options: `micro`, `small-1x`, `small-2x`, - `medium-1x`, `medium-2x`, `large-1x`, `large-2x` - - - The maximum number of attempts to retry the task run - - - The maximum duration in seconds of the task run - - - Tags to add to the task run. Must be less than 128 characters and cannot have more than 5 - - - The time to live of the task run. If the run doesn't start executing within this time, it will - be automatically cancelled. - - - - - -```json Example Usage -{ - "tool": "trigger_task", - "arguments": { - "projectRef": "proj_abc123", - "taskId": "email-notification", - "payload": "{\"email\": \"user@example.com\", \"subject\": \"Hello World\"}", - "options": { - "tags": ["urgent"], - "maxAttempts": 3 - } - } -} -``` - +**Example usage:** +- _"Run the email-notification task"_ +- _"Trigger my-task with userId 123"_ +- _"Execute the sync task in production"_ ## Run Monitoring Tools ### get_run_details -Get the details of a specific task run. +Get detailed information about a specific task run, including logs and status. Enable debug mode to get the full trace with all logs and spans. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to get the run details from. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to get the run details from, only used for preview environments - - - - The ID of the run to get the details of, starts with `run_` - - - - Enable debug mode to get more detailed information about the run, including the entire trace (all logs and spans for the run and any child run). Set this to true if prompted to debug a run. - - -### cancel_run - -Cancel a running task. - - - The ID of the run to cancel, starts with `run_` - - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to cancel the run in. Options: `dev`, `staging`, `prod`, `preview` - - - - The branch to cancel the run in, only used for preview environments - - - -```json Example Usage -{ - "tool": "cancel_run", - "arguments": { - "runId": "run_abc123", - "projectRef": "proj_abc123" - } -} -``` - +**Example usage:** +- _"Show me details for run run_abc123"_ +- _"Why did this run fail?"_ ### list_runs -List all runs for a project with comprehensive filtering options. +List runs for a project. Filter by status, task, tags, version, machine size, or time period. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - +**Example usage:** +- _"Show me recent runs"_ +- _"List failed runs from the last 7 days"_ +- _"What runs are currently executing?"_ - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - +### wait_for_run_to_complete - - The environment to list runs from. Options: `dev`, `staging`, `prod`, `preview` - +Wait for a specific run to finish and return the result. - - The branch to list runs from, only used for preview environments - +**Example usage:** +- _"Wait for run run_abc123 to complete"_ - - The cursor to use for pagination, starts with `run_` - +### cancel_run - - The number of runs to list in a single page. Up to 100 - +Cancel a running or queued run. - - Filter for runs with this run status. Options: `PENDING_VERSION`, `QUEUED`, `DEQUEUED`, - `EXECUTING`, `WAITING`, `COMPLETED`, `CANCELED`, `FAILED`, `CRASHED`, `SYSTEM_FAILURE`, `DELAYED`, - `EXPIRED`, `TIMED_OUT` - - - - Filter for runs that match this task identifier - - - - Filter for runs that match this version, e.g. `20250808.3` - - - - Filter for runs that include this tag - - - - Filter for runs created after this ISO 8601 timestamp - - - - Filter for runs created before this ISO 8601 timestamp - - - - Filter for runs created in the last N time period. Examples: `7d`, `30d`, `365d` - - - - Filter for runs that match this machine preset. Options: `micro`, `small-1x`, `small-2x`, - `medium-1x`, `medium-2x`, `large-1x`, `large-2x` - - - -```json Example Usage -{ - "tool": "list_runs", - "arguments": { - "projectRef": "proj_abc123", - "status": "COMPLETED", - "limit": 10, - "period": "7d" - } -} -``` - +**Example usage:** +- _"Cancel run run_abc123"_ +- _"Stop that task"_ ## Deployment Tools ### deploy -Deploy a project to staging or production environments. +Deploy your project to staging or production. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - +**Example usage:** +- _"Deploy to production"_ +- _"Deploy to staging"_ - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - +### list_deploys - - The environment to deploy to. Options: `staging`, `prod`, `preview` - +List deployments for a project. Filter by status or time period. - - The branch to deploy, only used for preview environments - - - - Skip promoting the deployment to the current deployment for the environment - - - - Skip syncing environment variables when using the syncEnvVars extension - - - - Skip checking for @trigger.dev package updates - - - -```json Example Usage -{ - "tool": "deploy", - "arguments": { - "projectRef": "proj_abc123", - "environment": "prod", - "skipUpdateCheck": true - } -} -``` - - -### list_deployments - -List deployments for a project with comprehensive filtering options. - - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). - - - - The environment to list deployments for. Options: `staging`, `prod`, `preview` - - - - The branch to list deployments from, only used for preview environments - - - - The deployment ID to start the search from, to get the next page - - - - The number of deployments to return, defaults to 20 (max 100) - - - - Filter deployments that are in this status. Options: `PENDING`, `BUILDING`, `DEPLOYING`, `DEPLOYED`, `FAILED`, `CANCELED`, `TIMED_OUT` - - - - The date to start the search from, in ISO 8601 format - - - - The date to end the search, in ISO 8601 format - - - - The period to search within. Examples: `1d`, `7d`, `3h` - - - -```json Example Usage -{ - "tool": "list_deployments", - "arguments": { - "projectRef": "proj_abc123", - "environment": "prod", - "status": "DEPLOYED", - "limit": 10 - } -} -``` - +**Example usage:** +- _"Show me recent deployments"_ +- _"What's deployed to production?"_ ### list_preview_branches List all preview branches in the project. - - The trigger.dev project ref, starts with `proj_`. We will attempt to automatically detect the - project ref if running inside a directory that includes a trigger.config.ts file. - - - - The path to the trigger.config.ts file. Only used when the trigger.config.ts file is not at the - root dir (like in a monorepo setup). If not provided, we will try to find the config file in the - current working directory. - - - -```json Example Usage -{ - "tool": "list_preview_branches", - "arguments": { - "projectRef": "proj_abc123" - } -} -``` - +**Example usage:** +- _"What preview branches exist?"_ +- _"Show me preview deployments"_ - The deploy tool and list_preview_branches tool are not available when the MCP server is running with the `--dev-only` flag. + The deploy and list_preview_branches tools are not available when the MCP server is running with the `--dev-only` flag. diff --git a/docs/quick-start.mdx b/docs/quick-start.mdx index d6253bc26..375d225b6 100644 --- a/docs/quick-start.mdx +++ b/docs/quick-start.mdx @@ -8,29 +8,14 @@ import CliDevStep from '/snippets/step-cli-dev.mdx'; import CliRunTestStep from '/snippets/step-run-test.mdx'; import CliViewRunStep from '/snippets/step-view-run.mdx'; -In this guide we will: -1. Create a `trigger.config.ts` file and a `/trigger` directory with an example task. -2. Get you to run the task using the CLI. -3. Show you how to view the run logs for that task. + -You can either: - -- Use the [Trigger.dev Cloud](https://cloud.trigger.dev). -- Or [self-host](/open-source-self-hosting) the service. - - - - - -Once you've created an account, follow the steps in the app to: - -1. Complete your account details. -2. Create your first Organization and Project. +Sign up at [Trigger.dev Cloud](https://cloud.trigger.dev) (or [self-host](/open-source-self-hosting)). The onboarding flow will guide you through creating your first organization and project. @@ -43,11 +28,18 @@ Once you've created an account, follow the steps in the app to: ## Next steps - + + + Learn how to build Trigger.dev projects using AI coding assistants + Learn how to trigger tasks from your code. Tasks are the core of Trigger.dev. Learn what they are and how to write them. + + Guides and examples for triggering tasks from your code. + + diff --git a/docs/skills.mdx b/docs/skills.mdx new file mode 100644 index 000000000..eb4add479 --- /dev/null +++ b/docs/skills.mdx @@ -0,0 +1,83 @@ +--- +title: "Skills" +description: "Install Trigger.dev skills to teach any AI coding assistant best practices for writing tasks, agents, and workflows." +sidebarTitle: "Skills" +tag: "new" +--- + +## What are agent skills? + +Skills are portable instruction sets that teach AI coding assistants how to use Trigger.dev effectively. Unlike vendor-specific config files (`.cursor/rules`, `CLAUDE.md`), skills use an open standard that works across all major AI assistants. For example, Cursor users and Claude Code users can get the same knowledge from a single install. + +Skills are installed as directories containing a `SKILL.md` file. Each `SKILL.md` includes YAML frontmatter (name, description) and markdown instructions with patterns, examples, and best practices that AI assistants automatically discover and follow. + +## Installation + +When you run `npx skills add triggerdotdev/skills`, the CLI detects your installed AI tools and copies the appropriate files to each tool's expected location. For example, `.claude/skills/`, `.cursor/skills/`, `.github/skills/`, etc. + +```bash +npx skills add triggerdotdev/skills +``` + +`skills` is an open-source CLI by Vercel. Learn more at [skills.sh](https://skills.sh). + +The result: your AI assistant understands Trigger.dev's specific patterns for exports, schema validation, error handling, retries, and more. + + +## Available skills + +Install all skills at once, or pick the ones relevant to your current work: + +```bash +# Install all Trigger.dev skills +npx skills add triggerdotdev/skills + +# Or install individual skills +npx skills add triggerdotdev/skills --skill trigger-tasks +npx skills add triggerdotdev/skills --skill trigger-agents +npx skills add triggerdotdev/skills --skill trigger-config +npx skills add triggerdotdev/skills --skill trigger-realtime +npx skills add triggerdotdev/skills --skill trigger-setup +``` + +| Skill | Use for | Covers | +|-------|---------|--------| +| `trigger-setup` | First time setup, new projects | SDK install, `npx trigger init`, project structure | +| `trigger-tasks` | Writing background tasks, async workflows, scheduled tasks | Triggering, waits, queues, retries, cron, metadata | +| `trigger-agents` | LLM workflows, orchestration, multi-step AI agents | Prompt chaining, routing, parallelization, human-in-the-loop | +| `trigger-realtime` | Live updates, progress indicators, streaming | React hooks, progress bars, streaming AI responses | +| `trigger-config` | Project setup, build configuration | `trigger.config.ts`, extensions (Prisma, FFmpeg, Playwright) | + +Not sure which skill to install? Install `trigger-tasks`; it covers the most common patterns for writing Trigger.dev tasks. + + +## Supported AI assistants + +Skills work with any AI coding assistant that supports the [Agent Skills standard](https://agentskills.io), including: + +- [Claude Code](https://docs.anthropic.com/en/docs/claude-code) +- [Cursor](https://cursor.com) +- [Windsurf](https://codeium.com/windsurf) +- [GitHub Copilot](https://github.com/features/copilot) +- [Cline](https://github.com/cline/cline) +- [Codex CLI](https://github.com/openai/codex) +- [Gemini CLI](https://github.com/google-gemini/gemini-cli) +- [OpenCode](https://opencode.ai) +- [View all →](https://skills.sh) + +## Next steps + + + + Give your AI assistant direct access to Trigger.dev tools and APIs. + + + Learn the task patterns that skills teach your AI assistant. + + + Build durable AI workflows with prompt chaining and human-in-the-loop. + + + Browse the full Agent Skills ecosystem. + + \ No newline at end of file diff --git a/docs/snippets/step-cli-init.mdx b/docs/snippets/step-cli-init.mdx index 265f3d9c8..84bb340dc 100644 --- a/docs/snippets/step-cli-init.mdx +++ b/docs/snippets/step-cli-init.mdx @@ -20,12 +20,19 @@ yarn dlx trigger.dev@latest init + It will do a few things: -1. Log you into the CLI if you're not already logged in. -2. Create a `trigger.config.ts` file in the root of your project. -3. Ask where you'd like to create the `/trigger` directory. -4. Create the `/trigger` directory with an example task, `/trigger/example.[ts/js]`. + + Our [Trigger.dev MCP server](/mcp-introduction) gives your AI assistant direct access to Trigger.dev tools; search docs, trigger tasks, deploy projects, and monitor runs. We recommend installing it for the best developer experience. + + +1. Ask if you want to install the [Trigger.dev MCP server](/mcp-introduction) for your AI assistant. +2. Log you into the CLI if you're not already logged in. +3. Ask you to select your project. +4. Install the required SDK packages. +5. Ask where you'd like to create the `/trigger` directory and create it with an example task. +6. Create a `trigger.config.ts` file in the root of your project. Install the "Hello World" example task when prompted. We'll use this task to test the setup. From 0674d74bbbe17ff4ab5b5583e7b7cddf4314df23 Mon Sep 17 00:00:00 2001 From: DKP <8297864+D-K-P@users.noreply.github.com> Date: Thu, 29 Jan 2026 13:59:27 +0000 Subject: [PATCH 05/22] Added Trigger.dark theme to the docs (#2967) --- Open with Devin --- docs/docs.json | 5 +++++ docs/style.css | 30 +++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/docs.json b/docs/docs.json index 405f4eb0a..dcf637aea 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -446,6 +446,11 @@ "display": "simple" } }, + "styling": { + "codeblocks": { + "theme": "css-variables" + } + }, "appearance": { "default": "dark", "strict": true diff --git a/docs/style.css b/docs/style.css index b209952ad..94aea582d 100644 --- a/docs/style.css +++ b/docs/style.css @@ -1,3 +1,31 @@ button~.absolute.peer-hover\:opacity-100 { color: #000 -} \ No newline at end of file +} + +:root { + /* Code block colors - Trigger.dark theme */ + --mint-color-background: #121317; + --mint-color-text: #D4D4D4; + --mint-token-constant: #9B99FF; + --mint-token-string: #AFEC73; + --mint-token-comment: #5F6570; + --mint-token-keyword: #E888F8; + --mint-token-parameter: #CCCBFF; + --mint-token-function: #D9F07C; + --mint-token-string-expression: #AFEC73; + --mint-token-punctuation: #878C99; + --mint-token-link: #826DFF; + + /* Shiki css-variables fallbacks */ + --shiki-foreground: #D4D4D4; + --shiki-background: #121317; + --shiki-token-constant: #9B99FF; + --shiki-token-string: #AFEC73; + --shiki-token-comment: #5F6570; + --shiki-token-keyword: #E888F8; + --shiki-token-parameter: #CCCBFF; + --shiki-token-function: #D9F07C; + --shiki-token-string-expression: #AFEC73; + --shiki-token-punctuation: #878C99; + --shiki-token-link: #826DFF; +} From 72c357125b3035ad515212f3f6456533ff4323e2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 29 Jan 2026 15:30:56 +0000 Subject: [PATCH 06/22] Query enabled via feature flag (#2968) - Renamed FeatureFlag functions to be singular where it makes sense. - Added function to handle multiple feature flags - canAccessQuery now checks the global feature flag and environment variable as well --- Open with Devin --- apps/webapp/app/env.server.ts | 3 + .../OrganizationsPresenter.server.ts | 16 +++-- .../presenters/v3/RegionsPresenter.server.ts | 4 +- .../route.tsx | 60 ++++------------ .../runsRepository/runsRepository.server.ts | 4 +- apps/webapp/app/v3/canAccessQuery.server.ts | 47 +++++++++++++ .../app/v3/eventRepository/index.server.ts | 24 +++---- apps/webapp/app/v3/featureFlags.server.ts | 69 ++++++++++++++++--- .../worker/workerGroupService.server.ts | 8 +-- 9 files changed, 154 insertions(+), 81 deletions(-) create mode 100644 apps/webapp/app/v3/canAccessQuery.server.ts diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 98c04b6f9..dcbcac079 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1190,6 +1190,9 @@ const EnvironmentSchema = z CLICKHOUSE_LOGS_DETAIL_MAX_THREADS: z.coerce.number().int().default(2), CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME: z.coerce.number().int().default(60), + // Query feature flag + QUERY_FEATURE_ENABLED: z.string().default("1"), + // Query page ClickHouse limits (for TSQL queries) QUERY_CLICKHOUSE_MAX_EXECUTION_TIME: z.coerce.number().int().default(10), QUERY_CLICKHOUSE_MAX_MEMORY_USAGE: z.coerce.number().int().default(1_073_741_824), // 1GB in bytes diff --git a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts index b52e69db9..c229a0d7f 100644 --- a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts +++ b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts @@ -1,4 +1,4 @@ -import { RuntimeEnvironment, type PrismaClient } from "@trigger.dev/database"; +import type { RuntimeEnvironment, PrismaClient } from "@trigger.dev/database"; import { redirect } from "remix-typedjson"; import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; @@ -10,7 +10,7 @@ import { } from "./SelectBestEnvironmentPresenter.server"; import { sortEnvironments } from "~/utils/environmentSort"; import { defaultAvatar, parseAvatar } from "~/components/primitives/Avatar"; -import { validatePartialFeatureFlags } from "~/v3/featureFlags.server"; +import { flags, validatePartialFeatureFlags } from "~/v3/featureFlags.server"; export class OrganizationsPresenter { #prismaClient: PrismaClient; @@ -153,18 +153,24 @@ export class OrganizationsPresenter { }, }); + // Get global feature flags (no overrides or defaults) + const globalFlags = await flags(); + return orgs.map((org) => { - const flagsResult = org.featureFlags + const orgFlagsResult = org.featureFlags ? validatePartialFeatureFlags(org.featureFlags as Record) : ({ success: false } as const); - const flags = flagsResult.success ? flagsResult.data : {}; + const orgFlags = orgFlagsResult.success ? orgFlagsResult.data : {}; + + // Combine global flags with org flags (org flags win) + const combinedFlags = { ...globalFlags, ...orgFlags }; return { id: org.id, slug: org.slug, title: org.title, avatar: parseAvatar(org.avatar, defaultAvatar), - featureFlags: flags, + featureFlags: combinedFlags, projects: org.projects.map((project) => ({ id: project.id, slug: project.slug, diff --git a/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts b/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts index c304597bb..7a35fb6fb 100644 --- a/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RegionsPresenter.server.ts @@ -1,6 +1,6 @@ import { type Project } from "~/models/project.server"; import { type User } from "~/models/user.server"; -import { FEATURE_FLAG, makeFlags } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; import { BasePresenter } from "./basePresenter.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; @@ -48,7 +48,7 @@ export class RegionsPresenter extends BasePresenter { throw new Error("Project not found"); } - const getFlag = makeFlags(this._replica); + const getFlag = makeFlag(this._replica); const defaultWorkerInstanceGroupId = await getFlag({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, }); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx index 996149a46..72020d8ad 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx @@ -75,7 +75,7 @@ import { executeQuery, type QueryScope } from "~/services/queryService.server"; import { requireUser } from "~/services/session.server"; import { downloadFile, rowsToCSV, rowsToJSON } from "~/utils/dataExport"; import { EnvironmentParamSchema, organizationBillingPath } from "~/utils/pathBuilder"; -import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server"; +import { canAccessQuery } from "~/v3/canAccessQuery.server"; import { querySchemas } from "~/v3/querySchemas"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; import { QueryHelpSidebar } from "./QueryHelpSidebar"; @@ -91,40 +91,6 @@ function toISOString(value: Date | string): string { return value.toISOString(); } -async function hasQueryAccess( - userId: string, - isAdmin: boolean, - isImpersonating: boolean, - organizationSlug: string -): Promise { - if (isAdmin || isImpersonating) { - return true; - } - - // Check organization feature flags - const organization = await prisma.organization.findFirst({ - where: { - slug: organizationSlug, - members: { some: { userId } }, - }, - select: { - featureFlags: true, - }, - }); - - if (!organization?.featureFlags) { - return false; - } - - const flags = organization.featureFlags as Record; - const hasQueryAccessResult = validateFeatureFlagValue( - FEATURE_FLAG.hasQueryAccess, - flags.hasQueryAccess - ); - - return hasQueryAccessResult.success && hasQueryAccessResult.data === true; -} - const scopeOptions = [ { value: "environment", label: "Environment" }, { value: "project", label: "Project" }, @@ -135,12 +101,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); - const canAccess = await hasQueryAccess( - user.id, - user.admin, - user.isImpersonating, - organizationSlug - ); + const canAccess = await canAccessQuery({ + userId: user.id, + isAdmin: user.admin, + isImpersonating: user.isImpersonating, + organizationSlug, + }); if (!canAccess) { throw redirect("/"); } @@ -200,12 +166,12 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const user = await requireUser(request); const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); - const canAccess = await hasQueryAccess( - user.id, - user.admin, - user.isImpersonating, - organizationSlug - ); + const canAccess = await canAccessQuery({ + userId: user.id, + isAdmin: user.admin, + isImpersonating: user.isImpersonating, + organizationSlug, + }); if (!canAccess) { return typedjson( { diff --git a/apps/webapp/app/services/runsRepository/runsRepository.server.ts b/apps/webapp/app/services/runsRepository/runsRepository.server.ts index 895c8b5fe..90b58b8a9 100644 --- a/apps/webapp/app/services/runsRepository/runsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/runsRepository.server.ts @@ -8,7 +8,7 @@ import parseDuration from "parse-duration"; import { z } from "zod"; import { timeFilters } from "~/components/runs/v3/SharedFilters"; import { type PrismaClient, type PrismaClientOrTransaction } from "~/db.server"; -import { FEATURE_FLAG, makeFlags } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; import { startActiveSpan } from "~/v3/tracer.server"; import { logger } from "../logger.server"; import { ClickHouseRunsRepository } from "./clickhouseRunsRepository.server"; @@ -163,7 +163,7 @@ export class RunsRepository implements IRunsRepository { async #getRepository(): Promise { return startActiveSpan("runsRepository.getRepository", async (span) => { - const getFlag = makeFlags(this.options.prisma); + const getFlag = makeFlag(this.options.prisma); const runsListRepository = await getFlag({ key: FEATURE_FLAG.runsListRepository, defaultValue: this.defaultRepository, diff --git a/apps/webapp/app/v3/canAccessQuery.server.ts b/apps/webapp/app/v3/canAccessQuery.server.ts new file mode 100644 index 000000000..87a248725 --- /dev/null +++ b/apps/webapp/app/v3/canAccessQuery.server.ts @@ -0,0 +1,47 @@ +import { prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { FEATURE_FLAG, makeFlag } from "~/v3/featureFlags.server"; + +export async function canAccessQuery(options: { + userId: string; + isAdmin: boolean; + isImpersonating: boolean; + organizationSlug: string; +}): Promise { + const { userId, isAdmin, isImpersonating, organizationSlug } = options; + + // 1. If it's on then we have access + const globallyEnabled = env.QUERY_FEATURE_ENABLED === "1"; + if (globallyEnabled) { + return true; + } + + // 2. Admins always have access + if (isAdmin || isImpersonating) { + return true; + } + + // 3. Check if org/global feature flag is on + const org = await prisma.organization.findFirst({ + where: { + slug: organizationSlug, + members: { some: { userId } }, + }, + select: { + featureFlags: true, + }, + }); + + const flag = makeFlag(); + const flagResult = await flag({ + key: FEATURE_FLAG.hasQueryAccess, + defaultValue: false, + overrides: (org?.featureFlags as Record) ?? {}, + }); + if (flagResult) { + return true; + } + + // 4. Not enabled anywhere + return false; +} diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index cb211e2b0..2f457e235 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -5,9 +5,9 @@ import { clickhouseEventRepositoryV2, } from "./clickhouseEventRepositoryInstance.server"; import { IEventRepository, TraceEventOptions } from "./eventRepository.types"; -import { prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; -import { FEATURE_FLAG, flags } from "../featureFlags.server"; +import { FEATURE_FLAG, flag } from "../featureFlags.server"; import { getTaskEventStore } from "../taskEventStore.server"; export function resolveEventRepositoryForStore(store: string | undefined): IEventRepository { @@ -24,13 +24,13 @@ export function resolveEventRepositoryForStore(store: string | undefined): IEven return eventRepository; } - export const EVENT_STORE_TYPES = { - POSTGRES: "postgres", - CLICKHOUSE: "clickhouse", - CLICKHOUSE_V2: "clickhouse_v2", - } as const; +export const EVENT_STORE_TYPES = { + POSTGRES: "postgres", + CLICKHOUSE: "clickhouse", + CLICKHOUSE_V2: "clickhouse_v2", +} as const; -export type EventStoreType = typeof EVENT_STORE_TYPES[keyof typeof EVENT_STORE_TYPES]; +export type EventStoreType = (typeof EVENT_STORE_TYPES)[keyof typeof EVENT_STORE_TYPES]; export async function getConfiguredEventRepository( organizationId: string @@ -122,21 +122,21 @@ export async function getV3EventRepository( async function resolveTaskEventRepositoryFlag( featureFlags: Record | undefined ): Promise<"clickhouse" | "clickhouse_v2" | "postgres"> { - const flag = await flags({ + const flagResult = await flag({ key: FEATURE_FLAG.taskEventRepository, defaultValue: env.EVENT_REPOSITORY_DEFAULT_STORE, overrides: featureFlags, }); - if (flag === "clickhouse_v2") { + if (flagResult === "clickhouse_v2") { return "clickhouse_v2"; } - if (flag === "clickhouse") { + if (flagResult === "clickhouse") { return "clickhouse"; } - return flag; + return flagResult; } export async function recordRunDebugLog( diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index 605c11def..e889b2123 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -25,14 +25,14 @@ export type FlagsOptions = { overrides?: Record; }; -export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) { - function flags( +export function makeFlag(_prisma: PrismaClientOrTransaction = prisma) { + function flag( opts: FlagsOptions & { defaultValue: z.infer<(typeof FeatureFlagCatalog)[T]> } ): Promise>; - function flags( + function flag( opts: FlagsOptions ): Promise | undefined>; - async function flags( + async function flag( opts: FlagsOptions ): Promise | undefined> { const value = await _prisma.featureFlag.findUnique({ @@ -60,11 +60,11 @@ export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) { return parsed.data; } - return flags; + return flag; } -export function makeSetFlags(_prisma: PrismaClientOrTransaction = prisma) { - return async function setFlags( +export function makeSetFlag(_prisma: PrismaClientOrTransaction = prisma) { + return async function setFlag( opts: FlagsOptions & { value: z.infer<(typeof FeatureFlagCatalog)[T]> } ): Promise { await _prisma.featureFlag.upsert({ @@ -82,8 +82,59 @@ export function makeSetFlags(_prisma: PrismaClientOrTransaction = prisma) { }; } +export type AllFlagsOptions = { + defaultValues?: Partial; + overrides?: Record; +}; + +export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) { + return async function flags(options?: AllFlagsOptions): Promise> { + const rows = await _prisma.featureFlag.findMany(); + + // Build a map of key -> value from database + const dbValues = new Map(); + for (const row of rows) { + dbValues.set(row.key, row.value); + } + + const result: Partial = {}; + + // Process each flag in the catalog + for (const key of Object.keys(FeatureFlagCatalog) as FeatureFlagKey[]) { + const schema = FeatureFlagCatalog[key]; + + // Priority: overrides > database > defaultValues + if (options?.overrides?.[key] !== undefined) { + const parsed = schema.safeParse(options.overrides[key]); + if (parsed.success) { + (result as any)[key] = parsed.data; + continue; + } + } + + if (dbValues.has(key)) { + const parsed = schema.safeParse(dbValues.get(key)); + if (parsed.success) { + (result as any)[key] = parsed.data; + continue; + } + } + + if (options?.defaultValues?.[key] !== undefined) { + const parsed = schema.safeParse(options.defaultValues[key]); + if (parsed.success) { + (result as any)[key] = parsed.data; + } + } + } + + return result; + }; +} + +export const flag = makeFlag(); export const flags = makeFlags(); -export const setFlags = makeSetFlags(); +export const setFlag = makeSetFlag(); // Create a Zod schema from the existing catalog export const FeatureFlagCatalogSchema = z.object(FeatureFlagCatalog); @@ -112,7 +163,7 @@ export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma return async function setMultipleFlags( flags: Partial> ): Promise<{ key: string; value: any }[]> { - const setFlag = makeSetFlags(_prisma); + const setFlag = makeSetFlag(_prisma); const updatedFlags: { key: string; value: any }[] = []; for (const [key, value] of Object.entries(flags)) { diff --git a/apps/webapp/app/v3/services/worker/workerGroupService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts index f05c8783e..936f8bbd4 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupService.server.ts @@ -2,7 +2,7 @@ import { WorkerInstanceGroup, WorkerInstanceGroupType } from "@trigger.dev/datab import { WithRunEngine } from "../baseService.server"; import { WorkerGroupTokenService } from "./workerGroupTokenService.server"; import { logger } from "~/services/logger.server"; -import { FEATURE_FLAG, makeFlags, makeSetFlags } from "~/v3/featureFlags.server"; +import { FEATURE_FLAG, makeFlag, makeSetFlag } from "~/v3/featureFlags.server"; export class WorkerGroupService extends WithRunEngine { private readonly defaultNamePrefix = "worker_group"; @@ -47,14 +47,14 @@ export class WorkerGroupService extends WithRunEngine { }, }); - const getFlag = makeFlags(this._prisma); + const getFlag = makeFlag(this._prisma); const defaultWorkerInstanceGroupId = await getFlag({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, }); // If there's no global default yet we should set it to the new worker group if (!defaultWorkerInstanceGroupId) { - const setFlag = makeSetFlags(this._prisma); + const setFlag = makeSetFlag(this._prisma); await setFlag({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, value: workerGroup.id, @@ -166,7 +166,7 @@ export class WorkerGroupService extends WithRunEngine { } async getGlobalDefaultWorkerGroup() { - const flags = makeFlags(this._prisma); + const flags = makeFlag(this._prisma); const defaultWorkerInstanceGroupId = await flags({ key: FEATURE_FLAG.defaultWorkerInstanceGroupId, From 5e049cde3a73e4a95ac648d60e2f8d6f3f6a87e3 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Thu, 29 Jan 2026 19:03:54 +0000 Subject: [PATCH 07/22] fix(run-engine): avoid NAPI string overflow in getExecutionSnapshotsSince by only fetching waitpoints for latest snapshot (#2972) --- .../engine/systems/executionSnapshotSystem.ts | 116 ++- .../engine/tests/getSnapshotsSince.test.ts | 670 ++++++++++++++++++ .../tests/helpers/executionStateMachine.ts | 257 +++++++ .../tests/helpers/snapshotTestHelpers.ts | 322 +++++++++ 4 files changed, 1354 insertions(+), 11 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts create mode 100644 internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts index b6f31bcff..a224e5a86 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts @@ -8,10 +8,14 @@ import { TaskRunExecutionSnapshot, TaskRunExecutionStatus, TaskRunStatus, + Waitpoint, } from "@trigger.dev/database"; import { HeartbeatTimeouts } from "../types.js"; import { SystemResources } from "./systems.js"; +/** Chunk size for fetching waitpoints to avoid NAPI string conversion limits */ +const WAITPOINT_CHUNK_SIZE = 100; + export type ExecutionSnapshotSystemOptions = { resources: SystemResources; heartbeatTimeouts: HeartbeatTimeouts; @@ -31,19 +35,41 @@ type ExecutionSnapshotWithCheckAndWaitpoints = Prisma.TaskRunExecutionSnapshotGe }; }>; +type ExecutionSnapshotWithCheckpoint = Prisma.TaskRunExecutionSnapshotGetPayload<{ + include: { + checkpoint: true; + }; +}>; + function enhanceExecutionSnapshot( snapshot: ExecutionSnapshotWithCheckAndWaitpoints +): EnhancedExecutionSnapshot { + return enhanceExecutionSnapshotWithWaitpoints( + snapshot, + snapshot.completedWaitpoints, + snapshot.completedWaitpointOrder + ); +} + +/** + * Transforms a snapshot (with checkpoint but without waitpoints) into an EnhancedExecutionSnapshot + * by combining it with pre-fetched waitpoints. + */ +function enhanceExecutionSnapshotWithWaitpoints( + snapshot: ExecutionSnapshotWithCheckpoint, + waitpoints: Waitpoint[], + completedWaitpointOrder: string[] ): EnhancedExecutionSnapshot { return { ...snapshot, friendlyId: SnapshotId.toFriendlyId(snapshot.id), runFriendlyId: RunId.toFriendlyId(snapshot.runId), - completedWaitpoints: snapshot.completedWaitpoints.flatMap((w) => { - //get all indexes of the waitpoint in the completedWaitpointOrder - //we do this because the same run can be in a batch multiple times (i.e. same idempotencyKey) + completedWaitpoints: waitpoints.flatMap((w) => { + // Get all indexes of the waitpoint in the completedWaitpointOrder + // We do this because the same run can be in a batch multiple times (i.e. same idempotencyKey) let indexes: (number | undefined)[] = []; - for (let i = 0; i < snapshot.completedWaitpointOrder.length; i++) { - if (snapshot.completedWaitpointOrder[i] === w.id) { + for (let i = 0; i < completedWaitpointOrder.length; i++) { + if (completedWaitpointOrder[i] === w.id) { indexes.push(i); } } @@ -60,9 +86,7 @@ function enhanceExecutionSnapshot( type: w.type, completedAt: w.completedAt ?? new Date(), idempotencyKey: - w.userProvidedIdempotencyKey && !w.inactiveIdempotencyKey - ? w.idempotencyKey - : undefined, + w.userProvidedIdempotencyKey && !w.inactiveIdempotencyKey ? w.idempotencyKey : undefined, completedByTaskRun: w.completedByTaskRunId ? { id: w.completedByTaskRunId, @@ -91,6 +115,42 @@ function enhanceExecutionSnapshot( }; } +/** + * Gets the waitpoint IDs linked to a snapshot via the _completedWaitpoints join table. + * Uses raw SQL to avoid fetching full waitpoint data. + */ +async function getSnapshotWaitpointIds( + prisma: PrismaClientOrTransaction, + snapshotId: string +): Promise { + const result = await prisma.$queryRaw<{ B: string }[]>` + SELECT "B" FROM "_completedWaitpoints" WHERE "A" = ${snapshotId} + `; + return result.map((r) => r.B); +} + +/** + * Fetches waitpoints in chunks to avoid NAPI string conversion limits. + * This is necessary because waitpoints can have large outputs (100KB+), + * and fetching many at once can exceed Node.js string limits. + */ +async function fetchWaitpointsInChunks( + prisma: PrismaClientOrTransaction, + waitpointIds: string[] +): Promise { + if (waitpointIds.length === 0) return []; + + const allWaitpoints: Waitpoint[] = []; + for (let i = 0; i < waitpointIds.length; i += WAITPOINT_CHUNK_SIZE) { + const chunk = waitpointIds.slice(i, i + WAITPOINT_CHUNK_SIZE); + const waitpoints = await prisma.waitpoint.findMany({ + where: { id: { in: chunk } }, + }); + allWaitpoints.push(...waitpoints); + } + return allWaitpoints; +} + /* Gets the most recent valid snapshot for a run */ export async function getLatestExecutionSnapshot( prisma: PrismaClientOrTransaction, @@ -191,12 +251,27 @@ export function executionDataFromSnapshot(snapshot: EnhancedExecutionSnapshot): }; } +/** + * Gets execution snapshots created after the specified snapshot. + * + * IMPORTANT: This function is optimized to avoid N×M data explosion when runs have many + * completed waitpoints. Due to the many-to-many relation, once waitpoints complete, + * all subsequent snapshots have the same waitpoints linked. For a run with 24 snapshots + * and 236 waitpoints with 100KB outputs each, fetching all waitpoints for all snapshots + * would result in ~570MB of data, causing "Failed to convert rust String into napi string" errors. + * + * Solution: Only the LATEST snapshot's waitpoints are fetched and included. The runner's + * SnapshotManager only processes completedWaitpoints from the latest snapshot anyway - + * intermediate snapshots' waitpoints are ignored. This reduces data from N×M to just M. + * + * Waitpoints are fetched in chunks (100 at a time) to handle batches up to 1000 items. + */ export async function getExecutionSnapshotsSince( prisma: PrismaClientOrTransaction, runId: string, sinceSnapshotId: string ): Promise { - // Find the createdAt of the sinceSnapshotId + // Step 1: Find the createdAt of the sinceSnapshotId const sinceSnapshot = await prisma.taskRunExecutionSnapshot.findFirst({ where: { id: sinceSnapshotId }, select: { createdAt: true }, @@ -206,6 +281,7 @@ export async function getExecutionSnapshotsSince( throw new Error(`No execution snapshot found for id ${sinceSnapshotId}`); } + // Step 2: Fetch snapshots WITHOUT waitpoints to avoid N×M data explosion const snapshots = await prisma.taskRunExecutionSnapshot.findMany({ where: { runId, @@ -213,14 +289,32 @@ export async function getExecutionSnapshotsSince( createdAt: { gt: sinceSnapshot.createdAt }, }, include: { - completedWaitpoints: true, checkpoint: true, + // DO NOT include completedWaitpoints here - this causes the N×M explosion }, orderBy: { createdAt: "desc" }, take: 50, }); - return snapshots.reverse().map(enhanceExecutionSnapshot); + if (snapshots.length === 0) return []; + + // Step 3: Get waitpoint IDs for the LATEST snapshot only (first in desc order) + const latestSnapshot = snapshots[0]; + const waitpointIds = await getSnapshotWaitpointIds(prisma, latestSnapshot.id); + + // Step 4: Fetch waitpoints in chunks to avoid NAPI string conversion limits + const waitpoints = await fetchWaitpointsInChunks(prisma, waitpointIds); + + // Step 5: Build enhanced snapshots - only latest gets waitpoints, others get empty arrays + // The runner only uses completedWaitpoints from the latest snapshot anyway + return snapshots.reverse().map((snapshot) => { + const isLatest = snapshot.id === latestSnapshot.id; + return enhanceExecutionSnapshotWithWaitpoints( + snapshot, + isLatest ? waitpoints : [], + latestSnapshot.completedWaitpointOrder + ); + }); } export class ExecutionSnapshotSystem { diff --git a/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts b/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts new file mode 100644 index 000000000..4352e7268 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts @@ -0,0 +1,670 @@ +import { containerTest } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { expect, describe } from "vitest"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; +import { setTimeout } from "node:timers/promises"; +import { + generateTestScenarios, + type SnapshotTestScenario, +} from "./helpers/executionStateMachine.js"; +import { + createWaitpointsWithOutput, + setupTestScenario, + generateLargeOutput, +} from "./helpers/snapshotTestHelpers.js"; +import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; + +vi.setConfig({ testTimeout: 120_000 }); + +describe("RunEngine getSnapshotsSince", () => { + containerTest( + "returns empty array when querying from latest snapshot", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_empty", + spanId: "s_empty", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_empty", + workerQueue: "main", + }); + + // Get all snapshots + const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({ + where: { runId: run.id, isValid: true }, + orderBy: { createdAt: "asc" }, + }); + + expect(allSnapshots.length).toBeGreaterThan(0); + + // Query from the last snapshot + const lastSnapshot = allSnapshots[allSnapshots.length - 1]; + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: lastSnapshot.id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "returns snapshots after the specified one with waitpoints only on latest", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_wp", + spanId: "s_wp", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_wp", + workerQueue: "main", + }); + + // Start attempt + await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + // Create and block with a waitpoint + const { waitpoint } = await engine.createDateTimeWaitpoint({ + projectId: authenticatedEnvironment.project.id, + environmentId: authenticatedEnvironment.id, + completedAfter: new Date(Date.now() + 50), + }); + + await engine.blockRunWithWaitpoint({ + runId: run.id, + waitpoints: [waitpoint.id], + projectId: authenticatedEnvironment.project.id, + organizationId: authenticatedEnvironment.organization.id, + }); + + // Wait for waitpoint completion + await setTimeout(200); + + // Get all snapshots + const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({ + where: { runId: run.id, isValid: true }, + orderBy: { createdAt: "asc" }, + }); + + expect(allSnapshots.length).toBeGreaterThanOrEqual(3); + + // Query from the first snapshot + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: allSnapshots[0].id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBeGreaterThanOrEqual(2); + + // The latest snapshot should have completedWaitpoints + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBeGreaterThan(0); + + // Earlier snapshots should have empty waitpoints (optimization) + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "handles multiple waitpoints correctly - only latest has them", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_mwp", + spanId: "s_mwp", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_mwp", + workerQueue: "main", + }); + + await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + // Create multiple waitpoints + const waitpointCount = 5; + const waitpointPromises = Array.from({ length: waitpointCount }).map(() => + engine.createManualWaitpoint({ + environmentId: authenticatedEnvironment.id, + projectId: authenticatedEnvironment.projectId, + }) + ); + const waitpoints = await Promise.all(waitpointPromises); + + // Block the run with all waitpoints + for (const { waitpoint } of waitpoints) { + await engine.blockRunWithWaitpoint({ + runId: run.id, + waitpoints: waitpoint.id, + projectId: authenticatedEnvironment.projectId, + organizationId: authenticatedEnvironment.organizationId, + }); + } + + // Complete all waitpoints + for (const { waitpoint } of waitpoints) { + await engine.completeWaitpoint({ id: waitpoint.id }); + } + + await setTimeout(500); + + // Get all snapshots + const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({ + where: { runId: run.id, isValid: true }, + orderBy: { createdAt: "asc" }, + }); + + // Query from early in the sequence + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: allSnapshots[0].id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBeGreaterThan(0); + + // Only the latest should have waitpoints + const latest = result![result!.length - 1]; + + // Earlier snapshots must have empty completedWaitpoints + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest("returns null for invalid snapshot ID", async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + const runFriendlyId = generateFriendlyId("run"); + const run = await engine.trigger( + { + number: 1, + friendlyId: runFriendlyId, + environment: authenticatedEnvironment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t_invalid", + spanId: "s_invalid", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + }, + prisma + ); + + // Query with invalid snapshot ID + const result = await engine.getSnapshotsSince({ + runId: run.id, + snapshotId: "invalid-snapshot-id", + }); + + // Should return null (caught by getSnapshotsSince error handler) + expect(result).toBeNull(); + } finally { + await engine.quit(); + } + }); + + // Direct database tests for the core function + containerTest( + "direct test: large waitpoint scenario - 100 waitpoints with 10KB outputs", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + // Create scenario directly in database + const scenario = await setupTestScenario(prisma, authenticatedEnvironment, { + totalWaitpoints: 100, + outputSizeKB: 10, + snapshotConfigs: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING", completedWaitpointCount: 100 }, + { status: "FINISHED", completedWaitpointCount: 100 }, + ], + }); + + // Query from early snapshot + const result = await engine.getSnapshotsSince({ + runId: scenario.run.id, + snapshotId: scenario.snapshots[2].id, // After PENDING_EXECUTING + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBe(6); // EXECUTING through FINISHED + + // Latest should have all 100 waitpoints + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBe(100); + + // Verify all earlier snapshots have empty waitpoints + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "direct test: zombie run scenario - 236 waitpoints with 100KB outputs, 24 snapshots", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + // This scenario matches the exact conditions that caused the NAPI error + // 24 snapshots × 236 waitpoints × 100KB = ~570MB if not optimized + const scenario = await setupTestScenario(prisma, authenticatedEnvironment, { + totalWaitpoints: 236, + outputSizeKB: 100, + snapshotConfigs: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 200 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + ], + }); + + expect(scenario.snapshots.length).toBe(24); + expect(scenario.waitpoints.length).toBe(236); + + // Query from the 6th snapshot (after waitpoints start completing) + const queryFromIndex = 5; + const result = await engine.getSnapshotsSince({ + runId: scenario.run.id, + snapshotId: scenario.snapshots[queryFromIndex].id, + }); + + expect(result).not.toBeNull(); + // Should return snapshots after index 5, which is 24 - 6 = 18 snapshots + expect(result!.length).toBe(24 - queryFromIndex - 1); + + // Latest should have all 236 waitpoints + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBe(236); + + // All other snapshots should have 0 waitpoints (optimization) + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + + // Verify the outputs are present and correct size + for (const wp of latest.completedWaitpoints) { + expect(wp.output).toBeDefined(); + // ~100KB output as JSON string + expect(typeof wp.output).toBe("string"); + } + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "direct test: verifies chunked fetching works with 500+ waitpoints", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + // 500 waitpoints requires 5 chunks (100 per chunk) + const scenario = await setupTestScenario(prisma, authenticatedEnvironment, { + totalWaitpoints: 500, + outputSizeKB: 10, // Smaller outputs for faster test + snapshotConfigs: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 500 }, + { status: "EXECUTING", completedWaitpointCount: 500 }, + ], + }); + + const result = await engine.getSnapshotsSince({ + runId: scenario.run.id, + snapshotId: scenario.snapshots[0].id, + }); + + expect(result).not.toBeNull(); + expect(result!.length).toBe(4); + + const latest = result![result!.length - 1]; + expect(latest.completedWaitpoints.length).toBe(500); + + // All other snapshots should be empty + for (let i = 0; i < result!.length - 1; i++) { + expect(result![i].completedWaitpoints.length).toBe(0); + } + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts b/internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts new file mode 100644 index 000000000..4dd92cdbd --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/executionStateMachine.ts @@ -0,0 +1,257 @@ +import { TaskRunExecutionStatus } from "@trigger.dev/database"; + +/** + * Defines valid execution status transitions for the Run Engine 2.0. + * This is a model of the state machine that governs run execution. + */ +export const EXECUTION_STATUS_TRANSITIONS: Record< + TaskRunExecutionStatus, + TaskRunExecutionStatus[] +> = { + RUN_CREATED: ["QUEUED", "DELAYED"], + DELAYED: ["QUEUED"], + QUEUED: ["PENDING_EXECUTING", "QUEUED_EXECUTING"], + QUEUED_EXECUTING: ["PENDING_EXECUTING", "QUEUED"], + PENDING_EXECUTING: ["EXECUTING", "PENDING_CANCEL", "FINISHED", "QUEUED"], + EXECUTING: ["EXECUTING_WITH_WAITPOINTS", "FINISHED", "PENDING_CANCEL", "QUEUED"], + EXECUTING_WITH_WAITPOINTS: ["EXECUTING", "SUSPENDED", "FINISHED", "PENDING_CANCEL"], + SUSPENDED: ["QUEUED", "PENDING_CANCEL", "FINISHED"], + PENDING_CANCEL: ["FINISHED"], + FINISHED: ["QUEUED"], // Retry case +}; + +/** + * Validates if a transition from one status to another is valid. + */ +export function isValidTransition( + from: TaskRunExecutionStatus, + to: TaskRunExecutionStatus +): boolean { + return EXECUTION_STATUS_TRANSITIONS[from]?.includes(to) ?? false; +} + +/** + * Configuration for a snapshot in a test scenario. + */ +export interface SnapshotConfig { + /** The execution status for this snapshot */ + status: TaskRunExecutionStatus; + /** Number of waitpoints completed at this snapshot (cumulative) */ + completedWaitpointCount: number; + /** Whether this snapshot has a checkpoint */ + hasCheckpoint?: boolean; + /** Description for the snapshot */ + description?: string; +} + +/** + * A test scenario for getSnapshotsSince testing. + */ +export interface SnapshotTestScenario { + /** Unique name for the scenario */ + name: string; + /** Description of what this scenario tests */ + description: string; + /** Total number of waitpoints to create */ + totalWaitpoints: number; + /** Size of each waitpoint's output in KB */ + outputSizeKB: number; + /** Configuration for each snapshot to create */ + snapshots: SnapshotConfig[]; + /** Which snapshot index to query "since" (0-based) */ + queryFromIndex: number; + /** Expected number of waitpoints on the latest snapshot returned */ + expectedWaitpointsOnLatest: number; +} + +/** + * Generates test scenarios for comprehensive getSnapshotsSince testing. + * These scenarios cover various edge cases and stress tests. + */ +export function generateTestScenarios(): SnapshotTestScenario[] { + return [ + { + name: "simple_no_waitpoints", + description: "Basic run without any waitpoints", + totalWaitpoints: 0, + outputSizeKB: 0, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "FINISHED", completedWaitpointCount: 0 }, + ], + queryFromIndex: 0, + expectedWaitpointsOnLatest: 0, + }, + { + name: "single_small_waitpoint", + description: "Single waitpoint with small output", + totalWaitpoints: 1, + outputSizeKB: 1, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 1 }, + ], + queryFromIndex: 2, + expectedWaitpointsOnLatest: 1, + }, + { + name: "batch_100_medium", + description: "Medium batch with 100 waitpoints and medium outputs", + totalWaitpoints: 100, + outputSizeKB: 10, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "SUSPENDED", completedWaitpointCount: 100, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 100 }, + { status: "EXECUTING", completedWaitpointCount: 100 }, + { status: "FINISHED", completedWaitpointCount: 100 }, + ], + queryFromIndex: 3, + expectedWaitpointsOnLatest: 100, + }, + { + name: "batch_236_large_zombie_scenario", + description: + "Matches the zombie run scenario: 24 snapshots, 236 waitpoints, 100KB outputs each", + totalWaitpoints: 236, + outputSizeKB: 100, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 150 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 200 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + ], + queryFromIndex: 6, + expectedWaitpointsOnLatest: 236, + }, + { + name: "batch_500_large", + description: "Large batch requiring chunked fetching", + totalWaitpoints: 500, + outputSizeKB: 50, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 250 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 400 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 500 }, + { status: "SUSPENDED", completedWaitpointCount: 500, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 500 }, + { status: "EXECUTING", completedWaitpointCount: 500 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 500 }, + { status: "SUSPENDED", completedWaitpointCount: 500, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 500 }, + { status: "EXECUTING", completedWaitpointCount: 500 }, + ], + queryFromIndex: 5, + expectedWaitpointsOnLatest: 500, + }, + { + name: "system_failure_finished", + description: "Latest snapshot is FINISHED status with completed waitpoints", + totalWaitpoints: 100, + outputSizeKB: 50, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 50 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 100 }, + { status: "EXECUTING", completedWaitpointCount: 100 }, + { status: "FINISHED", completedWaitpointCount: 100 }, + ], + queryFromIndex: 3, + expectedWaitpointsOnLatest: 100, + }, + { + name: "query_from_latest", + description: "Querying from the latest snapshot should return empty array", + totalWaitpoints: 10, + outputSizeKB: 10, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 10 }, + ], + queryFromIndex: 4, // The last snapshot + expectedWaitpointsOnLatest: 0, // No snapshots returned, so no waitpoints + }, + { + name: "requeue_loop", + description: "Multiple QUEUED->PENDING_EXECUTING cycles with waitpoints", + totalWaitpoints: 236, + outputSizeKB: 100, + snapshots: [ + { status: "RUN_CREATED", completedWaitpointCount: 0 }, + { status: "QUEUED", completedWaitpointCount: 0 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 0 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued again + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued again + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING_WITH_WAITPOINTS", completedWaitpointCount: 236 }, + { status: "SUSPENDED", completedWaitpointCount: 236, hasCheckpoint: true }, + { status: "QUEUED", completedWaitpointCount: 236 }, + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "QUEUED", completedWaitpointCount: 236 }, // Requeued + { status: "PENDING_EXECUTING", completedWaitpointCount: 236 }, + { status: "EXECUTING", completedWaitpointCount: 236 }, + ], + queryFromIndex: 7, + expectedWaitpointsOnLatest: 236, + }, + ]; +} diff --git a/internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts b/internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts new file mode 100644 index 000000000..f981f3514 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/snapshotTestHelpers.ts @@ -0,0 +1,322 @@ +import { generateFriendlyId, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { + PrismaClient, + TaskRunExecutionSnapshot, + TaskRunExecutionStatus, + Waitpoint, + WaitpointStatus, +} from "@trigger.dev/database"; +import type { AuthenticatedEnvironment } from "../setup.js"; + +/** + * Generates a large output string of the specified size in KB. + * The output is a valid JSON string to simulate realistic waitpoint output. + */ +export function generateLargeOutput(sizeKB: number): string { + if (sizeKB <= 0) return JSON.stringify({ data: "" }); + + // Create a string that's approximately the target size + // Account for JSON wrapper overhead + const targetBytes = sizeKB * 1024; + const overhead = JSON.stringify({ data: "" }).length; + const payloadSize = Math.max(0, targetBytes - overhead); + + // Generate a payload of repeating 'x' characters + const payload = "x".repeat(payloadSize); + return JSON.stringify({ data: payload }); +} + +/** + * Creates waitpoints with specified output sizes for testing. + */ +export async function createWaitpointsWithOutput( + prisma: PrismaClient, + count: number, + outputSizeKB: number, + environmentId: string, + projectId: string +): Promise { + if (count === 0) return []; + + const output = generateLargeOutput(outputSizeKB); + const waitpoints: Waitpoint[] = []; + + // Create waitpoints in batches to avoid overwhelming the database + const batchSize = 50; + for (let i = 0; i < count; i += batchSize) { + const batchCount = Math.min(batchSize, count - i); + const batch = await Promise.all( + Array.from({ length: batchCount }).map(async (_, j) => { + const waitpointIds = WaitpointId.generate(); + return prisma.waitpoint.create({ + data: { + id: waitpointIds.id, + friendlyId: waitpointIds.friendlyId, + type: "MANUAL", + status: "COMPLETED" as WaitpointStatus, + idempotencyKey: `test-idempotency-${waitpointIds.id}`, + userProvidedIdempotencyKey: false, + completedAt: new Date(), + output, + outputType: "application/json", + outputIsError: false, + environmentId, + projectId, + }, + }); + }) + ); + waitpoints.push(...batch); + } + + return waitpoints; +} + +/** + * Creates a snapshot directly in the database for testing purposes. + * This bypasses the normal engine flow to allow creating specific test scenarios. + */ +export async function createTestSnapshot( + prisma: PrismaClient, + { + runId, + status, + environmentId, + environmentType, + projectId, + organizationId, + completedWaitpointIds, + checkpointId, + previousSnapshotId, + batchId, + workerId, + runnerId, + attemptNumber, + }: { + runId: string; + status: TaskRunExecutionStatus; + environmentId: string; + environmentType: "PRODUCTION" | "STAGING" | "DEVELOPMENT" | "PREVIEW"; + projectId: string; + organizationId: string; + completedWaitpointIds?: string[]; + checkpointId?: string; + previousSnapshotId?: string; + batchId?: string; + workerId?: string; + runnerId?: string; + attemptNumber?: number; + } +): Promise { + // Determine run status based on execution status + const runStatus = getRunStatusFromExecutionStatus(status); + + const snapshot = await prisma.taskRunExecutionSnapshot.create({ + data: { + engine: "V2", + executionStatus: status, + description: `Test snapshot: ${status}`, + previousSnapshotId, + runId, + runStatus, + attemptNumber, + batchId, + environmentId, + environmentType, + projectId, + organizationId, + checkpointId, + workerId, + runnerId, + isValid: true, + completedWaitpoints: completedWaitpointIds + ? { + connect: completedWaitpointIds.map((id) => ({ id })), + } + : undefined, + completedWaitpointOrder: completedWaitpointIds ?? [], + }, + }); + + // Small delay to ensure different createdAt timestamps + await new Promise((resolve) => setTimeout(resolve, 5)); + + return snapshot; +} + +/** + * Maps execution status to run status for test snapshot creation. + */ +function getRunStatusFromExecutionStatus( + status: TaskRunExecutionStatus +): "PENDING" | "EXECUTING" | "WAITING_FOR_DEPLOY" | "COMPLETED_SUCCESSFULLY" | "SYSTEM_FAILURE" { + switch (status) { + case "RUN_CREATED": + case "QUEUED": + case "QUEUED_EXECUTING": + case "PENDING_EXECUTING": + case "DELAYED": + return "PENDING"; + case "EXECUTING": + case "EXECUTING_WITH_WAITPOINTS": + case "SUSPENDED": + case "PENDING_CANCEL": + return "EXECUTING"; + case "FINISHED": + return "COMPLETED_SUCCESSFULLY"; + default: + return "PENDING"; + } +} + +/** + * Creates a checkpoint for testing suspended snapshots. + */ +export async function createTestCheckpoint( + prisma: PrismaClient, + { + runId, + environmentId, + projectId, + }: { + runId: string; + environmentId: string; + projectId: string; + } +) { + return prisma.taskRunCheckpoint.create({ + data: { + friendlyId: generateFriendlyId("checkpoint"), + type: "DOCKER", + location: `s3://test-bucket/checkpoints/${runId}`, + imageRef: `test-image:${runId}`, + reason: "WAIT_FOR_DURATION", + runtimeEnvironment: { + connect: { id: environmentId }, + }, + project: { + connect: { id: projectId }, + }, + }, + }); +} + +/** + * Interface for a complete test scenario setup result. + */ +export interface TestScenarioResult { + run: { + id: string; + friendlyId: string; + }; + snapshots: TaskRunExecutionSnapshot[]; + waitpoints: Waitpoint[]; + checkpoints: Array<{ id: string }>; +} + +/** + * Sets up a complete test scenario with run, snapshots, waitpoints, and checkpoints. + * This creates the full database state needed for testing getSnapshotsSince. + */ +export async function setupTestScenario( + prisma: PrismaClient, + environment: AuthenticatedEnvironment, + { + totalWaitpoints, + outputSizeKB, + snapshotConfigs, + }: { + totalWaitpoints: number; + outputSizeKB: number; + snapshotConfigs: Array<{ + status: TaskRunExecutionStatus; + completedWaitpointCount: number; + hasCheckpoint?: boolean; + }>; + } +): Promise { + // Create waitpoints first + const waitpoints = await createWaitpointsWithOutput( + prisma, + totalWaitpoints, + outputSizeKB, + environment.id, + environment.project.id + ); + + // Create the run + const runFriendlyId = generateFriendlyId("run"); + const run = await prisma.taskRun.create({ + data: { + friendlyId: runFriendlyId, + engine: "V2", + status: "PENDING", + runtimeEnvironmentId: environment.id, + environmentType: environment.type, + organizationId: environment.organization.id, + projectId: environment.project.id, + taskIdentifier: "test-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${runFriendlyId}`, + spanId: `span_${runFriendlyId}`, + context: {}, + traceContext: {}, + isTest: false, + queue: "task/test-task", + workerQueue: "main", + }, + }); + + // Create snapshots in order + const snapshots: TaskRunExecutionSnapshot[] = []; + const checkpoints: Array<{ id: string }> = []; + let previousSnapshotId: string | undefined; + let attemptNumber = 0; + + for (const config of snapshotConfigs) { + // Create checkpoint if needed + let checkpointId: string | undefined; + if (config.hasCheckpoint) { + const checkpoint = await createTestCheckpoint(prisma, { + runId: run.id, + environmentId: environment.id, + projectId: environment.project.id, + }); + checkpointId = checkpoint.id; + checkpoints.push({ id: checkpoint.id }); + } + + // Increment attempt number when entering a new execution attempt + // PENDING_EXECUTING is the entry point - EXECUTING follows within the same attempt + if (config.status === "PENDING_EXECUTING") { + attemptNumber++; + } + + // Get the waitpoint IDs that should be "completed" at this snapshot + const completedWaitpointIds = waitpoints.slice(0, config.completedWaitpointCount).map((w) => w.id); + + const snapshot = await createTestSnapshot(prisma, { + runId: run.id, + status: config.status, + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.project.id, + organizationId: environment.organization.id, + completedWaitpointIds, + checkpointId, + previousSnapshotId, + attemptNumber, + }); + + snapshots.push(snapshot); + previousSnapshotId = snapshot.id; + } + + return { + run: { id: run.id, friendlyId: runFriendlyId }, + snapshots, + waitpoints, + checkpoints, + }; +} From 01208fde2750701016afe7170a4bf9197d1f4d58 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Fri, 30 Jan 2026 08:43:23 +0000 Subject: [PATCH 08/22] chore(docs): Add playwright workaround (#2973) Adds a workaround to playwright to browser download failures based on this GH issue: https://github.com/triggerdotdev/trigger.dev/issues/2440 --- Open with Devin --- docs/config/extensions/playwright.mdx | 26 +++ docs/docs.json | 8 + docs/management/deployments/get-latest.mdx | 4 + docs/management/deployments/promote.mdx | 4 + docs/management/deployments/retrieve.mdx | 4 + docs/v3-openapi.yaml | 228 +++++++++++++++++++++ 6 files changed, 274 insertions(+) create mode 100644 docs/management/deployments/get-latest.mdx create mode 100644 docs/management/deployments/promote.mdx create mode 100644 docs/management/deployments/retrieve.mdx diff --git a/docs/config/extensions/playwright.mdx b/docs/config/extensions/playwright.mdx index db9ad3425..ea8461301 100644 --- a/docs/config/extensions/playwright.mdx +++ b/docs/config/extensions/playwright.mdx @@ -91,6 +91,32 @@ The extension sets the following environment variables during the build: - `PLAYWRIGHT_SKIP_BROWSER_VALIDATION`: Set to `1` to skip browser validation at runtime - `DISPLAY`: Set to `:99` if `headless: false` (for Xvfb) +## Troubleshooting + +### Browser download failures + +If you encounter errors during the build process related to browser downloads (e.g., "failed to solve: process did not complete successfully: exit code: 9"), this is a known issue with certain Playwright versions. + +**Workaround:** Revert Playwright to version `1.40.0` in your project dependencies. You can specify this version explicitly in your config: + +```ts +import { defineConfig } from "@trigger.dev/sdk"; +import { playwright } from "@trigger.dev/build/extensions/playwright"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + playwright({ + version: "1.40.0", + }), + ], + }, +}); +``` + +For more details, see [GitHub issue #2440](https://github.com/triggerdotdev/trigger.dev/issues/2440#issuecomment-3815104376). + ## Managing browser instances To prevent issues with waits and resumes, you can use middleware and locals to manage the browser instance. This will ensure the browser is available for the whole run, and is properly cleaned up on waits, resumes, and after the run completes. diff --git a/docs/docs.json b/docs/docs.json index dcf637aea..c1f5d2738 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -276,6 +276,14 @@ "management/envvars/update", "management/envvars/delete" ] + }, + { + "group": "Deployments API", + "pages": [ + "management/deployments/retrieve", + "management/deployments/get-latest", + "management/deployments/promote" + ] } ] }, diff --git a/docs/management/deployments/get-latest.mdx b/docs/management/deployments/get-latest.mdx new file mode 100644 index 000000000..78be92be2 --- /dev/null +++ b/docs/management/deployments/get-latest.mdx @@ -0,0 +1,4 @@ +--- +title: "Get latest deployment" +openapi: "v3-openapi GET /api/v1/deployments/latest" +--- diff --git a/docs/management/deployments/promote.mdx b/docs/management/deployments/promote.mdx new file mode 100644 index 000000000..e1e885c0d --- /dev/null +++ b/docs/management/deployments/promote.mdx @@ -0,0 +1,4 @@ +--- +title: "Promote deployment" +openapi: "v3-openapi POST /api/v1/deployments/{version}/promote" +--- diff --git a/docs/management/deployments/retrieve.mdx b/docs/management/deployments/retrieve.mdx new file mode 100644 index 000000000..8a7eb2155 --- /dev/null +++ b/docs/management/deployments/retrieve.mdx @@ -0,0 +1,4 @@ +--- +title: "Get deployment" +openapi: "v3-openapi GET /api/v1/deployments/{deploymentId}" +--- diff --git a/docs/v3-openapi.yaml b/docs/v3-openapi.yaml index d406ce6c9..2fdcd0afd 100644 --- a/docs/v3-openapi.yaml +++ b/docs/v3-openapi.yaml @@ -505,6 +505,234 @@ paths: await runs.cancel("run_1234"); + "/api/v1/deployments/{deploymentId}": + parameters: + - in: path + name: deploymentId + required: true + schema: + type: string + description: The deployment ID. + get: + operationId: get_deployment_v1 + summary: Get deployment + description: Retrieve information about a specific deployment by its ID. + responses: + "200": + description: Successful request + content: + application/json: + schema: + type: object + properties: + id: + type: string + description: The deployment ID + status: + type: string + enum: ["PENDING", "INSTALLING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED", "TIMED_OUT"] + description: The current status of the deployment + contentHash: + type: string + description: Hash of the deployment content + shortCode: + type: string + description: The short code for the deployment + version: + type: string + description: The deployment version (e.g., "20250228.1") + imageReference: + type: string + nullable: true + description: Reference to the deployment image + imagePlatform: + type: string + description: Platform of the deployment image + externalBuildData: + type: object + nullable: true + description: External build data if applicable + errorData: + type: object + nullable: true + description: Error data if the deployment failed + worker: + type: object + nullable: true + description: Worker information if available + properties: + id: + type: string + version: + type: string + tasks: + type: array + items: + type: object + properties: + id: + type: string + slug: + type: string + filePath: + type: string + exportName: + type: string + "401": + description: Unauthorized - Access token is missing or invalid + "404": + description: Deployment not found + tags: + - deployments + security: + - secretKey: [] + x-codeSamples: + - lang: typescript + source: |- + const response = await fetch( + `https://api.trigger.dev/api/v1/deployments/${deploymentId}`, + { + method: "GET", + headers: { + "Authorization": `Bearer ${secretKey}`, + }, + } + ); + const deployment = await response.json(); + - lang: curl + source: |- + curl -X GET "https://api.trigger.dev/api/v1/deployments/deployment_1234" \ + -H "Authorization: Bearer tr_dev_1234" + + "/api/v1/deployments/latest": + get: + operationId: get_latest_deployment_v1 + summary: Get latest deployment + description: Retrieve information about the latest unmanaged deployment for the authenticated project. + responses: + "200": + description: Successful request + content: + application/json: + schema: + type: object + properties: + id: + type: string + description: The deployment ID + status: + type: string + enum: ["PENDING", "INSTALLING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED", "TIMED_OUT"] + description: The current status of the deployment + contentHash: + type: string + description: Hash of the deployment content + shortCode: + type: string + description: The short code for the deployment + version: + type: string + description: The deployment version (e.g., "20250228.1") + imageReference: + type: string + nullable: true + description: Reference to the deployment image + errorData: + type: object + nullable: true + description: Error data if the deployment failed + "401": + description: Unauthorized - API key is missing or invalid + "404": + description: No deployment found + tags: + - deployments + security: + - secretKey: [] + x-codeSamples: + - lang: typescript + source: |- + const response = await fetch( + "https://api.trigger.dev/api/v1/deployments/latest", + { + method: "GET", + headers: { + "Authorization": `Bearer ${secretKey}`, + }, + } + ); + const deployment = await response.json(); + - lang: curl + source: |- + curl -X GET "https://api.trigger.dev/api/v1/deployments/latest" \ + -H "Authorization: Bearer tr_dev_1234" + + "/api/v1/deployments/{version}/promote": + parameters: + - in: path + name: version + required: true + schema: + type: string + description: The deployment version to promote (e.g., "20250228.1"). + post: + operationId: promote_deployment_v1 + summary: Promote deployment + description: Promote a previously deployed version to be the current version for the environment. This makes the specified version active for new task runs. + responses: + "200": + description: Deployment promoted successfully + content: + application/json: + schema: + type: object + properties: + id: + type: string + description: The deployment ID + version: + type: string + description: The deployment version (e.g., "20250228.1") + shortCode: + type: string + description: The short code for the deployment + "400": + description: Invalid request + content: + application/json: + schema: + type: object + properties: + error: + type: string + "401": + description: Unauthorized - API key is missing or invalid + "404": + description: Deployment not found + tags: + - deployments + security: + - secretKey: [] + x-codeSamples: + - lang: typescript + source: |- + const response = await fetch( + `https://api.trigger.dev/api/v1/deployments/${version}/promote`, + { + method: "POST", + headers: { + "Authorization": `Bearer ${secretKey}`, + "Content-Type": "application/json", + }, + } + ); + const result = await response.json(); + - lang: curl + source: |- + curl -X POST "https://api.trigger.dev/api/v1/deployments/20250228.1/promote" \ + -H "Authorization: Bearer tr_dev_1234" \ + -H "Content-Type: application/json" + "/api/v1/runs/{runId}/reschedule": parameters: - $ref: "#/components/parameters/runId" From 3925f8cc49c0316b7ffdcf255433c208987107b4 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 30 Jan 2026 09:15:02 +0000 Subject: [PATCH 09/22] fix(core): vendor superjson to fix ESM/CJS compatibility (#2949) Bundle superjson and its dependency (copy-anything) during build to avoid ERR_REQUIRE_ESM errors on Node.js versions that don't support require(ESM) by default (< 22.12.0) and AWS Lambda which intentionally disables it. - Add scripts/bundle-superjson.mjs to bundle superjson with esbuild - Update build script to bundle vendor files before tshy compilation - Move superjson from dependencies to devDependencies - Update imports to use vendored bundles Fixes #2937 --- Open with Devin --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Eric Allam --- .changeset/vendor-superjson-esm-fix.md | 7 + .github/workflows/pr_checks.yml | 4 + .github/workflows/sdk-compat.yml | 178 +++++++ .gitignore | 3 + .../sdk-compat-tests/package.json | 20 + .../src/fixtures/bun/package.json | 5 + .../sdk-compat-tests/src/fixtures/bun/test.ts | 62 +++ .../src/fixtures/cjs-require/package.json | 4 + .../src/fixtures/cjs-require/test.cjs | 57 +++ .../fixtures/cloudflare-worker/package.json | 11 + .../fixtures/cloudflare-worker/src/index.ts | 41 ++ .../fixtures/cloudflare-worker/wrangler.toml | 4 + .../src/fixtures/deno/deno.json | 6 + .../src/fixtures/deno/test.ts | 65 +++ .../src/fixtures/esm-import/package.json | 5 + .../fixtures/esm-import/superjson-test.mjs | 59 +++ .../src/fixtures/esm-import/test.mjs | 54 ++ .../src/fixtures/typescript/package.json | 5 + .../src/fixtures/typescript/test.ts | 74 +++ .../src/fixtures/typescript/tsconfig.json | 12 + .../src/tests/bundler.test.ts | 117 +++++ .../sdk-compat-tests/src/tests/import.test.ts | 84 ++++ .../sdk-compat-tests/tsconfig.json | 16 + .../sdk-compat-tests/vitest.config.ts | 11 + packages/core/package.json | 12 +- packages/core/scripts/bundle-superjson.mjs | 93 ++++ .../core/src/v3/imports/superjson-cjs.cts | 8 +- packages/core/src/v3/imports/superjson.ts | 10 +- pnpm-lock.yaml | 464 +++++++++++++++++- 29 files changed, 1463 insertions(+), 28 deletions(-) create mode 100644 .changeset/vendor-superjson-esm-fix.md create mode 100644 .github/workflows/sdk-compat.yml create mode 100644 internal-packages/sdk-compat-tests/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/bun/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts create mode 100644 internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json create mode 100644 internal-packages/sdk-compat-tests/src/tests/bundler.test.ts create mode 100644 internal-packages/sdk-compat-tests/src/tests/import.test.ts create mode 100644 internal-packages/sdk-compat-tests/tsconfig.json create mode 100644 internal-packages/sdk-compat-tests/vitest.config.ts create mode 100644 packages/core/scripts/bundle-superjson.mjs diff --git a/.changeset/vendor-superjson-esm-fix.md b/.changeset/vendor-superjson-esm-fix.md new file mode 100644 index 000000000..ef04201d2 --- /dev/null +++ b/.changeset/vendor-superjson-esm-fix.md @@ -0,0 +1,7 @@ +--- +"@trigger.dev/core": patch +--- + +fix: vendor superjson to fix ESM/CJS compatibility + +Bundle superjson during build to avoid `ERR_REQUIRE_ESM` errors on Node.js versions that don't support `require(ESM)` by default (< 22.12.0) and AWS Lambda which intentionally disables it. diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index b6be1eddf..dab18223e 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -29,3 +29,7 @@ jobs: with: package: cli-v3 secrets: inherit + + sdk-compat: + uses: ./.github/workflows/sdk-compat.yml + secrets: inherit diff --git a/.github/workflows/sdk-compat.yml b/.github/workflows/sdk-compat.yml new file mode 100644 index 000000000..eb347c0f7 --- /dev/null +++ b/.github/workflows/sdk-compat.yml @@ -0,0 +1,178 @@ +name: "🔌 SDK Compatibility Tests" + +permissions: + contents: read + +on: + workflow_call: + +jobs: + node-compat: + name: "Node.js ${{ matrix.node }} (${{ matrix.os }})" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + node: ["20.20", "22.12"] + + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + shell: bash + run: pnpm run build --filter '@trigger.dev/sdk^...' + + - name: 🔨 Build SDK + shell: bash + run: pnpm run build --filter '@trigger.dev/sdk' + + - name: 🧪 Run SDK Compatibility Tests + shell: bash + run: pnpm --filter @internal/sdk-compat-tests test + + bun-compat: + name: "Bun Runtime" + runs-on: ubuntu-latest + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: 20.20.0 + cache: "pnpm" + + - name: 🥟 Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: 🔨 Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: 🧪 Run Bun Compatibility Test + working-directory: internal-packages/sdk-compat-tests/src/fixtures/bun + run: bun run test.ts + + deno-compat: + name: "Deno Runtime" + runs-on: ubuntu-latest + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: 20.20.0 + cache: "pnpm" + + - name: 🦕 Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: 🔨 Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: 🔗 Link node_modules for Deno fixture + working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno + run: ln -s ../../../../../node_modules node_modules + + - name: 🧪 Run Deno Compatibility Test + working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno + run: deno run --allow-read --allow-env --allow-sys test.ts + + cloudflare-compat: + name: "Cloudflare Workers" + runs-on: ubuntu-latest + steps: + - name: ⬇️ Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ⎔ Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.23.0 + + - name: ⎔ Setup node + uses: buildjet/setup-node@v4 + with: + node-version: 20.20.0 + cache: "pnpm" + + - name: 📥 Download deps + run: pnpm install --frozen-lockfile + + - name: 📀 Generate Prisma Client + run: pnpm run generate + + - name: 🔨 Build SDK dependencies + run: pnpm run build --filter @trigger.dev/sdk^... + + - name: 🔨 Build SDK + run: pnpm run build --filter @trigger.dev/sdk + + - name: 📥 Install Cloudflare fixture deps + working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker + run: pnpm install + + - name: 🧪 Run Cloudflare Workers Compatibility Test (dry-run) + working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker + run: npx wrangler deploy --dry-run --outdir dist diff --git a/.gitignore b/.gitignore index d0dfea89c..071b9b590 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ out/ dist packages/**/dist +# vendored bundles (generated during build) +packages/**/src/**/vendor + # Tailwind apps/**/styles/tailwind.css packages/**/styles/tailwind.css diff --git a/internal-packages/sdk-compat-tests/package.json b/internal-packages/sdk-compat-tests/package.json new file mode 100644 index 000000000..e903e69f3 --- /dev/null +++ b/internal-packages/sdk-compat-tests/package.json @@ -0,0 +1,20 @@ +{ + "name": "@internal/sdk-compat-tests", + "private": true, + "version": "0.0.1", + "type": "module", + "scripts": { + "test": "vitest", + "test:watch": "vitest", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@trigger.dev/sdk": "workspace:*" + }, + "devDependencies": { + "esbuild": "^0.24.0", + "execa": "^9.3.0", + "typescript": "^5.5.0", + "vitest": "3.1.4" + } +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/bun/package.json b/internal-packages/sdk-compat-tests/src/fixtures/bun/package.json new file mode 100644 index 000000000..c69e2dd23 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/bun/package.json @@ -0,0 +1,5 @@ +{ + "name": "bun-fixture", + "private": true, + "type": "module" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts b/internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts new file mode 100644 index 000000000..853869304 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/bun/test.ts @@ -0,0 +1,62 @@ +/** + * Bun Import Test Fixture + * + * Tests that the SDK works correctly with Bun runtime. + * Bun has high Node.js compatibility but uses its own module resolver. + */ + +import { task, logger, schedules, runs, configure, queue, retry, wait } from "@trigger.dev/sdk"; + +// Validate exports exist +const checks: [string, boolean][] = [ + ["task", typeof task === "function"], + ["logger", typeof logger === "object" && typeof logger.info === "function"], + ["schedules", typeof schedules === "object"], + ["runs", typeof runs === "object"], + ["configure", typeof configure === "function"], + ["queue", typeof queue === "function"], + ["retry", typeof retry === "object"], + ["wait", typeof wait === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition with types +interface Payload { + message: string; +} + +const myTask = task({ + id: "bun-test-task", + run: async (payload: Payload) => { + return { received: payload.message }; + }, +}); + +if (myTask.id !== "bun-test-task") { + console.error(`FAIL: task.id mismatch`); + failed = true; +} + +// Test queue definition +const myQueue = queue({ + name: "bun-test-queue", + concurrencyLimit: 5, +}); + +if (!myQueue) { + console.error(`FAIL: queue creation failed`); + failed = true; +} + +if (failed) { + process.exit(1); +} + +console.log("SUCCESS: Bun imports validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json new file mode 100644 index 000000000..953ed7d2d --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/package.json @@ -0,0 +1,4 @@ +{ + "name": "cjs-require-fixture", + "private": true +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs new file mode 100644 index 000000000..447d03970 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cjs-require/test.cjs @@ -0,0 +1,57 @@ +/** + * CJS Require Test Fixture + * + * This file validates that the SDK can be required using CommonJS syntax. + * This is critical for: + * - Node.js < 22.12.0 (where require(ESM) is not enabled by default) + * - AWS Lambda (intentionally disables require(ESM)) + * - Legacy Node.js applications + */ + +// Test main export +const sdk = require("@trigger.dev/sdk"); + +// Test /v3 subpath +const sdkV3 = require("@trigger.dev/sdk/v3"); + +// Validate exports exist +const checks = [ + ["task", typeof sdk.task === "function"], + ["taskV3", typeof sdkV3.task === "function"], + ["logger", typeof sdk.logger === "object" && typeof sdk.logger.info === "function"], + ["schedules", typeof sdk.schedules === "object"], + ["runs", typeof sdk.runs === "object"], + ["configure", typeof sdk.configure === "function"], + ["queue", typeof sdk.queue === "function"], + ["retry", typeof sdk.retry === "object"], + ["wait", typeof sdk.wait === "object"], + ["metadata", typeof sdk.metadata === "object"], + ["tags", typeof sdk.tags === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition works +const myTask = sdk.task({ + id: "cjs-test-task", + run: async (payload) => { + return { received: payload }; + }, +}); + +if (myTask.id !== "cjs-test-task") { + console.error(`FAIL: task.id mismatch: expected "cjs-test-task", got "${myTask.id}"`); + failed = true; +} + +if (failed) { + process.exit(1); +} + +console.log("SUCCESS: All CJS requires validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json new file mode 100644 index 000000000..d9fca987c --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/package.json @@ -0,0 +1,11 @@ +{ + "name": "cloudflare-worker-fixture", + "private": true, + "type": "module", + "scripts": { + "build": "wrangler deploy --dry-run --outdir dist" + }, + "devDependencies": { + "wrangler": "^3.0.0" + } +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts new file mode 100644 index 000000000..30b5fcc79 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/src/index.ts @@ -0,0 +1,41 @@ +/** + * Cloudflare Worker Test Fixture + * + * Tests that the SDK can be bundled for Cloudflare Workers (workerd runtime). + * This validates the bundling process works - actual execution would require + * a Trigger.dev API connection. + */ + +import { task, runs, configure } from "@trigger.dev/sdk"; + +// Define a task (won't execute in worker, but validates import) +const myTask = task({ + id: "cloudflare-test-task", + run: async (payload: { message: string }) => { + return { received: payload.message }; + }, +}); + +export default { + async fetch(request: Request, env: unknown, ctx: ExecutionContext): Promise { + // Validate SDK imports work + const checks = { + taskDefined: typeof task === "function", + runsDefined: typeof runs === "object", + configureDefined: typeof configure === "function", + taskIdCorrect: myTask.id === "cloudflare-test-task", + }; + + const allPassed = Object.values(checks).every((v) => v === true); + + return new Response( + JSON.stringify({ + success: allPassed, + checks, + }), + { + headers: { "Content-Type": "application/json" }, + } + ); + }, +}; diff --git a/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml new file mode 100644 index 000000000..f038e47bb --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker/wrangler.toml @@ -0,0 +1,4 @@ +name = "sdk-compat-test" +main = "src/index.ts" +compatibility_date = "2024-01-01" +compatibility_flags = ["nodejs_compat"] diff --git a/internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json b/internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json new file mode 100644 index 000000000..4525b34d3 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/deno/deno.json @@ -0,0 +1,6 @@ +{ + "tasks": { + "test": "deno run --allow-read --allow-env --allow-sys test.ts" + }, + "nodeModulesDir": "manual" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts b/internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts new file mode 100644 index 000000000..6894606fd --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/deno/test.ts @@ -0,0 +1,65 @@ +/** + * Deno Import Test Fixture + * + * Tests that the SDK can be imported in Deno using Node.js compatibility. + * The CI workflow installs the SDK into node_modules via npm for local resolution. + */ + +// Use bare specifier - resolved via node_modules when nodeModulesDir is enabled +import { task, logger, schedules, runs, configure, queue, retry, wait, metadata, tags } from "@trigger.dev/sdk"; + +// Validate exports exist +const checks: [string, boolean][] = [ + ["task", typeof task === "function"], + ["logger", typeof logger === "object" && typeof logger.info === "function"], + ["schedules", typeof schedules === "object"], + ["runs", typeof runs === "object"], + ["configure", typeof configure === "function"], + ["queue", typeof queue === "function"], + ["retry", typeof retry === "object"], + ["wait", typeof wait === "object"], + ["metadata", typeof metadata === "object"], + ["tags", typeof tags === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition with types +interface Payload { + message: string; +} + +const myTask = task({ + id: "deno-test-task", + run: async (payload: Payload) => { + return { received: payload.message }; + }, +}); + +if (myTask.id !== "deno-test-task") { + console.error(`FAIL: task.id mismatch`); + failed = true; +} + +// Test queue definition +const myQueue = queue({ + name: "deno-test-queue", + concurrencyLimit: 5, +}); + +if (!myQueue) { + console.error(`FAIL: queue creation failed`); + failed = true; +} + +if (failed) { + Deno.exit(1); +} + +console.log("SUCCESS: Deno imports validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json new file mode 100644 index 000000000..c0d56cd02 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/package.json @@ -0,0 +1,5 @@ +{ + "name": "esm-import-fixture", + "private": true, + "type": "module" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs new file mode 100644 index 000000000..fc034de19 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/superjson-test.mjs @@ -0,0 +1,59 @@ +/** + * SuperJSON Serialization Test + * + * This validates the fix for #2937 - ESM/CJS compatibility with superjson. + * Tests that complex types (Date, Set, Map, BigInt) serialize correctly. + */ + +import { task, logger } from "@trigger.dev/sdk"; + +// The SDK uses superjson internally for serialization +// This test ensures the vendored superjson works correctly + +const complexData = { + date: new Date("2024-01-15T12:00:00Z"), + set: new Set([1, 2, 3]), + map: new Map([ + ["key1", "value1"], + ["key2", "value2"], + ]), + bigint: BigInt("9007199254740991"), + nested: { + innerDate: new Date("2024-06-01"), + innerSet: new Set(["a", "b"]), + }, +}; + +// Create a task that uses complex types +const complexTask = task({ + id: "superjson-test-task", + run: async (payload) => { + // Just verify the payload structure matches expectations + return { + hasDate: payload.date instanceof Date, + hasSet: payload.set instanceof Set, + hasMap: payload.map instanceof Map, + hasBigInt: typeof payload.bigint === "bigint", + hasNestedDate: payload.nested?.innerDate instanceof Date, + }; + }, +}); + +// Verify task was created successfully +if (!complexTask.id) { + console.error("FAIL: Task creation failed"); + process.exit(1); +} + +// Test that logger works (it uses superjson for structured logging) +try { + logger.info("Testing superjson serialization", { + complexData, + timestamp: new Date(), + }); +} catch (error) { + console.error("FAIL: Logger with complex data failed:", error); + process.exit(1); +} + +console.log("SUCCESS: SuperJSON serialization validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs new file mode 100644 index 000000000..70b055aaa --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/esm-import/test.mjs @@ -0,0 +1,54 @@ +/** + * ESM Import Test Fixture + * + * This file validates that the SDK can be imported using ESM syntax. + * It tests all major export paths and verifies runtime functionality. + */ + +// Test main export +import { task, logger, schedules, runs, configure, queue, retry, wait, metadata, tags } from "@trigger.dev/sdk"; + +// Test /v3 subpath (legacy, but should still work) +import { task as taskV3 } from "@trigger.dev/sdk/v3"; + +// Validate exports are functions/objects +const checks = [ + ["task", typeof task === "function"], + ["taskV3", typeof taskV3 === "function"], + ["logger", typeof logger === "object" && typeof logger.info === "function"], + ["schedules", typeof schedules === "object"], + ["runs", typeof runs === "object"], + ["configure", typeof configure === "function"], + ["queue", typeof queue === "function"], + ["retry", typeof retry === "object"], + ["wait", typeof wait === "object"], + ["metadata", typeof metadata === "object"], + ["tags", typeof tags === "object"], +]; + +let failed = false; +for (const [name, passed] of checks) { + if (!passed) { + console.error(`FAIL: ${name} export check failed`); + failed = true; + } +} + +// Test task definition works +const myTask = task({ + id: "esm-test-task", + run: async (payload) => { + return { received: payload }; + }, +}); + +if (myTask.id !== "esm-test-task") { + console.error(`FAIL: task.id mismatch: expected "esm-test-task", got "${myTask.id}"`); + failed = true; +} + +if (failed) { + process.exit(1); +} + +console.log("SUCCESS: All ESM imports validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json b/internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json new file mode 100644 index 000000000..7663bc7ac --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/typescript/package.json @@ -0,0 +1,5 @@ +{ + "name": "typescript-fixture", + "private": true, + "type": "module" +} diff --git a/internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts b/internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts new file mode 100644 index 000000000..bfcb4892a --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/typescript/test.ts @@ -0,0 +1,74 @@ +/** + * TypeScript Import Test Fixture + * + * This file validates that the SDK types work correctly with TypeScript. + * It tests type inference, generics, and type-only imports. + */ + +import { + task, + logger, + schedules, + runs, + configure, + queue, + retry, + wait, + metadata, + tags, + type Context, + type RetryOptions, +} from "@trigger.dev/sdk"; + +// Type-only import test +import type { ApiClientConfiguration } from "@trigger.dev/sdk"; + +// Test typed task with payload +interface MyPayload { + message: string; + count: number; +} + +interface MyOutput { + processed: boolean; + result: string; +} + +const typedTask = task({ + id: "typescript-test-task", + run: async (payload: MyPayload, { ctx }): Promise => { + // Verify context type + const runId: string = ctx.run.id; + + return { + processed: true, + result: `Processed ${payload.message} with count ${payload.count}`, + }; + }, +}); + +// Verify task type inference +type TaskPayload = Parameters[0]; +type _PayloadCheck = TaskPayload extends MyPayload ? true : never; + +// Test queue definition +const myQueue = queue({ + name: "test-queue", + concurrencyLimit: 10, +}); + +// Test retry options type +const retryOpts: RetryOptions = { + maxAttempts: 3, + factor: 2, + minTimeoutInMs: 1000, + maxTimeoutInMs: 30000, +}; + +// Validate runtime +if (typedTask.id !== "typescript-test-task") { + console.error(`FAIL: task.id mismatch`); + process.exit(1); +} + +console.log("SUCCESS: TypeScript types validated"); diff --git a/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json b/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json new file mode 100644 index 000000000..432fff32a --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["test.ts"] +} diff --git a/internal-packages/sdk-compat-tests/src/tests/bundler.test.ts b/internal-packages/sdk-compat-tests/src/tests/bundler.test.ts new file mode 100644 index 000000000..e3e18c49f --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/tests/bundler.test.ts @@ -0,0 +1,117 @@ +/** + * Bundler Compatibility Tests + * + * These tests validate that the SDK can be bundled correctly using + * common bundlers like esbuild. + */ + +import { describe, it, expect } from "vitest"; +import * as esbuild from "esbuild"; +import { resolve, dirname } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const fixturesDir = resolve(__dirname, "../fixtures"); + +describe("esbuild Bundling Tests", () => { + it("should bundle ESM entrypoint without errors", async () => { + const result = await esbuild.build({ + entryPoints: [resolve(fixturesDir, "esm-import/test.mjs")], + bundle: true, + format: "esm", + platform: "node", + target: "node18", + write: false, + external: ["@trigger.dev/sdk", "@trigger.dev/sdk/*"], + logLevel: "silent", + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + }); + + it("should bundle CJS entrypoint without errors", async () => { + const result = await esbuild.build({ + entryPoints: [resolve(fixturesDir, "cjs-require/test.cjs")], + bundle: true, + format: "cjs", + platform: "node", + target: "node18", + write: false, + external: ["@trigger.dev/sdk", "@trigger.dev/sdk/*"], + logLevel: "silent", + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + }); + + it("should bundle SDK inline (simulating production build)", async () => { + // This simulates what happens when a user bundles their app with the SDK included + const entryContent = ` + import { task, logger } from "@trigger.dev/sdk"; + + export const myTask = task({ + id: "bundled-task", + run: async (payload) => { + logger.info("Processing", { payload }); + return { success: true }; + }, + }); + `; + + const result = await esbuild.build({ + stdin: { + contents: entryContent, + loader: "ts", + resolveDir: resolve(__dirname, "../../"), + }, + bundle: true, + format: "esm", + platform: "node", + target: "node18", + write: false, + // Don't externalize SDK - bundle it inline + logLevel: "silent", + metafile: true, + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + + // Verify the bundle contains the SDK code + const bundleContent = result.outputFiles[0].text; + expect(bundleContent).toBeTruthy(); + expect(bundleContent.length).toBeGreaterThan(1000); // Should be substantial + }); + + it("should handle tree-shaking correctly", async () => { + // Import only specific functions to test tree-shaking + const entryContent = ` + import { task } from "@trigger.dev/sdk"; + + export const myTask = task({ + id: "tree-shake-task", + run: async () => ({ done: true }), + }); + `; + + const result = await esbuild.build({ + stdin: { + contents: entryContent, + loader: "ts", + resolveDir: resolve(__dirname, "../../"), + }, + bundle: true, + format: "esm", + platform: "node", + target: "node18", + write: false, + treeShaking: true, + logLevel: "silent", + }); + + expect(result.errors).toHaveLength(0); + expect(result.outputFiles).toHaveLength(1); + }); +}); diff --git a/internal-packages/sdk-compat-tests/src/tests/import.test.ts b/internal-packages/sdk-compat-tests/src/tests/import.test.ts new file mode 100644 index 000000000..7d81ccce2 --- /dev/null +++ b/internal-packages/sdk-compat-tests/src/tests/import.test.ts @@ -0,0 +1,84 @@ +/** + * Import Validation Tests + * + * These tests validate that the SDK can be imported correctly across + * different module systems (ESM and CJS). + */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { execa, type Options as ExecaOptions } from "execa"; +import { resolve, dirname } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const fixturesDir = resolve(__dirname, "../fixtures"); + +// Find the SDK package in the monorepo +const sdkDir = resolve(__dirname, "../../../../packages/trigger-sdk"); + +// Common execa options +const execaOpts: ExecaOptions = { + env: { + ...process.env, + // Ensure Node.js can resolve workspace packages + NODE_PATH: resolve(__dirname, "../../../../node_modules"), + }, + timeout: 30_000, +}; + +describe("ESM Import Tests", () => { + it("should import SDK using ESM syntax", async () => { + const result = await execa("node", ["test.mjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "esm-import"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); + + it("should validate superjson serialization in ESM", async () => { + const result = await execa("node", ["superjson-test.mjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "esm-import"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); +}); + +describe("CJS Require Tests", () => { + it("should require SDK using CommonJS syntax", async () => { + const result = await execa("node", ["test.cjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "cjs-require"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); + + it("should work with --experimental-require-module flag on older Node", async () => { + // This flag is needed for Node < 22.12.0 to require ESM modules + // On newer Node.js, it's a no-op + const result = await execa("node", ["--experimental-require-module", "test.cjs"], { + ...execaOpts, + cwd: resolve(fixturesDir, "cjs-require"), + }); + + expect(result.stdout).toContain("SUCCESS"); + expect(result.exitCode).toBe(0); + }); +}); + +describe("TypeScript Compilation Tests", () => { + it("should typecheck SDK imports successfully", async () => { + const result = await execa("npx", ["tsc", "--noEmit"], { + ...execaOpts, + cwd: resolve(fixturesDir, "typescript"), + }); + + expect(result.exitCode).toBe(0); + }); +}); diff --git a/internal-packages/sdk-compat-tests/tsconfig.json b/internal-packages/sdk-compat-tests/tsconfig.json new file mode 100644 index 000000000..05afb6f35 --- /dev/null +++ b/internal-packages/sdk-compat-tests/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": false, + "outDir": "dist", + "rootDir": "src", + "types": ["vitest/globals"] + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "src/fixtures"] +} diff --git a/internal-packages/sdk-compat-tests/vitest.config.ts b/internal-packages/sdk-compat-tests/vitest.config.ts new file mode 100644 index 000000000..2617dd101 --- /dev/null +++ b/internal-packages/sdk-compat-tests/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/tests/**/*.test.ts"], + globals: true, + isolate: true, + testTimeout: 120_000, // Some framework builds can take time + hookTimeout: 60_000, + }, +}); diff --git a/packages/core/package.json b/packages/core/package.json index 989a707ea..d73b425f7 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -157,11 +157,13 @@ }, "sideEffects": false, "scripts": { - "clean": "rimraf dist .tshy .tshy-build .turbo", + "clean": "rimraf dist .tshy .tshy-build .turbo src/v3/vendor", "update-version": "tsx ../../scripts/updateVersion.ts", - "build": "tshy && pnpm run update-version", - "dev": "tshy --watch", - "typecheck": "tsc --noEmit -p tsconfig.src.json", + "bundle-vendor": "node scripts/bundle-superjson.mjs", + "build": "pnpm run bundle-vendor && tshy && node scripts/bundle-superjson.mjs --copy && pnpm run update-version", + "dev": "pnpm run bundle-vendor && tshy --watch", + "typecheck": "pnpm run bundle-vendor && tsc --noEmit -p tsconfig.src.json", + "pretest": "pnpm run bundle-vendor", "test": "vitest", "check-exports": "attw --pack ." }, @@ -193,7 +195,6 @@ "socket.io": "4.7.4", "socket.io-client": "4.7.5", "std-env": "^3.8.1", - "superjson": "^2.2.1", "tinyexec": "^0.3.2", "uncrypto": "^0.1.3", "zod": "3.25.76", @@ -212,6 +213,7 @@ "defu": "^6.1.4", "esbuild": "^0.23.0", "rimraf": "^3.0.2", + "superjson": "^2.2.1", "ts-essentials": "10.0.1", "tshy": "^3.0.2", "tsx": "4.17.0" diff --git a/packages/core/scripts/bundle-superjson.mjs b/packages/core/scripts/bundle-superjson.mjs new file mode 100644 index 000000000..c4e9a7b00 --- /dev/null +++ b/packages/core/scripts/bundle-superjson.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node + +/** + * This script bundles superjson and its dependency (copy-anything) into + * vendored CJS and ESM bundles to avoid the ERR_REQUIRE_ESM error. + * + * superjson v2.x is ESM-only, which causes issues on: + * - Node.js versions before 22.12.0 (require(ESM) not enabled by default) + * - AWS Lambda (intentionally disables require(ESM)) + * + * The output files are gitignored and regenerated during each build. + * This script runs automatically as part of `pnpm run build`. + * + * Usage: + * node scripts/bundle-superjson.mjs # Bundle to src/v3/vendor + * node scripts/bundle-superjson.mjs --copy # Also copy to dist directories + */ + +import * as esbuild from "esbuild"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { readFileSync, mkdirSync, copyFileSync } from "node:fs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const packageRoot = join(__dirname, ".."); +const vendorDir = join(packageRoot, "src", "v3", "vendor"); + +// Get the installed superjson version for the banner +const superjsonPkg = JSON.parse( + readFileSync(join(packageRoot, "node_modules", "superjson", "package.json"), "utf-8") +); +const banner = `/** + * Bundled superjson v${superjsonPkg.version} + * + * This file is auto-generated by scripts/bundle-superjson.mjs + * Do not edit directly - run the script to regenerate. + * + * Original package: https://github.com/flightcontrolhq/superjson + * License: MIT + */`; + +async function bundle() { + // Ensure vendor directory exists + mkdirSync(vendorDir, { recursive: true }); + + // Bundle for CommonJS + await esbuild.build({ + entryPoints: [join(packageRoot, "node_modules", "superjson", "dist", "index.js")], + bundle: true, + format: "cjs", + platform: "node", + target: "node18", + outfile: join(vendorDir, "superjson.cjs"), + banner: { js: banner }, + // Don't minify to keep it debuggable + minify: false, + }); + + // Bundle for ESM + await esbuild.build({ + entryPoints: [join(packageRoot, "node_modules", "superjson", "dist", "index.js")], + bundle: true, + format: "esm", + platform: "node", + target: "node18", + outfile: join(vendorDir, "superjson.mjs"), + banner: { js: banner }, + minify: false, + }); + + console.log("Bundled superjson v" + superjsonPkg.version); + console.log(" -> src/v3/vendor/superjson.cjs (CommonJS)"); + console.log(" -> src/v3/vendor/superjson.mjs (ESM)"); + + // Copy to dist directories if --copy flag is passed + if (process.argv.includes("--copy")) { + const distCommonjsVendor = join(packageRoot, "dist", "commonjs", "v3", "vendor"); + const distEsmVendor = join(packageRoot, "dist", "esm", "v3", "vendor"); + + mkdirSync(distCommonjsVendor, { recursive: true }); + mkdirSync(distEsmVendor, { recursive: true }); + + copyFileSync(join(vendorDir, "superjson.cjs"), join(distCommonjsVendor, "superjson.cjs")); + copyFileSync(join(vendorDir, "superjson.mjs"), join(distEsmVendor, "superjson.mjs")); + + console.log("Copied to dist directories"); + } +} + +bundle().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/core/src/v3/imports/superjson-cjs.cts b/packages/core/src/v3/imports/superjson-cjs.cts index a7f1466e7..89ed3d245 100644 --- a/packages/core/src/v3/imports/superjson-cjs.cts +++ b/packages/core/src/v3/imports/superjson-cjs.cts @@ -1,8 +1,10 @@ +// Use vendored superjson bundle to avoid ESM/CJS compatibility issues +// See: https://github.com/triggerdotdev/trigger.dev/issues/2937 // @ts-ignore -const { default: superjson } = require("superjson"); +const superjson = require("../vendor/superjson.cjs"); // @ts-ignore -superjson.registerCustom( +superjson.default.registerCustom( { isApplicable: (v: unknown): v is Buffer => typeof Buffer === "function" && Buffer.isBuffer(v), serialize: (v: Buffer) => [...v], @@ -12,4 +14,4 @@ superjson.registerCustom( ); // @ts-ignore -module.exports.default = superjson; +module.exports.default = superjson.default; diff --git a/packages/core/src/v3/imports/superjson.ts b/packages/core/src/v3/imports/superjson.ts index aa2925052..1545c083e 100644 --- a/packages/core/src/v3/imports/superjson.ts +++ b/packages/core/src/v3/imports/superjson.ts @@ -1,11 +1,13 @@ +// Use vendored superjson bundle to avoid ESM/CJS compatibility issues +// See: https://github.com/triggerdotdev/trigger.dev/issues/2937 // @ts-ignore -import superjson from "superjson"; +import superjson from "../vendor/superjson.mjs"; superjson.registerCustom( { - isApplicable: (v): v is Buffer => typeof Buffer === "function" && Buffer.isBuffer(v), - serialize: (v) => [...v], - deserialize: (v) => Buffer.from(v), + isApplicable: (v: unknown): v is Buffer => typeof Buffer === "function" && Buffer.isBuffer(v), + serialize: (v: Buffer) => [...v], + deserialize: (v: number[]) => Buffer.from(v), }, "buffer" ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 525beac47..b02c6cd73 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1251,6 +1251,25 @@ importers: specifier: 6.0.1 version: 6.0.1 + internal-packages/sdk-compat-tests: + dependencies: + '@trigger.dev/sdk': + specifier: workspace:* + version: link:../../packages/trigger-sdk + devDependencies: + esbuild: + specifier: ^0.24.0 + version: 0.24.2 + execa: + specifier: ^9.3.0 + version: 9.6.1 + typescript: + specifier: 5.5.4 + version: 5.5.4 + vitest: + specifier: 3.1.4 + version: 3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + internal-packages/testcontainers: dependencies: '@clickhouse/client': @@ -1726,9 +1745,6 @@ importers: std-env: specifier: ^3.8.1 version: 3.8.1 - superjson: - specifier: ^2.2.1 - version: 2.2.1 tinyexec: specifier: ^0.3.2 version: 0.3.2 @@ -1778,6 +1794,9 @@ importers: rimraf: specifier: ^3.0.2 version: 3.0.2 + superjson: + specifier: ^2.2.1 + version: 2.2.1 ts-essentials: specifier: 10.0.1 version: 10.0.1(typescript@5.5.4) @@ -4230,6 +4249,12 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.24.2': + resolution: {integrity: sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/aix-ppc64@0.25.1': resolution: {integrity: sha512-kfYGy8IdzTGy+z0vFGvExZtxkFlA4zAxgKEahG9KE1ScBjpQnFsNOX8KTU5ojNru5ed5CVoJYXFtoxaq5nFbjQ==} engines: {node: '>=18'} @@ -4266,6 +4291,12 @@ packages: cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.24.2': + resolution: {integrity: sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm64@0.25.1': resolution: {integrity: sha512-50tM0zCJW5kGqgG7fQ7IHvQOcAn9TKiVRuQ/lN0xR+T2lzEFvAi1ZcS8DiksFcEpf1t/GYOeOfCAgDHFpkiSmA==} engines: {node: '>=18'} @@ -4308,6 +4339,12 @@ packages: cpu: [arm] os: [android] + '@esbuild/android-arm@0.24.2': + resolution: {integrity: sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-arm@0.25.1': resolution: {integrity: sha512-dp+MshLYux6j/JjdqVLnMglQlFu+MuVeNrmT5nk6q07wNhCdSnB7QZj+7G8VMUGh1q+vj2Bq8kRsuyA00I/k+Q==} engines: {node: '>=18'} @@ -4344,6 +4381,12 @@ packages: cpu: [x64] os: [android] + '@esbuild/android-x64@0.24.2': + resolution: {integrity: sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/android-x64@0.25.1': resolution: {integrity: sha512-GCj6WfUtNldqUzYkN/ITtlhwQqGWu9S45vUXs7EIYf+7rCiiqH9bCloatO9VhxsL0Pji+PF4Lz2XXCES+Q8hDw==} engines: {node: '>=18'} @@ -4380,6 +4423,12 @@ packages: cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.24.2': + resolution: {integrity: sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-arm64@0.25.1': resolution: {integrity: sha512-5hEZKPf+nQjYoSr/elb62U19/l1mZDdqidGfmFutVUjjUZrOazAtwK+Kr+3y0C/oeJfLlxo9fXb1w7L+P7E4FQ==} engines: {node: '>=18'} @@ -4416,6 +4465,12 @@ packages: cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.24.2': + resolution: {integrity: sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/darwin-x64@0.25.1': resolution: {integrity: sha512-hxVnwL2Dqs3fM1IWq8Iezh0cX7ZGdVhbTfnOy5uURtao5OIVCEyj9xIzemDi7sRvKsuSdtCAhMKarxqtlyVyfA==} engines: {node: '>=18'} @@ -4452,6 +4507,12 @@ packages: cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.24.2': + resolution: {integrity: sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-arm64@0.25.1': resolution: {integrity: sha512-1MrCZs0fZa2g8E+FUo2ipw6jw5qqQiH+tERoS5fAfKnRx6NXH31tXBKI3VpmLijLH6yriMZsxJtaXUyFt/8Y4A==} engines: {node: '>=18'} @@ -4488,6 +4549,12 @@ packages: cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.24.2': + resolution: {integrity: sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/freebsd-x64@0.25.1': resolution: {integrity: sha512-0IZWLiTyz7nm0xuIs0q1Y3QWJC52R8aSXxe40VUxm6BB1RNmkODtW6LHvWRrGiICulcX7ZvyH6h5fqdLu4gkww==} engines: {node: '>=18'} @@ -4524,6 +4591,12 @@ packages: cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.24.2': + resolution: {integrity: sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm64@0.25.1': resolution: {integrity: sha512-jaN3dHi0/DDPelk0nLcXRm1q7DNJpjXy7yWaWvbfkPvI+7XNSc/lDOnCLN7gzsyzgu6qSAmgSvP9oXAhP973uQ==} engines: {node: '>=18'} @@ -4560,6 +4633,12 @@ packages: cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.24.2': + resolution: {integrity: sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-arm@0.25.1': resolution: {integrity: sha512-NdKOhS4u7JhDKw9G3cY6sWqFcnLITn6SqivVArbzIaf3cemShqfLGHYMx8Xlm/lBit3/5d7kXvriTUGa5YViuQ==} engines: {node: '>=18'} @@ -4596,6 +4675,12 @@ packages: cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.24.2': + resolution: {integrity: sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-ia32@0.25.1': resolution: {integrity: sha512-OJykPaF4v8JidKNGz8c/q1lBO44sQNUQtq1KktJXdBLn1hPod5rE/Hko5ugKKZd+D2+o1a9MFGUEIUwO2YfgkQ==} engines: {node: '>=18'} @@ -4638,6 +4723,12 @@ packages: cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.24.2': + resolution: {integrity: sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-loong64@0.25.1': resolution: {integrity: sha512-nGfornQj4dzcq5Vp835oM/o21UMlXzn79KobKlcs3Wz9smwiifknLy4xDCLUU0BWp7b/houtdrgUz7nOGnfIYg==} engines: {node: '>=18'} @@ -4674,6 +4765,12 @@ packages: cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.24.2': + resolution: {integrity: sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-mips64el@0.25.1': resolution: {integrity: sha512-1osBbPEFYwIE5IVB/0g2X6i1qInZa1aIoj1TdL4AaAb55xIIgbg8Doq6a5BzYWgr+tEcDzYH67XVnTmUzL+nXg==} engines: {node: '>=18'} @@ -4710,6 +4807,12 @@ packages: cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.24.2': + resolution: {integrity: sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-ppc64@0.25.1': resolution: {integrity: sha512-/6VBJOwUf3TdTvJZ82qF3tbLuWsscd7/1w+D9LH0W/SqUgM5/JJD0lrJ1fVIfZsqB6RFmLCe0Xz3fmZc3WtyVg==} engines: {node: '>=18'} @@ -4746,6 +4849,12 @@ packages: cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.24.2': + resolution: {integrity: sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-riscv64@0.25.1': resolution: {integrity: sha512-nSut/Mx5gnilhcq2yIMLMe3Wl4FK5wx/o0QuuCLMtmJn+WeWYoEGDN1ipcN72g1WHsnIbxGXd4i/MF0gTcuAjQ==} engines: {node: '>=18'} @@ -4782,6 +4891,12 @@ packages: cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.24.2': + resolution: {integrity: sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-s390x@0.25.1': resolution: {integrity: sha512-cEECeLlJNfT8kZHqLarDBQso9a27o2Zd2AQ8USAEoGtejOrCYHNtKP8XQhMDJMtthdF4GBmjR2au3x1udADQQQ==} engines: {node: '>=18'} @@ -4818,12 +4933,24 @@ packages: cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.24.2': + resolution: {integrity: sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/linux-x64@0.25.1': resolution: {integrity: sha512-xbfUhu/gnvSEg+EGovRc+kjBAkrvtk38RlerAzQxvMzlB4fXpCFCeUAYzJvrnhFtdeyVCDANSjJvOvGYoeKzFA==} engines: {node: '>=18'} cpu: [x64] os: [linux] + '@esbuild/netbsd-arm64@0.24.2': + resolution: {integrity: sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-arm64@0.25.1': resolution: {integrity: sha512-O96poM2XGhLtpTh+s4+nP7YCCAfb4tJNRVZHfIE7dgmax+yMP2WgMd2OecBuaATHKTHsLWHQeuaxMRnCsH8+5g==} engines: {node: '>=18'} @@ -4860,6 +4987,12 @@ packages: cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.24.2': + resolution: {integrity: sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/netbsd-x64@0.25.1': resolution: {integrity: sha512-X53z6uXip6KFXBQ+Krbx25XHV/NCbzryM6ehOAeAil7X7oa4XIq+394PWGnwaSQ2WRA0KI6PUO6hTO5zeF5ijA==} engines: {node: '>=18'} @@ -4872,6 +5005,12 @@ packages: cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.24.2': + resolution: {integrity: sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-arm64@0.25.1': resolution: {integrity: sha512-Na9T3szbXezdzM/Kfs3GcRQNjHzM6GzFBeU1/6IV/npKP5ORtp9zbQjvkDJ47s6BCgaAZnnnu/cY1x342+MvZg==} engines: {node: '>=18'} @@ -4908,6 +5047,12 @@ packages: cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.24.2': + resolution: {integrity: sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openbsd-x64@0.25.1': resolution: {integrity: sha512-T3H78X2h1tszfRSf+txbt5aOp/e7TAz3ptVKu9Oyir3IAOFPGV6O9c2naym5TOriy1l0nNf6a4X5UXRZSGX/dw==} engines: {node: '>=18'} @@ -4944,6 +5089,12 @@ packages: cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.24.2': + resolution: {integrity: sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/sunos-x64@0.25.1': resolution: {integrity: sha512-2H3RUvcmULO7dIE5EWJH8eubZAI4xw54H1ilJnRNZdeo8dTADEZ21w6J22XBkXqGJbe0+wnNJtw3UXRoLJnFEg==} engines: {node: '>=18'} @@ -4980,6 +5131,12 @@ packages: cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.24.2': + resolution: {integrity: sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-arm64@0.25.1': resolution: {integrity: sha512-GE7XvrdOzrb+yVKB9KsRMq+7a2U/K5Cf/8grVFRAGJmfADr/e/ODQ134RK2/eeHqYV5eQRFxb1hY7Nr15fv1NQ==} engines: {node: '>=18'} @@ -5016,6 +5173,12 @@ packages: cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.24.2': + resolution: {integrity: sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-ia32@0.25.1': resolution: {integrity: sha512-uOxSJCIcavSiT6UnBhBzE8wy3n0hOkJsBOzy7HDAuTDE++1DJMRRVCPGisULScHL+a/ZwdXPpXD3IyFKjA7K8A==} engines: {node: '>=18'} @@ -5052,6 +5215,12 @@ packages: cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.24.2': + resolution: {integrity: sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@esbuild/win32-x64@0.25.1': resolution: {integrity: sha512-Y1EQdcfwMSeQN/ujR5VayLOJ1BHaK+ssyk0AEzPjC+t1lITgsnccPqFjb6V+LsTp/9Iov4ysfjxLaGJ9RPtkVg==} engines: {node: '>=18'} @@ -9387,6 +9556,10 @@ packages: resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} + '@sindresorhus/merge-streams@4.0.0': + resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} + engines: {node: '>=18'} + '@slack/logger@4.0.0': resolution: {integrity: sha512-Wz7QYfPAlG/DR+DfABddUZeNgoeY7d1J39OCR2jR+v7VBsB8ezulDK5szTnDDPDwLH5IWhLvXIHlCFZV7MSKgA==} engines: {node: '>= 18', npm: '>= 8.6.0'} @@ -10492,9 +10665,6 @@ packages: '@types/estree@1.0.6': resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} - '@types/estree@1.0.7': - resolution: {integrity: sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==} - '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -13195,6 +13365,11 @@ packages: engines: {node: '>=18'} hasBin: true + esbuild@0.24.2: + resolution: {integrity: sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==} + engines: {node: '>=18'} + hasBin: true + esbuild@0.25.1: resolution: {integrity: sha512-BGO5LtrGC7vxnqucAe/rmvKdJllfGaYWdyABvyMoXQlfYMb2bbRuReWR5tEGE//4LcNJj9XrkovTqNYRFZHAMQ==} engines: {node: '>=18'} @@ -13537,6 +13712,10 @@ packages: resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==} engines: {node: '>=16.17'} + execa@9.6.1: + resolution: {integrity: sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==} + engines: {node: ^18.19.0 || >=20.5.0} + exit-hook@2.2.1: resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==} engines: {node: '>=6'} @@ -13707,6 +13886,10 @@ packages: fft.js@4.0.4: resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==} + figures@6.1.0: + resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} + engines: {node: '>=18'} + file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -14277,6 +14460,10 @@ packages: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} + human-signals@8.0.1: + resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} + engines: {node: '>=18.18.0'} + humanize-duration@3.27.3: resolution: {integrity: sha512-iimHkHPfIAQ8zCDQLgn08pRqSVioyWvnGfaQ8gond2wf7Jq2jJ+24ykmnRyiz3fIldcn4oUuQXpjqKLhSVR7lw==} @@ -14622,6 +14809,10 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} @@ -16168,6 +16359,10 @@ packages: resolution: {integrity: sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + npm-run-path@6.0.0: + resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} + engines: {node: '>=18'} + num2fraction@1.2.2: resolution: {integrity: sha512-Y1wZESM7VUThYY+4W+X4ySH2maqcA+p7UR+w8VWNWVAd6lwuXXWz/w/Cz43J/dI2I+PS6wD5N+bJUF+gjWvIqg==} @@ -16508,6 +16703,10 @@ packages: resolution: {integrity: sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==} engines: {node: '>=6'} + parse-ms@4.0.0: + resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==} + engines: {node: '>=18'} + parse5-htmlparser2-tree-adapter@6.0.1: resolution: {integrity: sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==} @@ -17094,6 +17293,10 @@ packages: resolution: {integrity: sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==} engines: {node: '>=10'} + pretty-ms@9.3.0: + resolution: {integrity: sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==} + engines: {node: '>=18'} + prism-react-renderer@2.1.0: resolution: {integrity: sha512-I5cvXHjA1PVGbGm1MsWCpvBCRrYyxEri0MC7/JbfIfYfcXAxHyO5PaUjs3A8H5GW6kJcLhTHxxMaOZZpRZD2iQ==} peerDependencies: @@ -18422,6 +18625,10 @@ packages: resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} engines: {node: '>=12'} + strip-final-newline@4.0.0: + resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} + engines: {node: '>=18'} + strip-indent@3.0.0: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'} @@ -19194,6 +19401,10 @@ packages: resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==} engines: {node: '>=18'} + unicorn-magic@0.3.0: + resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} + engines: {node: '>=18'} + unified@10.1.2: resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} @@ -19853,6 +20064,10 @@ packages: resolution: {integrity: sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==} engines: {node: '>=12.20'} + yoctocolors@2.1.2: + resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} + engines: {node: '>=18'} + yup@1.6.1: resolution: {integrity: sha512-JED8pB50qbA4FOkDol0bYF/p60qSEDQqBD0/qeIrUCG1KbPBIQ776fCUNb9ldbPcSTxA69g/47XTo4TqWiuXOA==} @@ -22840,6 +23055,9 @@ snapshots: '@esbuild/aix-ppc64@0.23.0': optional: true + '@esbuild/aix-ppc64@0.24.2': + optional: true + '@esbuild/aix-ppc64@0.25.1': optional: true @@ -22858,6 +23076,9 @@ snapshots: '@esbuild/android-arm64@0.23.0': optional: true + '@esbuild/android-arm64@0.24.2': + optional: true + '@esbuild/android-arm64@0.25.1': optional: true @@ -22879,6 +23100,9 @@ snapshots: '@esbuild/android-arm@0.23.0': optional: true + '@esbuild/android-arm@0.24.2': + optional: true + '@esbuild/android-arm@0.25.1': optional: true @@ -22897,6 +23121,9 @@ snapshots: '@esbuild/android-x64@0.23.0': optional: true + '@esbuild/android-x64@0.24.2': + optional: true + '@esbuild/android-x64@0.25.1': optional: true @@ -22915,6 +23142,9 @@ snapshots: '@esbuild/darwin-arm64@0.23.0': optional: true + '@esbuild/darwin-arm64@0.24.2': + optional: true + '@esbuild/darwin-arm64@0.25.1': optional: true @@ -22933,6 +23163,9 @@ snapshots: '@esbuild/darwin-x64@0.23.0': optional: true + '@esbuild/darwin-x64@0.24.2': + optional: true + '@esbuild/darwin-x64@0.25.1': optional: true @@ -22951,6 +23184,9 @@ snapshots: '@esbuild/freebsd-arm64@0.23.0': optional: true + '@esbuild/freebsd-arm64@0.24.2': + optional: true + '@esbuild/freebsd-arm64@0.25.1': optional: true @@ -22969,6 +23205,9 @@ snapshots: '@esbuild/freebsd-x64@0.23.0': optional: true + '@esbuild/freebsd-x64@0.24.2': + optional: true + '@esbuild/freebsd-x64@0.25.1': optional: true @@ -22987,6 +23226,9 @@ snapshots: '@esbuild/linux-arm64@0.23.0': optional: true + '@esbuild/linux-arm64@0.24.2': + optional: true + '@esbuild/linux-arm64@0.25.1': optional: true @@ -23005,6 +23247,9 @@ snapshots: '@esbuild/linux-arm@0.23.0': optional: true + '@esbuild/linux-arm@0.24.2': + optional: true + '@esbuild/linux-arm@0.25.1': optional: true @@ -23023,6 +23268,9 @@ snapshots: '@esbuild/linux-ia32@0.23.0': optional: true + '@esbuild/linux-ia32@0.24.2': + optional: true + '@esbuild/linux-ia32@0.25.1': optional: true @@ -23044,6 +23292,9 @@ snapshots: '@esbuild/linux-loong64@0.23.0': optional: true + '@esbuild/linux-loong64@0.24.2': + optional: true + '@esbuild/linux-loong64@0.25.1': optional: true @@ -23062,6 +23313,9 @@ snapshots: '@esbuild/linux-mips64el@0.23.0': optional: true + '@esbuild/linux-mips64el@0.24.2': + optional: true + '@esbuild/linux-mips64el@0.25.1': optional: true @@ -23080,6 +23334,9 @@ snapshots: '@esbuild/linux-ppc64@0.23.0': optional: true + '@esbuild/linux-ppc64@0.24.2': + optional: true + '@esbuild/linux-ppc64@0.25.1': optional: true @@ -23098,6 +23355,9 @@ snapshots: '@esbuild/linux-riscv64@0.23.0': optional: true + '@esbuild/linux-riscv64@0.24.2': + optional: true + '@esbuild/linux-riscv64@0.25.1': optional: true @@ -23116,6 +23376,9 @@ snapshots: '@esbuild/linux-s390x@0.23.0': optional: true + '@esbuild/linux-s390x@0.24.2': + optional: true + '@esbuild/linux-s390x@0.25.1': optional: true @@ -23134,9 +23397,15 @@ snapshots: '@esbuild/linux-x64@0.23.0': optional: true + '@esbuild/linux-x64@0.24.2': + optional: true + '@esbuild/linux-x64@0.25.1': optional: true + '@esbuild/netbsd-arm64@0.24.2': + optional: true + '@esbuild/netbsd-arm64@0.25.1': optional: true @@ -23155,12 +23424,18 @@ snapshots: '@esbuild/netbsd-x64@0.23.0': optional: true + '@esbuild/netbsd-x64@0.24.2': + optional: true + '@esbuild/netbsd-x64@0.25.1': optional: true '@esbuild/openbsd-arm64@0.23.0': optional: true + '@esbuild/openbsd-arm64@0.24.2': + optional: true + '@esbuild/openbsd-arm64@0.25.1': optional: true @@ -23179,6 +23454,9 @@ snapshots: '@esbuild/openbsd-x64@0.23.0': optional: true + '@esbuild/openbsd-x64@0.24.2': + optional: true + '@esbuild/openbsd-x64@0.25.1': optional: true @@ -23197,6 +23475,9 @@ snapshots: '@esbuild/sunos-x64@0.23.0': optional: true + '@esbuild/sunos-x64@0.24.2': + optional: true + '@esbuild/sunos-x64@0.25.1': optional: true @@ -23215,6 +23496,9 @@ snapshots: '@esbuild/win32-arm64@0.23.0': optional: true + '@esbuild/win32-arm64@0.24.2': + optional: true + '@esbuild/win32-arm64@0.25.1': optional: true @@ -23233,6 +23517,9 @@ snapshots: '@esbuild/win32-ia32@0.23.0': optional: true + '@esbuild/win32-ia32@0.24.2': + optional: true + '@esbuild/win32-ia32@0.25.1': optional: true @@ -23251,6 +23538,9 @@ snapshots: '@esbuild/win32-x64@0.23.0': optional: true + '@esbuild/win32-x64@0.24.2': + optional: true + '@esbuild/win32-x64@0.25.1': optional: true @@ -29012,6 +29302,8 @@ snapshots: '@sindresorhus/is@4.6.0': {} + '@sindresorhus/merge-streams@4.0.0': {} + '@slack/logger@4.0.0': dependencies: '@types/node': 20.14.14 @@ -30480,14 +30772,12 @@ snapshots: '@types/estree-jsx@1.0.0': dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@types/estree@1.0.0': {} '@types/estree@1.0.6': {} - '@types/estree@1.0.7': {} - '@types/estree@1.0.8': {} '@types/eventsource@1.1.15': {} @@ -31108,6 +31398,14 @@ snapshots: optionalDependencies: vite: 5.4.21(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) + '@vitest/mocker@3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1))': + dependencies: + '@vitest/spy': 3.1.4 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + '@vitest/pretty-format@2.1.9': dependencies: tinyrainbow: 1.2.0 @@ -33637,6 +33935,34 @@ snapshots: '@esbuild/win32-ia32': 0.23.0 '@esbuild/win32-x64': 0.23.0 + esbuild@0.24.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.24.2 + '@esbuild/android-arm': 0.24.2 + '@esbuild/android-arm64': 0.24.2 + '@esbuild/android-x64': 0.24.2 + '@esbuild/darwin-arm64': 0.24.2 + '@esbuild/darwin-x64': 0.24.2 + '@esbuild/freebsd-arm64': 0.24.2 + '@esbuild/freebsd-x64': 0.24.2 + '@esbuild/linux-arm': 0.24.2 + '@esbuild/linux-arm64': 0.24.2 + '@esbuild/linux-ia32': 0.24.2 + '@esbuild/linux-loong64': 0.24.2 + '@esbuild/linux-mips64el': 0.24.2 + '@esbuild/linux-ppc64': 0.24.2 + '@esbuild/linux-riscv64': 0.24.2 + '@esbuild/linux-s390x': 0.24.2 + '@esbuild/linux-x64': 0.24.2 + '@esbuild/netbsd-arm64': 0.24.2 + '@esbuild/netbsd-x64': 0.24.2 + '@esbuild/openbsd-arm64': 0.24.2 + '@esbuild/openbsd-x64': 0.24.2 + '@esbuild/sunos-x64': 0.24.2 + '@esbuild/win32-arm64': 0.24.2 + '@esbuild/win32-ia32': 0.24.2 + '@esbuild/win32-x64': 0.24.2 + esbuild@0.25.1: optionalDependencies: '@esbuild/aix-ppc64': 0.25.1 @@ -34085,6 +34411,21 @@ snapshots: signal-exit: 4.1.0 strip-final-newline: 3.0.0 + execa@9.6.1: + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + cross-spawn: 7.0.6 + figures: 6.1.0 + get-stream: 9.0.1 + human-signals: 8.0.1 + is-plain-obj: 4.1.0 + is-stream: 4.0.1 + npm-run-path: 6.0.0 + pretty-ms: 9.3.0 + signal-exit: 4.1.0 + strip-final-newline: 4.0.0 + yoctocolors: 2.1.2 + exit-hook@2.2.1: {} expand-template@2.0.3: {} @@ -34315,6 +34656,10 @@ snapshots: fft.js@4.0.4: {} + figures@6.1.0: + dependencies: + is-unicode-supported: 2.1.0 + file-entry-cache@6.0.1: dependencies: flat-cache: 3.0.4 @@ -34908,7 +35253,7 @@ snapshots: hast-util-to-estree@2.1.0: dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.0 '@types/hast': 2.3.4 '@types/unist': 2.0.6 @@ -34942,7 +35287,7 @@ snapshots: hast-util-to-jsx-runtime@2.3.6: dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@types/hast': 3.0.4 '@types/unist': 3.0.3 comma-separated-tokens: 2.0.3 @@ -35073,6 +35418,8 @@ snapshots: human-signals@5.0.0: {} + human-signals@8.0.1: {} + humanize-duration@3.27.3: {} humanize-ms@1.2.1: @@ -35373,6 +35720,8 @@ snapshots: is-unicode-supported@0.1.0: {} + is-unicode-supported@2.1.0: {} + is-weakref@1.0.2: dependencies: call-bind: 1.0.8 @@ -36961,7 +37310,7 @@ snapshots: nano-css@5.6.2(react-dom@18.2.0(react@18.2.0))(react@18.2.0): dependencies: - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 css-tree: 1.1.3 csstype: 3.2.0 fastest-stable-stringify: 2.0.2 @@ -37254,6 +37603,11 @@ snapshots: dependencies: path-key: 4.0.0 + npm-run-path@6.0.0: + dependencies: + path-key: 4.0.0 + unicorn-magic: 0.3.0 + num2fraction@1.2.2: {} nypm@0.3.9: @@ -37673,6 +38027,8 @@ snapshots: parse-ms@2.1.0: {} + parse-ms@4.0.0: {} + parse5-htmlparser2-tree-adapter@6.0.1: dependencies: parse5: 6.0.1 @@ -37758,7 +38114,7 @@ snapshots: periscopic@3.1.0: dependencies: - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 estree-walker: 3.0.3 is-reference: 3.0.3 @@ -38228,6 +38584,10 @@ snapshots: dependencies: parse-ms: 2.1.0 + pretty-ms@9.3.0: + dependencies: + parse-ms: 4.0.0 + prism-react-renderer@2.1.0(react@18.3.1): dependencies: '@types/prismjs': 1.26.0 @@ -40152,6 +40512,8 @@ snapshots: strip-final-newline@3.0.0: {} + strip-final-newline@4.0.0: {} + strip-indent@3.0.0: dependencies: min-indent: 1.0.1 @@ -41024,6 +41386,8 @@ snapshots: unicorn-magic@0.1.0: {} + unicorn-magic@0.3.0: {} + unified@10.1.2: dependencies: '@types/unist': 2.0.6 @@ -41371,6 +41735,24 @@ snapshots: - supports-color - terser + vite-node@3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + dependencies: + cac: 6.7.14 + debug: 4.4.1(supports-color@10.0.0) + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vite-tsconfig-paths@4.0.5(typescript@5.5.4): dependencies: debug: 4.3.7(supports-color@10.0.0) @@ -41402,6 +41784,17 @@ snapshots: lightningcss: 1.29.2 terser: 5.44.1 + vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + dependencies: + esbuild: 0.21.5 + postcss: 8.5.6 + rollup: 4.36.0 + optionalDependencies: + '@types/node': 22.13.9 + fsevents: 2.3.3 + lightningcss: 1.29.2 + terser: 5.44.1 + vitest@3.1.4(@types/debug@4.1.12)(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: '@vitest/expect': 3.1.4 @@ -41412,9 +41805,9 @@ snapshots: '@vitest/spy': 3.1.4 '@vitest/utils': 3.1.4 chai: 5.2.0 - debug: 4.4.0 + debug: 4.4.1(supports-color@10.0.0) expect-type: 1.2.1 - magic-string: 0.30.17 + magic-string: 0.30.21 pathe: 2.0.3 std-env: 3.9.0 tinybench: 2.9.0 @@ -41439,6 +41832,43 @@ snapshots: - supports-color - terser + vitest@3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + dependencies: + '@vitest/expect': 3.1.4 + '@vitest/mocker': 3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1)) + '@vitest/pretty-format': 3.1.4 + '@vitest/runner': 3.1.4 + '@vitest/snapshot': 3.1.4 + '@vitest/spy': 3.1.4 + '@vitest/utils': 3.1.4 + chai: 5.2.0 + debug: 4.4.1(supports-color@10.0.0) + expect-type: 1.2.1 + magic-string: 0.30.21 + pathe: 2.0.3 + std-env: 3.9.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.13 + tinypool: 1.0.2 + tinyrainbow: 2.0.0 + vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + vite-node: 3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/debug': 4.1.12 + '@types/node': 22.13.9 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vscode-jsonrpc@8.2.0: {} vscode-languageserver-protocol@3.17.5: @@ -41552,7 +41982,7 @@ snapshots: webpack@5.88.2(@swc/core@1.3.101(@swc/helpers@0.5.15))(esbuild@0.19.11): dependencies: '@types/eslint-scope': 3.7.4 - '@types/estree': 1.0.7 + '@types/estree': 1.0.8 '@webassemblyjs/ast': 1.11.5 '@webassemblyjs/wasm-edit': 1.11.5 '@webassemblyjs/wasm-parser': 1.11.5 @@ -41761,6 +42191,8 @@ snapshots: yocto-queue@1.1.1: {} + yoctocolors@2.1.2: {} + yup@1.6.1: dependencies: property-expr: 2.0.6 From 9937823a7f174b316d567d7e8b2c77ee27e2fbbe Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 30 Jan 2026 09:15:35 +0000 Subject: [PATCH 10/22] Standardize @types/node to version 20.14.14 across monorepo (#2970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ✅ Checklist - [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [ ] The PR title follows the convention. - [ ] I ran and tested the code works --- ## Description This PR standardizes the `@types/node` dependency across the entire monorepo to version `20.14.14`. Previously, different packages were using different versions (ranging from 12.20.55 to 22.13.9), which could cause type conflicts and inconsistencies. ### Changes Made 1. **tsconfig.json** - Added `"node"` to the `types` array in `apps/webapp/tsconfig.json` to ensure Node.js types are properly recognized 2. **package.json overrides** - Added `@types/node` version override to `20.14.14` in the root `package.json` 3. **pnpm-lock.yaml** - Updated lock file to reflect the standardized version across all packages and their dependencies 4. **Fixture package.json** - Updated `packages/cli-v3/e2e/fixtures/emit-decorator-metadata/package.json` to use the standardized version This ensures consistent type definitions across the monorepo and prevents version mismatches that could lead to type errors or unexpected behavior. --- ## Testing - Verified that all package references to `@types/node` now point to version `20.14.14` - Confirmed that the lock file properly reflects the override across all transitive dependencies - Ensured TypeScript configuration includes Node.js types for proper type checking --- ## Changelog - Standardized `@types/node` to version `20.14.14` across all packages in the monorepo - Added `"node"` to TypeScript compiler types in webapp configuration - Updated all package dependencies to use the consistent version through pnpm overrides 💯 https://claude.ai/code/session_018eqp2LvvErkFSN9oK5xBh1 --- Open with Devin --------- Co-authored-by: Claude Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Eric Allam --- apps/webapp/tsconfig.json | 2 +- package.json | 1 + pnpm-lock.yaml | 125 ++++++++++++++------------------------ 3 files changed, 46 insertions(+), 82 deletions(-) diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index a10eda99c..36944e395 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -2,7 +2,7 @@ "exclude": ["./cypress", "./cypress.config.ts"], "include": ["remix.env.d.ts", "global.d.ts", "**/*.ts", "**/*.tsx"], "compilerOptions": { - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "lib": ["DOM", "DOM.Iterable", "DOM.AsyncIterable", "ES2020"], "isolatedModules": true, "esModuleInterop": true, diff --git a/package.json b/package.json index 61ec8c56f..6420ec299 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ }, "overrides": { "typescript": "5.5.4", + "@types/node": "20.14.14", "express@^4>body-parser": "1.20.3", "@remix-run/dev@2.1.0>tar-fs": "2.1.3", "testcontainers@10.28.0>tar-fs": "3.0.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b02c6cd73..ba4facc3f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,7 @@ settings: overrides: typescript: 5.5.4 + '@types/node': 20.14.14 express@^4>body-parser: 1.20.3 '@remix-run/dev@2.1.0>tar-fs': 2.1.3 testcontainers@10.28.0>tar-fs: 3.0.9 @@ -814,7 +815,7 @@ importers: version: link:../../internal-packages/testcontainers '@remix-run/dev': specifier: 2.1.0 - version: 2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@22.13.9)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4) + version: 2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@20.14.14)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4) '@remix-run/eslint-config': specifier: 2.1.0 version: 2.1.0(eslint@8.31.0)(react@18.2.0)(typescript@5.5.4) @@ -1091,7 +1092,7 @@ importers: version: 18.3.1 react-email: specifier: ^2.1.1 - version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0) + version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0) resend: specifier: ^3.2.0 version: 3.2.0 @@ -1119,7 +1120,7 @@ importers: version: 7.3.2 devDependencies: '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 rimraf: specifier: ^3.0.2 @@ -1949,7 +1950,7 @@ importers: specifier: workspace:^4.3.3 version: link:../build '@types/node': - specifier: ^20.14.14 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: '*' @@ -2211,7 +2212,7 @@ importers: specifier: ^4.0.3 version: 4.0.8 '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2293,7 +2294,7 @@ importers: specifier: workspace:* version: link:../../packages/build '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2674,7 +2675,7 @@ importers: specifier: ^4 version: 4.0.17 '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2729,7 +2730,7 @@ importers: specifier: ^4 version: 4.0.17 '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 '@types/react': specifier: ^19 @@ -2791,7 +2792,7 @@ importers: version: link:../../packages/trigger-sdk devDependencies: '@types/node': - specifier: ^20 + specifier: 20.14.14 version: 20.14.14 trigger.dev: specifier: workspace:* @@ -10782,24 +10783,9 @@ packages: '@types/node-fetch@2.6.4': resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==} - '@types/node@12.20.55': - resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} - - '@types/node@18.19.20': - resolution: {integrity: sha512-SKXZvI375jkpvAj8o+5U2518XQv76mAsixqfXiVyWyXZbVWQK25RurFovYpVIxVzul0rZoH58V/3SkEnm7s3qA==} - - '@types/node@20.11.22': - resolution: {integrity: sha512-/G+IxWxma6V3E+pqK1tSl2Fo1kl41pK1yeCyDsgkF9WlVAme4j5ISYM2zR11bgLFJGLN5sVK40T4RJNuiZbEjA==} - - '@types/node@20.12.14': - resolution: {integrity: sha512-scnD59RpYD91xngrQQLGkE+6UrHUPzeKZWhhjBSa3HSkwjbQc38+q3RoIVEwxQGRw3M+j5hpNAM+lgV3cVormg==} - '@types/node@20.14.14': resolution: {integrity: sha512-d64f00982fS9YoOgJkAMolK7MN8Iq3TDdVjchbYHdEmjth/DHowx82GnoA+tVUAN+7vxfYUgAzi+JXbKNd2SDQ==} - '@types/node@22.13.9': - resolution: {integrity: sha512-acBjXdRJ3A6Pb3tqnw9HZmyR3Fiol3aGxRCK1x3d+6CDAMjl7I649wpSd+yNURCjbOUGu9tqtLKnTGxmK6CyGw==} - '@types/nodemailer@7.0.4': resolution: {integrity: sha512-ee8fxWqOchH+Hv6MDDNNy028kwvVnLplrStm4Zf/3uHWw5zzo8FoYYeffpJtGs2wWysEumMH0ZIdMGMY1eMAow==} @@ -19386,9 +19372,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.20.0: - resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} - undici@5.29.0: resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} engines: {node: '>=14.0'} @@ -19679,7 +19662,7 @@ packages: engines: {node: ^14.18.0 || >=16.0.0} hasBin: true peerDependencies: - '@types/node': '>= 14' + '@types/node': 20.14.14 less: '*' lightningcss: ^1.21.0 sass: '*' @@ -19707,7 +19690,7 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true peerDependencies: - '@types/node': ^18.0.0 || >=20.0.0 + '@types/node': 20.14.14 less: '*' lightningcss: ^1.21.0 sass: '*' @@ -19740,7 +19723,7 @@ packages: peerDependencies: '@edge-runtime/vm': '*' '@types/debug': ^4.1.12 - '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@types/node': 20.14.14 '@vitest/browser': 3.1.4 '@vitest/ui': 3.1.4 happy-dom: '*' @@ -24104,7 +24087,7 @@ snapshots: '@kubernetes/client-node@0.20.0(bufferutil@4.0.9)': dependencies: '@types/js-yaml': 4.0.9 - '@types/node': 20.11.22 + '@types/node': 20.14.14 '@types/request': 2.48.12 '@types/ws': 8.5.10 byline: 5.0.0 @@ -24126,7 +24109,7 @@ snapshots: '@kubernetes/client-node@1.0.0(patch_hash=ba1a06f46256cdb8d6faf7167246692c0de2e7cd846a9dc0f13be0137e1c3745)(bufferutil@4.0.9)(encoding@0.1.13)': dependencies: '@types/js-yaml': 4.0.9 - '@types/node': 22.13.9 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.12 '@types/stream-buffers': 3.0.7 '@types/tar': 6.1.4 @@ -24196,7 +24179,7 @@ snapshots: '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.28.4 - '@types/node': 12.20.55 + '@types/node': 20.14.14 find-up: 4.1.0 fs-extra: 8.1.0 @@ -28814,7 +28797,7 @@ snapshots: transitivePeerDependencies: - encoding - '@remix-run/dev@2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@22.13.9)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4)': + '@remix-run/dev@2.1.0(@remix-run/serve@2.1.0(typescript@5.5.4))(@types/node@20.14.14)(bufferutil@4.0.9)(encoding@0.1.13)(lightningcss@1.29.2)(terser@5.44.1)(typescript@5.5.4)': dependencies: '@babel/core': 7.22.17 '@babel/generator': 7.24.7 @@ -28827,7 +28810,7 @@ snapshots: '@npmcli/package-json': 4.0.1 '@remix-run/server-runtime': 2.1.0(typescript@5.5.4) '@types/mdx': 2.0.5 - '@vanilla-extract/integration': 6.2.1(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + '@vanilla-extract/integration': 6.2.1(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) arg: 5.0.2 cacache: 17.1.4 chalk: 4.1.2 @@ -30878,7 +30861,7 @@ snapshots: '@types/morgan@1.9.4': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/ms@0.7.31': {} @@ -30893,7 +30876,7 @@ snapshots: '@types/node-fetch@2.6.2': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 form-data: 3.0.4 '@types/node-fetch@2.6.4': @@ -30901,28 +30884,10 @@ snapshots: '@types/node': 20.14.14 form-data: 3.0.4 - '@types/node@12.20.55': {} - - '@types/node@18.19.20': - dependencies: - undici-types: 5.26.5 - - '@types/node@20.11.22': - dependencies: - undici-types: 5.26.5 - - '@types/node@20.12.14': - dependencies: - undici-types: 5.26.5 - '@types/node@20.14.14': dependencies: undici-types: 5.26.5 - '@types/node@22.13.9': - dependencies: - undici-types: 6.20.0 - '@types/nodemailer@7.0.4': dependencies: '@aws-sdk/client-sesv2': 3.940.0 @@ -30958,7 +30923,7 @@ snapshots: '@types/pg@8.6.6': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 pg-protocol: 1.6.1 pg-types: 2.2.0 @@ -31012,7 +30977,7 @@ snapshots: '@types/readable-stream@4.0.14': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 safe-buffer: 5.1.2 '@types/regression@2.0.6': {} @@ -31072,7 +31037,7 @@ snapshots: '@types/ssh2@1.15.1': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/stream-buffers@3.0.7': dependencies: @@ -31092,7 +31057,7 @@ snapshots: '@types/tar@6.1.4': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 minipass: 4.0.0 '@types/tedious@4.0.14': @@ -31137,7 +31102,7 @@ snapshots: '@types/ws@8.5.4': dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/yauzl@2.10.3': dependencies: @@ -31316,7 +31281,7 @@ snapshots: media-query-parser: 2.0.2 outdent: 0.8.0 - '@vanilla-extract/integration@6.2.1(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1)': + '@vanilla-extract/integration@6.2.1(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1)': dependencies: '@babel/core': 7.22.17 '@babel/plugin-syntax-typescript': 7.21.4(@babel/core@7.22.17) @@ -31329,8 +31294,8 @@ snapshots: lodash: 4.17.23 mlly: 1.7.4 outdent: 0.8.0 - vite: 4.4.9(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - vite-node: 0.28.5(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + vite: 4.4.9(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) + vite-node: 0.28.5(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) transitivePeerDependencies: - '@types/node' - less @@ -32297,7 +32262,7 @@ snapshots: bun-types@1.1.17: dependencies: - '@types/node': 20.12.14 + '@types/node': 20.14.14 '@types/ws': 8.5.10 bundle-name@4.1.0: @@ -33537,7 +33502,7 @@ snapshots: dependencies: '@types/cookie': 0.4.1 '@types/cors': 2.8.17 - '@types/node': 18.19.20 + '@types/node': 20.14.14 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.4.2 @@ -37769,7 +37734,7 @@ snapshots: openai@4.33.1(encoding@0.1.13): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.4 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -37782,7 +37747,7 @@ snapshots: openai@4.68.4(encoding@0.1.13)(zod@3.25.76): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.4 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -37796,7 +37761,7 @@ snapshots: openai@4.97.0(encoding@0.1.13)(ws@8.12.0(bufferutil@4.0.9))(zod@3.25.76): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.12 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -37811,7 +37776,7 @@ snapshots: openai@4.97.0(encoding@0.1.13)(ws@8.18.3(bufferutil@4.0.9))(zod@3.25.76): dependencies: - '@types/node': 18.19.20 + '@types/node': 20.14.14 '@types/node-fetch': 2.6.12 abort-controller: 3.0.0 agentkeepalive: 4.5.0 @@ -38715,7 +38680,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 - '@types/node': 18.19.20 + '@types/node': 20.14.14 long: 5.2.3 proxy-addr@2.0.7: @@ -38967,7 +38932,7 @@ snapshots: react: 19.1.0 scheduler: 0.26.0 - react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0): + react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0): dependencies: '@babel/parser': 7.24.1 '@radix-ui/colors': 1.0.1 @@ -39004,8 +38969,8 @@ snapshots: react: 18.3.1 react-dom: 18.2.0(react@18.3.1) shelljs: 0.8.5 - socket.io: 4.7.3 - socket.io-client: 4.7.3 + socket.io: 4.7.3(bufferutil@4.0.9) + socket.io-client: 4.7.3(bufferutil@4.0.9) sonner: 1.3.1(react-dom@18.2.0(react@18.3.1))(react@18.3.1) source-map-js: 1.0.2 stacktrace-parser: 0.1.10 @@ -40152,7 +40117,7 @@ snapshots: - supports-color - utf-8-validate - socket.io-client@4.7.3: + socket.io-client@4.7.3(bufferutil@4.0.9): dependencies: '@socket.io/component-emitter': 3.1.0 debug: 4.3.7(supports-color@10.0.0) @@ -40181,7 +40146,7 @@ snapshots: transitivePeerDependencies: - supports-color - socket.io@4.7.3: + socket.io@4.7.3(bufferutil@4.0.9): dependencies: accepts: 1.3.8 base64id: 2.0.0 @@ -41376,8 +41341,6 @@ snapshots: undici-types@5.26.5: {} - undici-types@6.20.0: {} - undici@5.29.0: dependencies: '@fastify/busboy': 2.1.1 @@ -41697,7 +41660,7 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite-node@0.28.5(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + vite-node@0.28.5(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: cac: 6.7.14 debug: 4.4.1(supports-color@10.0.0) @@ -41706,7 +41669,7 @@ snapshots: picocolors: 1.1.1 source-map: 0.6.1 source-map-support: 0.5.21 - vite: 4.4.9(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + vite: 4.4.9(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) transitivePeerDependencies: - '@types/node' - less @@ -41762,13 +41725,13 @@ snapshots: - supports-color - typescript - vite@4.4.9(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): + vite@4.4.9(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: esbuild: 0.18.11 postcss: 8.5.6 rollup: 3.29.1 optionalDependencies: - '@types/node': 22.13.9 + '@types/node': 20.14.14 fsevents: 2.3.3 lightningcss: 1.29.2 terser: 5.44.1 From bc7ce781030423e570c132c1004608a93d0f23e5 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 30 Jan 2026 09:17:37 +0000 Subject: [PATCH 11/22] fix(sdk): export AnyOnStartAttemptHookFunction type (#2966) Export AnyOnStartAttemptHookFunction type to allow defining onStartAttempt hooks for individual tasks. https://claude.ai/code/session_018jgSVcFtKVyv65ktGNQFFq --- Open with Devin Co-authored-by: Claude --- .changeset/export-start-attempt-hook-type.md | 5 +++++ packages/trigger-sdk/src/v3/hooks.ts | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/export-start-attempt-hook-type.md diff --git a/.changeset/export-start-attempt-hook-type.md b/.changeset/export-start-attempt-hook-type.md new file mode 100644 index 000000000..bad7c5258 --- /dev/null +++ b/.changeset/export-start-attempt-hook-type.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Export `AnyOnStartAttemptHookFunction` type to allow defining `onStartAttempt` hooks for individual tasks. diff --git a/packages/trigger-sdk/src/v3/hooks.ts b/packages/trigger-sdk/src/v3/hooks.ts index c6811ca6e..9c4bd8eb6 100644 --- a/packages/trigger-sdk/src/v3/hooks.ts +++ b/packages/trigger-sdk/src/v3/hooks.ts @@ -17,6 +17,7 @@ import { export type { AnyOnStartHookFunction, + AnyOnStartAttemptHookFunction, TaskStartHookParams, OnStartHookFunction, AnyOnFailureHookFunction, From e6861f4fe4761b18d3f70d7bfb4cbdebe989c377 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 30 Jan 2026 09:47:56 +0000 Subject: [PATCH 12/22] Fix: run page logs keep refreshing when a run finishes (#2971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2798 When a run finished the logs UI could get stuck and so be pending and never update again. If you did a hard reload it would be correct. This happened because when we insert a log/span we ping Redis which causes a reload of the UI. However there was a race condition – the insert into ClickHouse can take a while so we were refreshing the UI too early. Then never refreshing it again. Changes - Send refresh pings every 5s to keep run page logs live - Throttle updates so the run UI is never updated more than once per second - Stop auto-reloading when a run has been completed for >= 30s - Add type inference improvements for the throttle function --- Open with Devin --- .../v3/RunStreamPresenter.server.ts | 59 ++++++++++--------- .../route.tsx | 25 +++++++- apps/webapp/app/utils/throttle.ts | 8 +-- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts index 9d54020ad..1dd4edc62 100644 --- a/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts @@ -1,12 +1,12 @@ -import { PrismaClient, prisma } from "~/db.server"; +import { type PrismaClient, prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; -import { createSSELoader } from "~/utils/sse"; +import { createSSELoader, SendFunction } from "~/utils/sse"; import { throttle } from "~/utils/throttle"; import { tracePubSub } from "~/v3/services/tracePubSub.server"; -const PING_INTERVAL = 1000; -const STREAM_TIMEOUT = 30 * 1000; // 30 seconds +const PING_INTERVAL = 5_000; +const STREAM_TIMEOUT = 30_000; export class RunStreamPresenter { #prismaClient: PrismaClient; @@ -49,36 +49,40 @@ export class RunStreamPresenter { // Subscribe to trace updates const { unsubscribe, eventEmitter } = await tracePubSub.subscribeToTrace(run.traceId); - // Store throttled send function and message listener for cleanup - let throttledSend: ReturnType | undefined; + // Only send max every 1 second + const throttledSend = throttle( + (args: { send: SendFunction; event?: string; data: string }) => { + try { + args.send({ event: args.event, data: args.data }); + } catch (error) { + if (error instanceof Error) { + if (error.name !== "TypeError") { + logger.debug("Error sending SSE in RunStreamPresenter", { + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + }); + } + } + // Abort the stream on send error + context.controller.abort("Send error"); + } + }, + 1000 + ); + let messageListener: ((event: string) => void) | undefined; return { initStream: ({ send }) => { // Create throttled send function - throttledSend = throttle((args: { event?: string; data: string }) => { - try { - send(args); - } catch (error) { - if (error instanceof Error) { - if (error.name !== "TypeError") { - logger.debug("Error sending SSE in RunStreamPresenter", { - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, - }); - } - } - // Abort the stream on send error - context.controller.abort("Send error"); - } - }, 1000); + throttledSend({ send, event: "message", data: new Date().toISOString() }); // Set up message listener for pub/sub events messageListener = (event: string) => { - throttledSend?.({ data: event }); + throttledSend({ send, event: "message", data: event }); }; eventEmitter.addListener("message", messageListener); @@ -88,7 +92,8 @@ export class RunStreamPresenter { iterator: ({ send }) => { // Send ping to keep connection alive try { - send({ event: "ping", data: new Date().toISOString() }); + // Send an actual message so the client refreshes + throttledSend({ send, event: "message", data: new Date().toISOString() }); } catch (error) { // If we can't send a ping, the connection is likely dead return false; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx index 899306eb8..1ffd128b3 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx @@ -436,6 +436,24 @@ export default function Page() { ); } +function shouldLiveReload({ + events, + maximumLiveReloadingSetting, + run, +}: { + events: TraceEvent[]; + maximumLiveReloadingSetting: number; + run: { completedAt: string | null }; +}): boolean { + // We don't live reload if there are a ton of spans/logs + if (events.length > maximumLiveReloadingSetting) return false; + + // If the run was completed a while ago, we don't need to live reload anymore + if (run.completedAt && new Date(run.completedAt).getTime() < Date.now() - 30_000) return false; + + return true; +} + function TraceView({ run, trace, @@ -453,18 +471,19 @@ function TraceView({ const { events, duration, rootSpanStatus, rootStartedAt, queuedDuration, overridesBySpanId } = trace; - const shouldLiveReload = events.length <= maximumLiveReloadingSetting; const changeToSpan = useDebounce((selectedSpan: string) => { replaceSearchParam("span", selectedSpan, { replace: true }); }, 250); + const isLiveReloading = shouldLiveReload({ events, maximumLiveReloadingSetting, run }); + const revalidator = useRevalidator(); const streamedEvents = useEventSource( v3RunStreamingPath(organization, project, environment, run), { event: "message", - disabled: !shouldLiveReload, + disabled: !isLiveReloading, } ); useEffect(() => { @@ -511,7 +530,7 @@ function TraceView({ rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined} queuedDuration={queuedDuration} environmentType={run.environment.type} - shouldLiveReload={shouldLiveReload} + shouldLiveReload={isLiveReloading} maximumLiveReloadingSetting={maximumLiveReloadingSetting} rootRun={run.rootTaskRun} parentRun={run.parentTaskRun} diff --git a/apps/webapp/app/utils/throttle.ts b/apps/webapp/app/utils/throttle.ts index a6c1a77a3..5b264ebd8 100644 --- a/apps/webapp/app/utils/throttle.ts +++ b/apps/webapp/app/utils/throttle.ts @@ -1,13 +1,13 @@ //From: https://kettanaito.com/blog/debounce-vs-throttle /** A very simple throttle. Will execute the function at the end of each period and discard any other calls during that period. */ -export function throttle( - func: (...args: any[]) => void, +export function throttle( + func: (...args: TArgs) => void, durationMs: number -): (...args: any[]) => void { +): (...args: TArgs) => void { let isPrimedToFire = false; - return (...args: any[]) => { + return (...args: TArgs) => { if (!isPrimedToFire) { isPrimedToFire = true; From b221719c09a9c48ec937bfaa68db5e49d31dbd8c Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 30 Jan 2026 10:02:30 +0000 Subject: [PATCH 13/22] chore(repo): fixed missing dependency in pnpm lockfile (#2976) --- Open with Devin --- pnpm-lock.yaml | 88 ++++---------------------------------------------- 1 file changed, 7 insertions(+), 81 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ba4facc3f..99024a016 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1092,7 +1092,7 @@ importers: version: 18.3.1 react-email: specifier: ^2.1.1 - version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0) + version: 2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0) resend: specifier: ^3.2.0 version: 3.2.0 @@ -1269,7 +1269,7 @@ importers: version: 5.5.4 vitest: specifier: 3.1.4 - version: 3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) + version: 3.1.4(@types/debug@4.1.12)(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) internal-packages/testcontainers: dependencies: @@ -31363,14 +31363,6 @@ snapshots: optionalDependencies: vite: 5.4.21(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1) - '@vitest/mocker@3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1))': - dependencies: - '@vitest/spy': 3.1.4 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - '@vitest/pretty-format@2.1.9': dependencies: tinyrainbow: 1.2.0 @@ -38932,7 +38924,7 @@ snapshots: react: 19.1.0 scheduler: 0.26.0 - react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(bufferutil@4.0.9)(eslint@8.31.0): + react-email@2.1.2(@opentelemetry/api@1.9.0)(@swc/helpers@0.5.15)(eslint@8.31.0): dependencies: '@babel/parser': 7.24.1 '@radix-ui/colors': 1.0.1 @@ -38969,8 +38961,8 @@ snapshots: react: 18.3.1 react-dom: 18.2.0(react@18.3.1) shelljs: 0.8.5 - socket.io: 4.7.3(bufferutil@4.0.9) - socket.io-client: 4.7.3(bufferutil@4.0.9) + socket.io: 4.7.3 + socket.io-client: 4.7.3 sonner: 1.3.1(react-dom@18.2.0(react@18.3.1))(react@18.3.1) source-map-js: 1.0.2 stacktrace-parser: 0.1.10 @@ -40117,7 +40109,7 @@ snapshots: - supports-color - utf-8-validate - socket.io-client@4.7.3(bufferutil@4.0.9): + socket.io-client@4.7.3: dependencies: '@socket.io/component-emitter': 3.1.0 debug: 4.3.7(supports-color@10.0.0) @@ -40146,7 +40138,7 @@ snapshots: transitivePeerDependencies: - supports-color - socket.io@4.7.3(bufferutil@4.0.9): + socket.io@4.7.3: dependencies: accepts: 1.3.8 base64id: 2.0.0 @@ -41698,24 +41690,6 @@ snapshots: - supports-color - terser - vite-node@3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): - dependencies: - cac: 6.7.14 - debug: 4.4.1(supports-color@10.0.0) - es-module-lexer: 1.7.0 - pathe: 2.0.3 - vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - transitivePeerDependencies: - - '@types/node' - - less - - lightningcss - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - vite-tsconfig-paths@4.0.5(typescript@5.5.4): dependencies: debug: 4.3.7(supports-color@10.0.0) @@ -41747,17 +41721,6 @@ snapshots: lightningcss: 1.29.2 terser: 5.44.1 - vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): - dependencies: - esbuild: 0.21.5 - postcss: 8.5.6 - rollup: 4.36.0 - optionalDependencies: - '@types/node': 22.13.9 - fsevents: 2.3.3 - lightningcss: 1.29.2 - terser: 5.44.1 - vitest@3.1.4(@types/debug@4.1.12)(@types/node@20.14.14)(lightningcss@1.29.2)(terser@5.44.1): dependencies: '@vitest/expect': 3.1.4 @@ -41795,43 +41758,6 @@ snapshots: - supports-color - terser - vitest@3.1.4(@types/debug@4.1.12)(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1): - dependencies: - '@vitest/expect': 3.1.4 - '@vitest/mocker': 3.1.4(vite@5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1)) - '@vitest/pretty-format': 3.1.4 - '@vitest/runner': 3.1.4 - '@vitest/snapshot': 3.1.4 - '@vitest/spy': 3.1.4 - '@vitest/utils': 3.1.4 - chai: 5.2.0 - debug: 4.4.1(supports-color@10.0.0) - expect-type: 1.2.1 - magic-string: 0.30.21 - pathe: 2.0.3 - std-env: 3.9.0 - tinybench: 2.9.0 - tinyexec: 0.3.2 - tinyglobby: 0.2.13 - tinypool: 1.0.2 - tinyrainbow: 2.0.0 - vite: 5.4.21(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - vite-node: 3.1.4(@types/node@22.13.9)(lightningcss@1.29.2)(terser@5.44.1) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/debug': 4.1.12 - '@types/node': 22.13.9 - transitivePeerDependencies: - - less - - lightningcss - - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - vscode-jsonrpc@8.2.0: {} vscode-languageserver-protocol@3.17.5: From 279102c17c4f21f1875b5a2bfcbd52baa5553641 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 30 Jan 2026 16:44:46 +0000 Subject: [PATCH 14/22] fix(cli): reject execute() immediately when child process is dead (#2978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - When a child process crashes and a retry (`RETRY_IMMEDIATELY`) is attempted on the same `TaskRunProcess`, `execute()` hangs forever because the IPC send is silently skipped and the attempt promise can never resolve - This caused runner pods to stay up indefinitely with no heartbeats or polls - Fix: reject the attempt promise immediately when the child is not connected, so the controller can proceed to warm start or exit ## Test plan - [x] Added `taskRunProcess.test.ts` — verifies `execute()` rejects promptly instead of hanging when the child process is dead - [x] Deploy and verify no more stuck runner pods accumulate over time --- .changeset/fix-dead-process-execute-hang.md | 5 + .../src/executions/taskRunProcess.test.ts | 121 ++++++++++++++++++ .../cli-v3/src/executions/taskRunProcess.ts | 13 ++ 3 files changed, 139 insertions(+) create mode 100644 .changeset/fix-dead-process-execute-hang.md create mode 100644 packages/cli-v3/src/executions/taskRunProcess.test.ts diff --git a/.changeset/fix-dead-process-execute-hang.md b/.changeset/fix-dead-process-execute-hang.md new file mode 100644 index 000000000..fa96e9c88 --- /dev/null +++ b/.changeset/fix-dead-process-execute-hang.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Fix runner getting stuck indefinitely when `execute()` is called on a dead child process. diff --git a/packages/cli-v3/src/executions/taskRunProcess.test.ts b/packages/cli-v3/src/executions/taskRunProcess.test.ts new file mode 100644 index 000000000..82ab19639 --- /dev/null +++ b/packages/cli-v3/src/executions/taskRunProcess.test.ts @@ -0,0 +1,121 @@ +import { TaskRunProcess, type TaskRunProcessOptions } from "./taskRunProcess.js"; +import { describe, it, expect, vi } from "vitest"; +import { UnexpectedExitError } from "@trigger.dev/core/v3/errors"; +import type { + TaskRunExecution, + TaskRunExecutionPayload, + WorkerManifest, + ServerBackgroundWorker, + MachinePresetResources, +} from "@trigger.dev/core/v3"; + +function createTaskRunProcessOptions( + overrides: Partial = {} +): TaskRunProcessOptions { + return { + workerManifest: { + runtime: "node", + workerEntryPoint: "/dev/null", + configEntryPoint: "/dev/null", + otelImportHook: {}, + } as unknown as WorkerManifest, + serverWorker: {} as unknown as ServerBackgroundWorker, + env: {}, + machineResources: { cpu: 1, memory: 1 } as MachinePresetResources, + ...overrides, + }; +} + +function createExecution(runId: string, attemptNumber: number): TaskRunExecution { + return { + run: { + id: runId, + payload: "{}", + payloadType: "application/json", + tags: [], + isTest: false, + createdAt: new Date(), + startedAt: new Date(), + maxAttempts: 3, + version: "1", + durationMs: 0, + costInCents: 0, + baseCostInCents: 0, + }, + attempt: { + number: attemptNumber, + startedAt: new Date(), + id: "deprecated", + backgroundWorkerId: "deprecated", + backgroundWorkerTaskId: "deprecated", + status: "deprecated" as any, + }, + task: { id: "test-task", filePath: "test.ts" }, + queue: { id: "queue-1", name: "test-queue" }, + environment: { id: "env-1", slug: "test", type: "DEVELOPMENT" }, + organization: { id: "org-1", slug: "test-org", name: "Test Org" }, + project: { id: "proj-1", ref: "proj_test", slug: "test", name: "Test" }, + machine: { name: "small-1x", cpu: 0.5, memory: 0.5, centsPerMs: 0 }, + } as unknown as TaskRunExecution; +} + +describe("TaskRunProcess", () => { + describe("execute() on a dead child process", () => { + it("should reject when child process has already exited and IPC send is skipped", async () => { + const proc = new TaskRunProcess(createTaskRunProcessOptions()); + + // Simulate a child process that has exited: _child exists but is not connected + const fakeChild = { + connected: false, + killed: false, + pid: 12345, + kill: vi.fn(), + on: vi.fn(), + stdout: { on: vi.fn() }, + stderr: { on: vi.fn() }, + }; + + // Set internal state to mimic a process whose child has crashed + (proc as any)._child = fakeChild; + (proc as any)._childPid = 12345; + (proc as any)._isBeingKilled = false; + + const execution = createExecution("run-1", 2); + + // This should NOT hang forever - it should reject promptly. + // + // BUG: Currently execute() creates a promise, skips the IPC send because + // _child.connected is false, then awaits the promise which will never + // resolve because the child is dead and #handleExit already ran. + // + // The Promise.race with a timeout detects the hang. + const result = await Promise.race([ + proc + .execute( + { + payload: { execution, traceContext: {}, metrics: [] }, + messageId: "run_run-1", + env: {}, + }, + true + ) + .then( + (v) => ({ type: "resolved" as const, value: v }), + (e) => ({ type: "rejected" as const, error: e }) + ), + new Promise<{ type: "hung" }>((resolve) => + setTimeout(() => resolve({ type: "hung" as const }), 2000) + ), + ]); + + // The test fails (proving the bug) if execute() hangs + expect(result.type).not.toBe("hung"); + expect(result.type).toBe("rejected"); + + if (result.type === "rejected") { + expect(result.error).toBeInstanceOf(UnexpectedExitError); + expect(result.error.stderr).toContain("not connected"); + } + }); + }); +}); diff --git a/packages/cli-v3/src/executions/taskRunProcess.ts b/packages/cli-v3/src/executions/taskRunProcess.ts index 098b0f261..1e274ba02 100644 --- a/packages/cli-v3/src/executions/taskRunProcess.ts +++ b/packages/cli-v3/src/executions/taskRunProcess.ts @@ -297,6 +297,19 @@ export class TaskRunProcess { env: params.env, isWarmStart: isWarmStart ?? this.options.isWarmStart, }); + } else { + // Child process is dead or disconnected — the IPC send was skipped so the attempt + // promise would hang forever. Reject it immediately to let the caller handle it. + this._attemptStatuses.set(key, "REJECTED"); + + // @ts-expect-error - rejecter is assigned in the promise constructor above + rejecter( + new UnexpectedExitError( + -1, + null, + "Child process is not connected, cannot execute task run" + ) + ); } const result = await promise; From 1ccb8c186fa50743d6acf4c002645a788a894e90 Mon Sep 17 00:00:00 2001 From: Mihai Popescu Date: Mon, 2 Feb 2026 02:43:39 +0200 Subject: [PATCH 15/22] changed schema id (#2983) Fixed duplicate schema id for clickhouse --- ..._serch_indexes.sql => 015_add_task_runs_v2_search_indexes.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal-packages/clickhouse/schema/{014_add_task_runs_v2_serch_indexes.sql => 015_add_task_runs_v2_search_indexes.sql} (100%) diff --git a/internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql b/internal-packages/clickhouse/schema/015_add_task_runs_v2_search_indexes.sql similarity index 100% rename from internal-packages/clickhouse/schema/014_add_task_runs_v2_serch_indexes.sql rename to internal-packages/clickhouse/schema/015_add_task_runs_v2_search_indexes.sql From b72cacc6710296c5095e9e14c2829aa82b714964 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 2 Feb 2026 20:15:06 +0000 Subject: [PATCH 16/22] feat(debounce): add maxDelay option to limit total debounce time (#2984) --- .changeset/add-debounce-maxdelay.md | 16 + .../src/engine/systems/debounceSystem.ts | 30 +- .../src/engine/tests/debounce.test.ts | 327 ++++++++++++++++++ .../run-engine/src/engine/types.ts | 1 + packages/core/src/v3/isomorphic/duration.ts | 23 +- packages/core/src/v3/schemas/api.ts | 2 + packages/core/src/v3/types/tasks.ts | 16 + .../hello-world/src/trigger/debounce.ts | 156 +++++++++ 8 files changed, 562 insertions(+), 9 deletions(-) create mode 100644 .changeset/add-debounce-maxdelay.md diff --git a/.changeset/add-debounce-maxdelay.md b/.changeset/add-debounce-maxdelay.md new file mode 100644 index 000000000..a70b95d47 --- /dev/null +++ b/.changeset/add-debounce-maxdelay.md @@ -0,0 +1,16 @@ +--- +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Add `maxDelay` option to debounce feature. This allows setting a maximum time limit for how long a debounced run can be delayed, ensuring execution happens within a specified window even with continuous triggers. + +```typescript +await myTask.trigger(payload, { + debounce: { + key: "my-key", + delay: "5s", + maxDelay: "30m", // Execute within 30 minutes regardless of continuous triggers + }, +}); +``` diff --git a/internal-packages/run-engine/src/engine/systems/debounceSystem.ts b/internal-packages/run-engine/src/engine/systems/debounceSystem.ts index af25a3155..8cd06d077 100644 --- a/internal-packages/run-engine/src/engine/systems/debounceSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/debounceSystem.ts @@ -6,7 +6,10 @@ import { type Result, } from "@internal/redis"; import { startSpan } from "@internal/tracing"; -import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic"; +import { + parseNaturalLanguageDuration, + parseNaturalLanguageDurationInMs, +} from "@trigger.dev/core/v3/isomorphic"; import { PrismaClientOrTransaction, TaskRun, Waitpoint } from "@trigger.dev/database"; import { nanoid } from "nanoid"; import { SystemResources } from "./systems.js"; @@ -17,6 +20,12 @@ export type DebounceOptions = { key: string; delay: string; mode?: "leading" | "trailing"; + /** + * Maximum total delay before the run must execute, regardless of subsequent triggers. + * This prevents indefinite delays when continuous triggers keep pushing the execution time. + * If not specified, falls back to the server's maxDebounceDurationMs config. + */ + maxDelay?: string; /** When mode: "trailing", these fields will be used to update the existing run */ updateData?: { payload: string; @@ -521,8 +530,22 @@ return 0 } // Check if max debounce duration would be exceeded + // Use per-trigger maxDelay if provided, otherwise use global config + let maxDurationMs = this.maxDebounceDurationMs; + if (debounce.maxDelay) { + const parsedMaxDelay = parseNaturalLanguageDurationInMs(debounce.maxDelay); + if (parsedMaxDelay !== undefined) { + maxDurationMs = parsedMaxDelay; + } else { + this.$.logger.warn("handleExistingRun: invalid maxDelay duration, using global config", { + maxDelay: debounce.maxDelay, + fallbackMs: this.maxDebounceDurationMs, + }); + } + } + const runCreatedAt = existingRun.createdAt; - const maxDelayUntil = new Date(runCreatedAt.getTime() + this.maxDebounceDurationMs); + const maxDelayUntil = new Date(runCreatedAt.getTime() + maxDurationMs); if (newDelayUntil > maxDelayUntil) { this.$.logger.debug("handleExistingRun: max debounce duration would be exceeded", { @@ -531,7 +554,8 @@ return 0 runCreatedAt, newDelayUntil, maxDelayUntil, - maxDebounceDurationMs: this.maxDebounceDurationMs, + maxDurationMs, + maxDelayProvided: debounce.maxDelay, }); // Clean up Redis key since this debounce window is closed await this.redis.del(redisKey); diff --git a/internal-packages/run-engine/src/engine/tests/debounce.test.ts b/internal-packages/run-engine/src/engine/tests/debounce.test.ts index 0c3d09d88..1c201c4b4 100644 --- a/internal-packages/run-engine/src/engine/tests/debounce.test.ts +++ b/internal-packages/run-engine/src/engine/tests/debounce.test.ts @@ -2170,5 +2170,332 @@ describe("RunEngine debounce", () => { } } ); + + containerTest( + "Debounce: per-trigger maxDelay overrides global maxDebounceDuration", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // Set a long global max debounce duration (1 minute) + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + debounce: { + maxDebounceDurationMs: 60_000, // 1 minute global max + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + // First trigger with a very short per-trigger maxDelay (1 second) + const run1 = await engine.trigger( + { + number: 1, + friendlyId: "run_maxwait1", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "first"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "maxwait-key", + delay: "5s", + maxDelay: "1s", // Very short per-trigger maxDelay (1 second) + }, + }, + prisma + ); + + expect(run1.friendlyId).toBe("run_maxwait1"); + + // Wait for the per-trigger maxDelay to be exceeded (1.5s > 1s) + await setTimeout(1500); + + // Second trigger should create a new run because per-trigger maxDelay exceeded + // (even though global maxDebounceDurationMs is 60 seconds) + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_maxwait2", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "second"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "maxwait-key", + delay: "5s", + maxDelay: "1s", + }, + }, + prisma + ); + + // Should be a different run because per-trigger maxDelay was exceeded + expect(run2.id).not.toBe(run1.id); + expect(run2.friendlyId).toBe("run_maxwait2"); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "Debounce: falls back to global config when maxDelay not specified", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // Set a very short global max debounce duration (1 second) + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + debounce: { + maxDebounceDurationMs: 1000, // 1 second global max + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + // First trigger without maxDelay - should use global config + const run1 = await engine.trigger( + { + number: 1, + friendlyId: "run_noglobal1", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "first"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "global-fallback-key", + delay: "5s", + // No maxDelay specified - should use global maxDebounceDurationMs + }, + }, + prisma + ); + + // Wait for global maxDebounceDurationMs to be exceeded (1.5s > 1s) + await setTimeout(1500); + + // Second trigger should create a new run because global max exceeded + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_noglobal2", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "second"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 5000), + debounce: { + key: "global-fallback-key", + delay: "5s", + }, + }, + prisma + ); + + // Should be a different run because global max exceeded + expect(run2.id).not.toBe(run1.id); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "Debounce: long maxDelay allows more debounce time than global config", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // Set a short global max debounce duration (1 second) + const engine = new RunEngine({ + prisma, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + debounce: { + maxDebounceDurationMs: 1000, // 1 second global max + }, + tracer: trace.getTracer("test", "0.0.0"), + }); + + try { + const taskIdentifier = "test-task"; + + await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); + + // First trigger with long maxDelay that overrides the short global config + const run1 = await engine.trigger( + { + number: 1, + friendlyId: "run_longmax1", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "first"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 2000), + debounce: { + key: "long-maxwait-key", + delay: "2s", + maxDelay: "60s", // Long per-trigger maxDelay overrides short global config + }, + }, + prisma + ); + + // Wait past the global maxDebounceDurationMs (1s) but within our per-trigger maxDelay (60s) + await setTimeout(1500); + + // Second trigger should return SAME run because per-trigger maxDelay is 60s + const run2 = await engine.trigger( + { + number: 2, + friendlyId: "run_longmax2", + environment: authenticatedEnvironment, + taskIdentifier, + payload: '{"data": "second"}', + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12346", + spanId: "s12346", + workerQueue: "main", + queue: "task/test-task", + isTest: false, + tags: [], + delayUntil: new Date(Date.now() + 2000), + debounce: { + key: "long-maxwait-key", + delay: "2s", + maxDelay: "60s", + }, + }, + prisma + ); + + // Should be the SAME run because per-trigger maxDelay allows it + expect(run2.id).toBe(run1.id); + } finally { + await engine.quit(); + } + } + ); }); diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index ee5176c2f..2adc63415 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -180,6 +180,7 @@ export type TriggerParams = { key: string; delay: string; mode?: "leading" | "trailing"; + maxDelay?: string; }; /** * Called when a run is debounced (existing delayed run found with triggerAndWait). diff --git a/packages/core/src/v3/isomorphic/duration.ts b/packages/core/src/v3/isomorphic/duration.ts index b4c5cd20d..9315f7968 100644 --- a/packages/core/src/v3/isomorphic/duration.ts +++ b/packages/core/src/v3/isomorphic/duration.ts @@ -1,5 +1,15 @@ -export function parseNaturalLanguageDuration(duration: string): Date | undefined { - // Handle Code scanning alert #44 (https://github.com/triggerdotdev/trigger.dev/security/code-scanning/44) by limiting the length of the input string +/** + * Parses a natural language duration string into milliseconds. + * + * @param duration - Duration string like "1s", "5m", "2h", "1d", "1w" + * @returns The duration in milliseconds, or undefined if invalid + * + * @example + * parseNaturalLanguageDurationInMs("30m") // 1800000 + * parseNaturalLanguageDurationInMs("2h") // 7200000 + */ +export function parseNaturalLanguageDurationInMs(duration: string): number | undefined { + // Handle Code scanning alert #44 by limiting the length of the input string if (duration.length > 100) { return undefined; } @@ -60,11 +70,12 @@ export function parseNaturalLanguageDuration(duration: string): Date | undefined } } - if (hasMatch) { - return new Date(Date.now() + totalMilliseconds); - } + return hasMatch ? totalMilliseconds : undefined; +} - return undefined; +export function parseNaturalLanguageDuration(duration: string): Date | undefined { + const ms = parseNaturalLanguageDurationInMs(duration); + return ms !== undefined ? new Date(Date.now() + ms) : undefined; } export function safeParseNaturalLanguageDuration(duration: string): Date | undefined { diff --git a/packages/core/src/v3/schemas/api.ts b/packages/core/src/v3/schemas/api.ts index 9080a7f59..0291d2a05 100644 --- a/packages/core/src/v3/schemas/api.ts +++ b/packages/core/src/v3/schemas/api.ts @@ -218,6 +218,7 @@ export const TriggerTaskRequestBody = z.object({ key: z.string().max(512), delay: z.string(), mode: z.enum(["leading", "trailing"]).optional(), + maxDelay: z.string().optional(), }) .optional(), }) @@ -275,6 +276,7 @@ export const BatchTriggerTaskItem = z.object({ key: z.string().max(512), delay: z.string(), mode: z.enum(["leading", "trailing"]).optional(), + maxDelay: z.string().optional(), }) .optional(), }) diff --git a/packages/core/src/v3/types/tasks.ts b/packages/core/src/v3/types/tasks.ts index f463b20f4..3b8b2e9ec 100644 --- a/packages/core/src/v3/types/tasks.ts +++ b/packages/core/src/v3/types/tasks.ts @@ -945,6 +945,22 @@ export type TriggerOptions = { * @default "leading" */ mode?: "leading" | "trailing"; + /** + * Maximum total delay before the run must execute, regardless of subsequent triggers. + * This prevents indefinite delays when continuous triggers keep pushing the execution time. + * + * When specified, if a new trigger would push the execution time beyond this limit + * (measured from the first trigger), the current debounced run will be allowed to execute + * and a new run will be created for subsequent triggers. + * + * If not specified, falls back to the server's default maximum (typically 1 hour). + * + * Supported formats: `{number}s` (seconds), `{number}m` (minutes), `{number}h` (hours), + * `{number}d` (days), `{number}w` (weeks). + * + * @example "30m", "2h", "1d" + */ + maxDelay?: string; }; }; diff --git a/references/hello-world/src/trigger/debounce.ts b/references/hello-world/src/trigger/debounce.ts index e396714eb..f0a7e8b63 100644 --- a/references/hello-world/src/trigger/debounce.ts +++ b/references/hello-world/src/trigger/debounce.ts @@ -1056,3 +1056,159 @@ export const demonstrateTrailingWithMetadata = task({ }; }, }); + +/** + * Example 12: Debounce with maxDelay + * + * The maxDelay option limits how long a debounced run can be delayed. + * Even if triggers keep coming, the run will eventually execute after maxDelay + * from the first trigger. This is useful for scenarios where you want to + * debounce but also guarantee execution within a certain time window. + * + * Use case: Summarizing AI conversation threads that need to stay relatively + * up to date. You want to debounce as messages come in, but also guarantee + * the summary runs at least every 30 minutes. + */ +export const processConversationSummary = task({ + id: "process-conversation-summary", + run: async (payload: { conversationId: string; messageCount: number }) => { + logger.info("Generating conversation summary", { payload }); + + // Simulate AI summarization work + await wait.for({ seconds: 2 }); + + logger.info("Conversation summary generated", { + conversationId: payload.conversationId, + messageCount: payload.messageCount, + }); + + return { + summarized: true, + conversationId: payload.conversationId, + messageCount: payload.messageCount, + summarizedAt: new Date().toISOString(), + }; + }, +}); + +/** + * Demonstrates maxDelay in action. + * + * This simulates a chat application where messages come in continuously. + * With just debounce, the summary task would keep getting delayed forever. + * With maxDelay: "30s", the summary will run at most 30 seconds after the first trigger, + * even if messages keep coming. + * + * Run this task and observe: + * - Messages trigger the summary task with debounce + * - Each trigger extends the delay by 5s + * - But maxDelay ensures execution happens within 30s of the first trigger + */ +export const simulateChatWithMaxWait = task({ + id: "simulate-chat-with-max-wait", + run: async (payload: { conversationId?: string; simulateDelay?: number }) => { + const conversationId = payload.conversationId ?? "conv-123"; + const delayBetweenMessages = payload.simulateDelay ?? 3000; // 3 seconds + + logger.info("Starting chat simulation with maxDelay", { + conversationId, + delayBetweenMessages, + }); + + logger.info( + "Debounce delay is 5s, maxDelay is 30s. Messages arrive every 3s, so debounce would normally keep extending. But maxDelay ensures execution within 30s." + ); + + const handles: string[] = []; + + // Simulate 15 messages over ~45 seconds + // Without maxDelay, the task would never run because each trigger resets the 5s delay + // With maxDelay: "30s", the task will run after 30 seconds from the first trigger + for (let i = 1; i <= 15; i++) { + logger.info(`Message ${i}/15 received`, { messageNumber: i }); + + const handle = await processConversationSummary.trigger( + { + conversationId, + messageCount: i, + }, + { + debounce: { + key: `conversation-${conversationId}`, + delay: "5s", + mode: "trailing", // Use latest message count + maxDelay: "30s", // Ensure execution within 30s of first trigger + }, + } + ); + + handles.push(handle.id); + logger.info(`Message ${i} triggered, run ID: ${handle.id}`, { + messageNumber: i, + runId: handle.id, + }); + + // Wait between messages (simulating real chat) + if (i < 15) { + await new Promise((resolve) => setTimeout(resolve, delayBetweenMessages)); + } + } + + const uniqueHandles = [...new Set(handles)]; + + logger.info("Chat simulation complete", { + totalMessages: 15, + uniqueRuns: uniqueHandles.length, + note: + "With maxDelay, runs should have been created periodically despite continuous triggering", + }); + + return { + conversationId, + totalMessages: 15, + uniqueRunsCreated: uniqueHandles.length, + runIds: uniqueHandles, + message: + "Due to maxDelay: '30s', the summary task runs periodically even with continuous triggers", + }; + }, +}); + +/** + * A simpler maxDelay example showing the basic usage pattern. + * + * This is the recommended pattern for using maxDelay: + * - delay: How long to wait after each trigger before executing + * - maxDelay: Maximum total wait time from the first trigger + */ +export const onNewMessage = task({ + id: "on-new-message", + run: async (payload: { conversationId: string; message: string }) => { + logger.info("New message received", { + conversationId: payload.conversationId, + messagePreview: payload.message.substring(0, 50), + }); + + // Trigger summarization with debounce and maxDelay + const handle = await processConversationSummary.trigger( + { + conversationId: payload.conversationId, + messageCount: 1, // In real code, you'd track actual count + }, + { + debounce: { + key: `summary-${payload.conversationId}`, + delay: "10s", // Wait 10s after last message before summarizing + mode: "trailing", // Use latest state + maxDelay: "5m", // But always summarize within 5 minutes + }, + } + ); + + logger.info("Summary task triggered (debounced with maxDelay)", { + runId: handle.id, + }); + + return { summaryRunId: handle.id }; + }, +}); From 8e0034484cf42b2f6dfd43cceb621b162b5f16e4 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 4 Feb 2026 14:39:47 +0100 Subject: [PATCH 17/22] feat(supervisor): project-based scheduling affinity for image cache locality (#2995) Adds optional pod affinity so pods from the same project prefer scheduling on the same node. This can help improve image cache hit rates; subsequent pods benefit from already-pulled image layers, reducing startup time. Complements the built-in ImageLocality scheduler plugin by helping during burst scheduling scenarios. Pod affinity sees scheduled pods immediately, while ImageLocality only sees images after they're fully pulled. Configuration: - `KUBERNETES_PROJECT_AFFINITY_ENABLED` - Enable/disable (default: false) - `KUBERNETES_PROJECT_AFFINITY_WEIGHT` - Scheduler weight 1-100 (default: 50) - `KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY` - Topology key (default: kubernetes.io/hostname) Uses soft (preferred) affinity so pods always schedule even if preferred node is full. --- Open with Devin --- apps/supervisor/src/env.ts | 5 + .../src/workloadManager/kubernetes.ts | 98 +++++++++++++------ 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 9ef0cff25..faf34bcd0 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -112,6 +112,11 @@ const Env = z.object({ KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods KUBERNETES_LARGE_MACHINE_POOL_LABEL: z.string().optional(), // if set, large-* presets affinity for machinepool= + // Project affinity settings - pods from the same project prefer the same node + KUBERNETES_PROJECT_AFFINITY_ENABLED: BoolEnv.default(false), + KUBERNETES_PROJECT_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(50), + KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z.string().trim().min(1).default("kubernetes.io/hostname"), + // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"), diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index a725971a8..16c5eff9d 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -120,7 +120,7 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, spec: { ...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags), - affinity: this.#getNodeAffinity(opts.machine), + affinity: this.#getAffinity(opts.machine, opts.projectId), terminationGracePeriodSeconds: 60 * 60, containers: [ { @@ -390,7 +390,21 @@ export class KubernetesWorkloadManager implements WorkloadManager { return preset.name.startsWith("large-"); } - #getNodeAffinity(preset: MachinePreset): k8s.V1Affinity | undefined { + #getAffinity(preset: MachinePreset, projectId: string): k8s.V1Affinity | undefined { + const nodeAffinity = this.#getNodeAffinityRules(preset); + const podAffinity = this.#getProjectPodAffinity(projectId); + + if (!nodeAffinity && !podAffinity) { + return undefined; + } + + return { + ...(nodeAffinity && { nodeAffinity }), + ...(podAffinity && { podAffinity }), + }; + } + + #getNodeAffinityRules(preset: MachinePreset): k8s.V1NodeAffinity | undefined { if (!env.KUBERNETES_LARGE_MACHINE_POOL_LABEL) { return undefined; } @@ -398,42 +412,64 @@ export class KubernetesWorkloadManager implements WorkloadManager { if (this.#isLargeMachine(preset)) { // soft preference for the large-machine pool, falls back to standard if unavailable return { - nodeAffinity: { - preferredDuringSchedulingIgnoredDuringExecution: [ - { - weight: 100, - preference: { - matchExpressions: [ - { - key: "node.cluster.x-k8s.io/machinepool", - operator: "In", - values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], - }, - ], - }, + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 100, + preference: { + matchExpressions: [ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "In", + values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + }, + ], }, - ], - }, + }, + ], }; } // not schedulable in the large-machine pool return { - nodeAffinity: { - requiredDuringSchedulingIgnoredDuringExecution: { - nodeSelectorTerms: [ - { - matchExpressions: [ - { - key: "node.cluster.x-k8s.io/machinepool", - operator: "NotIn", - values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], - }, - ], - }, - ], - }, + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { + matchExpressions: [ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "NotIn", + values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + }, + ], + }, + ], }, }; } + + #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined { + if (!env.KUBERNETES_PROJECT_AFFINITY_ENABLED) { + return undefined; + } + + return { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: env.KUBERNETES_PROJECT_AFFINITY_WEIGHT, + podAffinityTerm: { + labelSelector: { + matchExpressions: [ + { + key: "project", + operator: "In", + values: [projectId], + }, + ], + }, + topologyKey: env.KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY, + }, + }, + ], + }; + } } From 7781e2aad1e629b1df31b93f709716fc2bdef20f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 4 Feb 2026 06:05:59 -0800 Subject: [PATCH 18/22] docs: usage function examples were missing the imports (#2830) Closes #2828 --- docs/run-usage.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/run-usage.mdx b/docs/run-usage.mdx index 0b9163db1..4c0042331 100644 --- a/docs/run-usage.mdx +++ b/docs/run-usage.mdx @@ -8,6 +8,8 @@ description: "Get compute duration and cost from inside a run, or for a specific You can get the cost and duration of the current including retries of the same run. ```ts +import { task, usage, wait } from "@trigger.dev/sdk"; + export const heavyTask = task({ id: "heavy-task", machine: { @@ -87,6 +89,8 @@ console.log("Total cost", totalCost); You can also wrap code with `usage.measure` to get the cost and duration of that block of code: ```ts +import { usage, logger } from "@trigger.dev/sdk"; + // Inside a task run function, or inside a function that's called from there. const { result, compute } = await usage.measure(async () => { //...Do something for 1 second From e0179130214801dfc20c6be7dc03229f1e46bf03 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:29:01 -0500 Subject: [PATCH 19/22] docs(self-hosting): added graphile worker troubleshooting to docs (#2883) Add troubleshooting documentation for graphile worker schema migration failures and PostgreSQL SSL certificate issues that prevent worker initialization. --- docs/self-hosting/docker.mdx | 2 ++ docs/self-hosting/kubernetes.mdx | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx index dbbc3578e..add4e06ba 100644 --- a/docs/self-hosting/docker.mdx +++ b/docs/self-hosting/docker.mdx @@ -354,6 +354,8 @@ TRIGGER_IMAGE_TAG=v4.0.0 docker compose logs -f webapp ``` +- **Deploy fails with `ERROR: schema "graphile_worker" does not exist`.** This error occurs when Graphile Worker migrations fail to run during webapp startup. Check the webapp logs for certificate-related errors like `self-signed certificate in certificate chain`. This is often caused by PostgreSQL SSL certificate issues when using an external PostgreSQL instance with SSL enabled. Ensure that both the webapp and supervisor containers have access to the same CA certificate used by your PostgreSQL instance. You can configure this by mounting the certificate file and setting the `NODE_EXTRA_CA_CERTS` environment variable to point to the certificate path. Once the certificate issue is resolved, the migrations will complete and create the required `graphile_worker` schema. + ## CLI usage This section highlights some of the CLI commands and options that are useful when self-hosting. Please check the [CLI reference](/cli-introduction) for more in-depth documentation. diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx index 4506d6da9..eba66f4ed 100644 --- a/docs/self-hosting/kubernetes.mdx +++ b/docs/self-hosting/kubernetes.mdx @@ -555,6 +555,7 @@ kubectl delete namespace trigger - **Deploy fails**: Verify registry access and authentication - **Pods stuck pending**: Describe the pod and check the events - **Worker token issues**: Check webapp and supervisor logs for errors +- **Deploy fails with `ERROR: schema "graphile_worker" does not exist`**: See the [Docker troubleshooting](/self-hosting/docker#troubleshooting) section for details on resolving PostgreSQL SSL certificate issues that prevent Graphile Worker migrations. See the [Docker troubleshooting](/self-hosting/docker#troubleshooting) section for more information. From 104f720f6ff0b27403a0c01c3cbbf85fc4e21f20 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:30:51 -0500 Subject: [PATCH 20/22] docs(troubleshooting): add COULD_NOT_FIND_EXECUTOR error and IPv4 support (#2950) Document COULD_NOT_FIND_EXECUTOR error with dynamic imports and IPv4 database connection limitation in troubleshooting guide. --- docs/troubleshooting.mdx | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index b6254f62d..c5040e592 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -151,6 +151,10 @@ Your code is deployed separately from the rest of your app(s) so you need to mak Prisma uses code generation to create the client from your schema file. This means you need to add a bit of config so we can generate this file before your tasks run: [Read the guide](/config/extensions/prismaExtension). +### Database connection requires IPv4 + +Trigger.dev currently only supports IPv4 database connections. If your database provider only provides an IPv6 connection string, you'll need to use an IPv4 address instead. [Upvote IPv6 support](https://triggerdev.featurebase.app/p/support-ipv6-database-connections). + ### `Parallel waits are not supported` In the current version, you can't perform more that one "wait" in parallel. @@ -171,6 +175,36 @@ The most common situation this happens is if you're using `Promise.all` around s Make sure that you always use `await` when you call `trigger`, `triggerAndWait`, `batchTrigger`, and `batchTriggerAndWait`. If you don't then it's likely the task(s) won't be triggered because the calling function process can be terminated before the networks calls are sent. +### `COULD_NOT_FIND_EXECUTOR` + +If you see a `COULD_NOT_FIND_EXECUTOR` error when triggering a task, it may be caused by dynamically importing the child task. When tasks are dynamically imported, the executor may not be properly registered. + +Use a top-level import instead: + +```ts +import { myChildTask } from "~/trigger/my-child-task"; + +export const myTask = task({ + id: "my-task", + run: async (payload: string) => { + await myChildTask.trigger({ payload: "data" }); + }, +}); +``` + +Alternatively, use `tasks.trigger()` or `batch.triggerAndWait()` without importing the task: + +```ts +import { batch } from "@trigger.dev/sdk"; + +export const myTask = task({ + id: "my-task", + run: async (payload: string) => { + await batch.triggerAndWait([{ id: "my-child-task", payload: "data" }]); + }, +}); +``` + ### Rate limit exceeded From 6a45f5623b89ed107176c5f8e461a4c2f2dde54d Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:33:05 -0500 Subject: [PATCH 21/22] docs: multi-tenant applications and concurrency limits (#2961) Adds an example of multi-tenant applications as alternative to project/limit increase, and more info about queue times and concurrency --- docs/deploy-environment-variables.mdx | 53 ++++++++++++++++++++++++++- docs/limits.mdx | 8 ++++ docs/troubleshooting.mdx | 10 +++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/docs/deploy-environment-variables.mdx b/docs/deploy-environment-variables.mdx index e123f3417..e7b649640 100644 --- a/docs/deploy-environment-variables.mdx +++ b/docs/deploy-environment-variables.mdx @@ -360,4 +360,55 @@ This will read your .env.production file using dotenvx and sync the variables to - Trigger.dev does not automatically detect .env.production or dotenvx files - You can paste them manually into the dashboard -- Or sync them automatically using a build extension \ No newline at end of file +- Or sync them automatically using a build extension + +## Multi-tenant applications + +If you're building a multi-tenant application where each tenant needs different environment variables (like tenant-specific API keys or database credentials), you don't need a separate project for each tenant. Instead, use a single project and load tenant-specific secrets at runtime. + + + This is different from [syncing environment variables at deploy time](#sync-env-vars-from-another-service). + Here, secrets are loaded dynamically during task execution, not synced to Trigger.dev's environment variables. + + +### Recommended approach + +Use a secrets service (Infisical, AWS Secrets Manager, HashiCorp Vault, etc.) to store tenant-specific secrets, then retrieve them at the start of each task run based on the tenant identifier in your payload or context. + +**Important:** Never pass secrets in the task payload, as payloads are logged and visible in the dashboard. + +### Example implementation + +```ts +import { task } from "@trigger.dev/sdk"; +import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager"; + +export const processTenantData = task({ + id: "process-tenant-data", + run: async (payload: { tenantId: string; data: unknown }) => { + // Retrieve tenant-specific secret at runtime + const client = new SecretsManagerClient({ region: "us-east-1" }); + const response = await client.send( + new GetSecretValueCommand({ + SecretId: `tenants/${payload.tenantId}/supabase-key`, + }) + ); + + const supabaseKey = JSON.parse(response.SecretString!).SUPABASE_SERVICE_KEY; + + // Your task logic using the tenant-specific secret + // ... + }, +}); +``` + +You can use any secrets service - see the [sync env vars section](#sync-env-vars-from-another-service) for an example with Infisical. + +### Benefits + +- **Single codebase** - Deploy once, works for all tenants +- **Secure** - Secrets never appear in payloads or logs +- **Scalable** - No project limit constraints +- **Flexible** - Easy to add new tenants without redeploying + +This approach allows you to support unlimited tenants with a single Trigger.dev project, avoiding the [project limit](/limits#projects) while maintaining security and separation of tenant data. \ No newline at end of file diff --git a/docs/limits.mdx b/docs/limits.mdx index b4df2001a..45da4e89a 100644 --- a/docs/limits.mdx +++ b/docs/limits.mdx @@ -55,6 +55,14 @@ If you add them [dynamically using code](/management/schedules/create) make sure If you're creating schedules for your user you will definitely need to request more schedules from us. +## Projects + +| Pricing tier | Limit | +| :----------- | :----------------- | +| All tiers | 10 per organization | + +Each project receives its own concurrency allocation. If you need to support multiple tenants with the same codebase but different environment variables, see the [Multi-tenant applications](/deploy-environment-variables#multi-tenant-applications) section for a recommended workaround. + ## Preview branches | Pricing tier | Limit | diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index c5040e592..7a003194f 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -211,6 +211,16 @@ export const myTask = task({ View the [rate limits](/limits) page for more information. +### Runs waiting in queue due to concurrency limits + +If runs are staying in the `QUEUED` state for extended periods, check your concurrency usage in the dashboard. Review how many runs are `EXECUTING` or `DEQUEUED` (these count against limits) and check if any runs are stuck in `EXECUTING` state, as they may be blocking new runs. + +**Solutions:** + +- **Increase concurrency limits** - If you're on a paid plan, increase your environment concurrency limit via the dashboard +- **Review queue concurrency limits** - Check if individual queues have restrictive `concurrencyLimit` settings +- **Check for stuck runs** - See if stalled runs are blocking new executions + ### `Crypto is not defined` This can happen in different situations, for example when using plain strings as idempotency keys. Support for `Crypto` without a special flag was added in Node `v19.0.0`. You will have to upgrade Node - we recommend even-numbered major releases, e.g. `v20` or `v22`. Alternatively, you can switch from plain strings to the `idempotencyKeys.create` SDK function. [Read the guide](/idempotency). From c0595700f8506cf92b91816338f3f174297eb1d2 Mon Sep 17 00:00:00 2001 From: Iss <74388823+isshaddad@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:37:35 -0500 Subject: [PATCH 22/22] docs: clarify .env.local loading and idempotency key reset scope (#2996) Document that .env.local is automatically loaded during dev, and clarify that backend-triggered idempotency keys should be reset with global scope --- docs/deploy-environment-variables.mdx | 12 ++++++++++++ docs/idempotency.mdx | 8 ++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/deploy-environment-variables.mdx b/docs/deploy-environment-variables.mdx index e7b649640..a4c3b75da 100644 --- a/docs/deploy-environment-variables.mdx +++ b/docs/deploy-environment-variables.mdx @@ -66,6 +66,18 @@ You can edit an environment variable's values. You cannot edit the key name, you +## Local development + +When running `npx trigger.dev dev`, the CLI automatically loads environment variables from these files in order (later files override any duplicate keys from earlier ones): + +- `.env` +- `.env.development` +- `.env.local` +- `.env.development.local` +- `dev.vars` + +These variables are available to your tasks via `process.env`. You don't need to use the `--env-file` flag for this automatic loading. + ## In your code You can use our SDK to get and manipulate environment variables. You can also easily sync environment variables from another service into Trigger.dev. diff --git a/docs/idempotency.mdx b/docs/idempotency.mdx index 9ed3933d5..034246eaf 100644 --- a/docs/idempotency.mdx +++ b/docs/idempotency.mdx @@ -428,18 +428,22 @@ export const parentTask = task({ }); ``` -When resetting from outside a task (e.g., from your backend code), you must provide the `parentRunId`: +When resetting from outside a task, you must provide the `parentRunId` if the key was created within a task context: ```ts import { idempotencyKeys } from "@trigger.dev/sdk"; -// From your backend code - you need to know the parent run ID +// If the key was created within a task, you need the parent run ID await idempotencyKeys.reset("my-task", "my-key", { scope: "run", parentRunId: "run_abc123" }); ``` + +If you triggered the task from backend code, all scopes behave as global (see [Triggering from backend code](#triggering-from-backend-code)). Use `scope: "global"` when resetting. + + ### Resetting attempt-scoped keys Keys created with `"attempt"` scope include both the parent run ID and attempt number. When resetting from outside a task, you must provide both: