github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/compute-workload-manager

Browse Source
This commit is contained in:
nicktrn
2026-02-20 17:47:05 +00:00
parent 4d603adf67 676525279a
commit 9466a47a51
113 changed files with 4802 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/add-debounce-maxdelay.md
-16
View File
@@ -1,16 +0,0 @@
---
"@trigger.dev/core": patch
"@trigger.dev/sdk": patch
---
Add `maxDelay` option to debounce feature. This allows setting a maximum time limit for how long a debounced run can be delayed, ensuring execution happens within a specified window even with continuous triggers.
```typescript
await myTask.trigger(payload, {
debounce: {
key: "my-key",
delay: "5s",
maxDelay: "30m", // Execute within 30 minutes regardless of continuous triggers
},
});
```
.changeset/afraid-gorillas-jump.md
-34
View File
@@ -1,34 +0,0 @@
---
"@trigger.dev/sdk": minor
---
Added `query.execute()` which lets you query your Trigger.dev data using TRQL (Trigger Query Language) and returns results as typed JSON rows or CSV. It supports configurable scope (environment, project, or organization), time filtering via `period` or `from`/`to` ranges, and a `format` option for JSON or CSV output.
```typescript
import { query } from "@trigger.dev/sdk";
import type { QueryTable } from "@trigger.dev/sdk";
// Basic untyped query
const result = await query.execute("SELECT run_id, status FROM runs LIMIT 10");
// Type-safe query using QueryTable to pick specific columns
const typedResult = await query.execute<QueryTable<"runs", "run_id" | "status" | "triggered_at">>(
"SELECT run_id, status, triggered_at FROM runs LIMIT 10"
);
typedResult.results.forEach(row => {
console.log(row.run_id, row.status); // Fully typed
});
// Aggregation query with inline types
const stats = await query.execute<{ status: string; count: number }>(
"SELECT status, COUNT(*) as count FROM runs GROUP BY status",
{ scope: "project", period: "30d" }
);
// CSV export
const csv = await query.execute(
"SELECT run_id, status FROM runs",
{ format: "csv", period: "7d" }
);
console.log(csv.results); // Raw CSV string
```
.changeset/calm-hooks-wait.md
-5
View File
@@ -1,5 +0,0 @@
---
"@trigger.dev/react-hooks": patch
---
Fix `onComplete` callback firing prematurely when the realtime stream disconnects before the run finishes.
.changeset/consistent-stream-targets.md
-5
View File
@@ -1,5 +0,0 @@
---
"@trigger.dev/sdk": patch
---
Aligned the SDK's `getRunIdForOptions` logic with the Core package to handle semantic targets (`root`, `parent`) in root tasks.
.changeset/export-start-attempt-hook-type.md
-5
View File
@@ -1,5 +0,0 @@
---
"@trigger.dev/sdk": patch
---
Export `AnyOnStartAttemptHookFunction` type to allow defining `onStartAttempt` hooks for individual tasks.
.changeset/fix-dead-process-execute-hang.md
-5
View File
@@ -1,5 +0,0 @@
---
"trigger.dev": patch
---
Fix runner getting stuck indefinitely when `execute()` is called on a dead child process.
.changeset/mcp-wait-timeout.md
-5
View File
@@ -1,5 +0,0 @@
---
"trigger.dev": patch
---
Add optional `timeoutInSeconds` parameter to the `wait_for_run_to_complete` MCP tool. Defaults to 60 seconds. If the run doesn't complete within the timeout, the current state of the run is returned instead of waiting indefinitely.
.changeset/tricky-suits-design.md
-7
View File
@@ -1,7 +0,0 @@
---
"@trigger.dev/sdk": patch
"trigger.dev": patch
"@trigger.dev/core": patch
---
Fixed a minor issue in the deployment command on distinguishing between local builds for the cloud vs local builds for self-hosting setups.
.changeset/vendor-superjson-esm-fix.md
-7
View File
@@ -1,7 +0,0 @@
---
"@trigger.dev/core": patch
---
fix: vendor superjson to fix ESM/CJS compatibility
Bundle superjson during build to avoid `ERR_REQUIRE_ESM` errors on Node.js versions that don't support `require(ESM)` by default (< 22.12.0) and AWS Lambda which intentionally disables it.
.changeset/vercel-integration.md
-5
View File
@@ -1,5 +0,0 @@
---
"@trigger.dev/core": patch
---
Add Vercel integration support to API schemas: `commitSHA` and `integrationDeployments` on deployment responses, and `source` field for environment variable imports.
apps/webapp/app/components/code/QueryResultsChart.tsx
+72 -5
View File
@@ -1,6 +1,9 @@
import type { OutputColumnMetadata } from "@internal/clickhouse";
import type { ColumnFormatType, OutputColumnMetadata } from "@internal/clickhouse";
import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
import { BarChart3, LineChart } from "lucide-react";
import { memo, useMemo } from "react";
import { createValueFormatter } from "~/utils/columnFormat";
import { formatCurrencyAccurate } from "~/utils/numberFormatter";
import type { ChartConfig } from "~/components/primitives/charts/Chart";
import { Chart } from "~/components/primitives/charts/ChartCompound";
import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -855,8 +858,24 @@ export const QueryResultsChart = memo(function QueryResultsChart({
};
}, [isDateBased, timeGranularity]);
// Create dynamic Y-axis formatter based on data range
const yAxisFormatter = useMemo(() => createYAxisFormatter(data, series), [data, series]);
// Resolve the Y-axis column format for formatting
const yAxisFormat = useMemo(() => {
if (yAxisColumns.length === 0) return undefined;
const col = columns.find((c) => c.name === yAxisColumns[0]);
return (col?.format ?? col?.customRenderType) as ColumnFormatType | undefined;
}, [yAxisColumns, columns]);
// Create dynamic Y-axis formatter based on data range and format
const yAxisFormatter = useMemo(
() => createYAxisFormatter(data, series, yAxisFormat),
[data, series, yAxisFormat]
);
// Create value formatter for tooltips and legend based on column format
const tooltipValueFormatter = useMemo(
() => createValueFormatter(yAxisFormat),
[yAxisFormat]
);
// Check if the group-by column has a runStatus customRenderType
const groupByIsRunStatus = useMemo(() => {
@@ -1081,6 +1100,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
showLegend={showLegend}
maxLegendItems={fullLegend ? Infinity : 5}
legendAggregation={config.aggregation}
legendValueFormatter={tooltipValueFormatter}
minHeight="300px"
fillContainer
onViewAllLegendItems={onViewAllLegendItems}
@@ -1093,6 +1113,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
yAxisProps={yAxisProps}
stackId={stacked ? "stack" : undefined}
tooltipLabelFormatter={tooltipLabelFormatter}
tooltipValueFormatter={tooltipValueFormatter}
/>
</Chart.Root>
);
@@ -1110,6 +1131,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
showLegend={showLegend}
maxLegendItems={fullLegend ? Infinity : 5}
legendAggregation={config.aggregation}
legendValueFormatter={tooltipValueFormatter}
minHeight="300px"
fillContainer
onViewAllLegendItems={onViewAllLegendItems}
@@ -1122,6 +1144,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
yAxisProps={yAxisProps}
stacked={stacked && visibleSeries.length > 1}
tooltipLabelFormatter={tooltipLabelFormatter}
tooltipValueFormatter={tooltipValueFormatter}
lineType="linear"
/>
</Chart.Root>
@@ -1129,9 +1152,13 @@ export const QueryResultsChart = memo(function QueryResultsChart({
});
/**
* Creates a Y-axis value formatter based on the data range
* Creates a Y-axis value formatter based on the data range and optional format hint
*/
function createYAxisFormatter(data: Record<string, unknown>[], series: string[]) {
function createYAxisFormatter(
data: Record<string, unknown>[],
series: string[],
format?: ColumnFormatType
) {
// Find min and max values across all series
let minVal = Infinity;
let maxVal = -Infinity;
@@ -1148,6 +1175,46 @@ function createYAxisFormatter(data: Record<string, unknown>[], series: string[])
const range = maxVal - minVal;
// Format-aware formatters
if (format === "bytes" || format === "decimalBytes") {
const divisor = format === "bytes" ? 1024 : 1000;
const units =
format === "bytes"
? ["B", "KiB", "MiB", "GiB", "TiB"]
: ["B", "KB", "MB", "GB", "TB"];
return (value: number): string => {
if (value === 0) return "0 B";
// Use consistent unit for all ticks based on max value
const i = Math.min(
Math.max(0, Math.floor(Math.log(Math.abs(maxVal || 1)) / Math.log(divisor))),
units.length - 1
);
const scaled = value / Math.pow(divisor, i);
return `${scaled.toFixed(scaled < 10 ? 1 : 0)} ${units[i]}`;
};
}
if (format === "percent") {
return (value: number): string => `${value.toFixed(range < 1 ? 2 : 1)}%`;
}
if (format === "duration") {
return (value: number): string => formatDurationMilliseconds(value, { style: "short" });
}
if (format === "durationSeconds") {
return (value: number): string =>
formatDurationMilliseconds(value * 1000, { style: "short" });
}
if (format === "costInDollars" || format === "cost") {
return (value: number): string => {
const dollars = format === "cost" ? value / 100 : value;
return formatCurrencyAccurate(dollars);
};
}
// Default formatter
return (value: number): string => {
// Use abbreviations for large numbers
if (Math.abs(value) >= 1_000_000) {
apps/webapp/app/components/code/TSQLResultsTable.tsx
+85 -3
View File
@@ -35,6 +35,7 @@ import { useCopy } from "~/hooks/useCopy";
import { useOrganization } from "~/hooks/useOrganizations";
import { useProject } from "~/hooks/useProject";
import { cn } from "~/utils/cn";
import { formatBytes, formatDecimalBytes, formatQuantity } from "~/utils/columnFormat";
import { formatCurrencyAccurate, formatNumber } from "~/utils/numberFormatter";
import { v3ProjectPath, v3RunPathFromFriendlyId } from "~/utils/pathBuilder";
import { ChartBlankState } from "../primitives/charts/ChartBlankState";
@@ -66,9 +67,10 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
if (value === null) return "NULL";
if (value === undefined) return "";
// Handle custom render types
if (column.customRenderType) {
switch (column.customRenderType) {
// Handle format hints (from prettyFormat() or auto-populated from customRenderType)
const formatType = column.format ?? column.customRenderType;
if (formatType) {
switch (formatType) {
case "duration":
if (typeof value === "number") {
return formatDurationMilliseconds(value, { style: "short" });
@@ -95,6 +97,26 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string
return value;
}
break;
case "bytes":
if (typeof value === "number") {
return formatBytes(value);
}
break;
case "decimalBytes":
if (typeof value === "number") {
return formatDecimalBytes(value);
}
break;
case "percent":
if (typeof value === "number") {
return `${value.toFixed(2)}%`;
}
break;
case "quantity":
if (typeof value === "number") {
return formatQuantity(value);
}
break;
}
}
@@ -222,6 +244,21 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number
if (value === null) return 4; // "NULL"
if (value === undefined) return 9; // "UNDEFINED"
// Handle format hint types - estimate their rendered width
const fmt = column.format;
if (fmt === "bytes" || fmt === "decimalBytes") {
// e.g., "1.50 GiB" or "256.00 MB"
return 12;
}
if (fmt === "percent") {
// e.g., "45.23%"
return 8;
}
if (fmt === "quantity") {
// e.g., "1.50M"
return 8;
}
// Handle custom render types - estimate their rendered width
if (column.customRenderType) {
switch (column.customRenderType) {
@@ -263,6 +300,8 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number
return typeof value === "string" ? Math.min(value.length, 20) : 12;
case "queue":
return typeof value === "string" ? Math.min(value.length, 25) : 15;
case "deploymentId":
return typeof value === "string" ? Math.min(value.length, 25) : 20;
}
}
@@ -394,6 +433,10 @@ function isRightAlignedColumn(column: OutputColumnMetadata): boolean {
) {
return true;
}
const fmt = column.format;
if (fmt === "bytes" || fmt === "decimalBytes" || fmt === "percent" || fmt === "quantity") {
return true;
}
return isNumericType(column.type);
}
@@ -476,6 +519,32 @@ function CellValue({
return <pre className="text-text-dimmed">UNDEFINED</pre>;
}
// Check format hint for new format types (from prettyFormat())
if (column.format && !column.customRenderType) {
switch (column.format) {
case "bytes":
if (typeof value === "number") {
return <span className="tabular-nums">{formatBytes(value)}</span>;
}
break;
case "decimalBytes":
if (typeof value === "number") {
return <span className="tabular-nums">{formatDecimalBytes(value)}</span>;
}
break;
case "percent":
if (typeof value === "number") {
return <span className="tabular-nums">{value.toFixed(2)}%</span>;
}
break;
case "quantity":
if (typeof value === "number") {
return <span className="tabular-nums">{formatQuantity(value)}</span>;
}
break;
}
}
// First check customRenderType for special rendering
if (column.customRenderType) {
switch (column.customRenderType) {
@@ -577,6 +646,19 @@ function CellValue({
}
return <span>{String(value)}</span>;
}
case "deploymentId": {
if (typeof value === "string" && value.startsWith("deployment_")) {
return (
<SimpleTooltip
content="Jump to deployment"
disableHoverableContent
hidden={!hovered}
button={<TextLink to={`/deployments/${value}`}>{value}</TextLink>}
/>
);
}
return <span>{String(value)}</span>;
}
}
}
apps/webapp/app/components/logs/LogDetailView.tsx
+1 -1
View File
@@ -186,7 +186,7 @@ function DetailsTab({
<CopyableText value={log.runId} copyValue={log.runId} asChild />
<LinkButton
to={runPath}
variant="tertiary/small"
variant="secondary/small"
shortcut={{ key: "v" }}
className="mt-2"
>
apps/webapp/app/components/logs/LogsTable.tsx
+7 -3
View File
@@ -26,6 +26,7 @@ import {
TableRow,
type TableVariant,
} from "../primitives/Table";
import { RunsIcon } from "~/assets/icons/RunsIcon";
type LogsTableProps = {
logs: LogEntry[];
@@ -124,6 +125,7 @@ export function LogsTable({
<TableHeaderCell
className="min-w-24 whitespace-nowrap"
tooltip={<LogLevelTooltipInfo />}
disableTooltipHoverableContent
>
Level
</TableHeaderCell>
@@ -165,7 +167,7 @@ export function LogsTable({
>
<DateTimeAccurate date={log.triggeredTimestamp} hour12={false} />
</TableCell>
<TableCell className="min-w-24">
<TableCell className="min-w-24" onClick={handleRowClick} hasAction>
<TruncatedCopyableValue value={log.runId} />
</TableCell>
<TableCell className="min-w-32" onClick={handleRowClick} hasAction>
@@ -185,9 +187,11 @@ export function LogsTable({
<LinkButton
to={runPath}
variant="minimal/small"
TrailingIcon={ArrowTopRightOnSquareIcon}
TrailingIcon={RunsIcon}
trailingIconClassName="text-text-bright"
className="h-[1.375rem] pl-1.5 pr-2"
>
View run
<span className="text-[0.6875rem] text-text-bright">View run</span>
</LinkButton>
}
/>
apps/webapp/app/components/navigation/SideMenu.tsx
+105 -2
View File
@@ -2,6 +2,7 @@ import {
AdjustmentsHorizontalIcon,
ArrowPathRoundedSquareIcon,
ArrowRightOnRectangleIcon,
ArrowTopRightOnSquareIcon,
BeakerIcon,
BellAlertIcon,
ChartBarIcon,
@@ -9,6 +10,7 @@ import {
ClockIcon,
Cog8ToothIcon,
CogIcon,
ExclamationTriangleIcon,
FolderIcon,
FolderOpenIcon,
GlobeAmericasIcon,
@@ -45,7 +47,7 @@ import { useHasAdminAccess } from "~/hooks/useUser";
import { type UserWithDashboardPreferences } from "~/models/user.server";
import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
import { type FeedbackType } from "~/routes/resources.feedback";
import { IncidentStatusPanel } from "~/routes/resources.incidents";
import { IncidentStatusPanel, useIncidentStatus } from "~/routes/resources.incidents";
import { cn } from "~/utils/cn";
import {
accountPath,
@@ -164,6 +166,8 @@ export function SideMenu({
const isAdmin = useHasAdminAccess();
const { isManagedCloud } = useFeatures();
const featureFlags = useFeatureFlags();
const incidentStatus = useIncidentStatus();
const isV3Project = project.engine === "V1";
const persistSideMenuPreferences = useCallback(
(data: {
@@ -598,7 +602,18 @@ export function SideMenu({
</div>
</div>
<div>
<IncidentStatusPanel isCollapsed={isCollapsed} />
<IncidentStatusPanel
isCollapsed={isCollapsed}
title={incidentStatus.title}
hasIncident={incidentStatus.hasIncident}
isManagedCloud={incidentStatus.isManagedCloud}
/>
<V3DeprecationPanel
isCollapsed={isCollapsed}
isV3={isV3Project}
hasIncident={incidentStatus.hasIncident}
isManagedCloud={incidentStatus.isManagedCloud}
/>
<motion.div
layout
transition={{ duration: 0.2, ease: "easeInOut" }}
@@ -623,6 +638,94 @@ export function SideMenu({
);
}
function V3DeprecationPanel({
isCollapsed,
isV3,
hasIncident,
isManagedCloud,
}: {
isCollapsed: boolean;
isV3: boolean;
hasIncident: boolean;
isManagedCloud: boolean;
}) {
if (!isManagedCloud || !isV3 || hasIncident) {
return null;
}
return (
<Popover>
<div className="p-1">
<motion.div
initial={false}
animate={{
height: isCollapsed ? 0 : "auto",
opacity: isCollapsed ? 0 : 1,
}}
transition={{ duration: 0.15 }}
className="overflow-hidden"
>
<V3DeprecationContent />
</motion.div>
<motion.div
initial={false}
animate={{
height: isCollapsed ? "auto" : 0,
opacity: isCollapsed ? 1 : 0,
}}
transition={{ duration: 0.15 }}
className="overflow-hidden"
>
<SimpleTooltip
button={
<PopoverTrigger className="flex !h-8 w-full items-center justify-center rounded border border-amber-500/30 bg-amber-500/15 transition-colors hover:border-amber-500/50 hover:bg-amber-500/25">
<ExclamationTriangleIcon className="size-5 text-amber-400" />
</PopoverTrigger>
}
content="V3 deprecation warning"
side="right"
sideOffset={8}
disableHoverableContent
asChild
/>
</motion.div>
</div>
<PopoverContent side="right" sideOffset={8} align="start" className="w-52 !min-w-0 p-0">
<V3DeprecationContent />
</PopoverContent>
</Popover>
);
}
function V3DeprecationContent() {
return (
<div className="flex flex-col gap-2 rounded border border-amber-500/30 bg-amber-500/10 p-2 pt-1.5">
<div className="flex items-center gap-1 border-b border-amber-500/30 pb-1">
<ExclamationTriangleIcon className="size-4 text-amber-400" />
<Paragraph variant="small/bright" className="text-amber-300">
V3 deprecation warning
</Paragraph>
</div>
<Paragraph variant="extra-small/bright" className="text-amber-300">
This is a v3 project. V3 deploys will stop working on 1 April 2026. Full shutdown is 1 July
2026 where all v3 runs will stop executing. Migrate to v4 to avoid downtime.
</Paragraph>
<LinkButton
variant="secondary/small"
to="https://trigger.dev/docs/migrating-from-v3"
target="_blank"
fullWidth
TrailingIcon={ArrowTopRightOnSquareIcon}
trailingIconClassName="text-amber-300"
className="border-amber-500/30 bg-amber-500/15 hover:!border-amber-500/50 hover:!bg-amber-500/25"
>
<span className="text-amber-300">View migration guide</span>
</LinkButton>
</div>
);
}
function ProjectSelector({
project,
organization,
apps/webapp/app/components/primitives/Table.tsx
+14 -1
View File
@@ -176,10 +176,22 @@ type TableCellBasicProps = {
type TableHeaderCellProps = TableCellBasicProps & {
hiddenLabel?: boolean;
tooltip?: ReactNode;
disableTooltipHoverableContent?: boolean;
};
export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellProps>(
({ className, alignment = "left", children, colSpan, hiddenLabel = false, tooltip }, ref) => {
(
{
className,
alignment = "left",
children,
colSpan,
hiddenLabel = false,
tooltip,
disableTooltipHoverableContent = false,
},
ref
) => {
const { variant } = useContext(TableContext);
let alignmentClassName = "text-left";
switch (alignment) {
@@ -222,6 +234,7 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
content={tooltip}
contentClassName="normal-case tracking-normal"
enabled={isHovered}
disableHoverableContent={disableTooltipHoverableContent}
/>
</div>
) : (
apps/webapp/app/components/primitives/charts/BigNumberCard.tsx
+27 -2
View File
@@ -1,10 +1,11 @@
import type { OutputColumnMetadata } from "@internal/tsql";
import type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql";
import { Hash } from "lucide-react";
import { useMemo } from "react";
import type {
BigNumberAggregationType,
BigNumberConfiguration,
} from "~/components/metrics/QueryWidget";
import { createValueFormatter } from "~/utils/columnFormat";
import { AnimatedNumber } from "../AnimatedNumber";
import { ChartBlankState } from "./ChartBlankState";
import { Spinner } from "../Spinner";
@@ -130,6 +131,15 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
return aggregateValues(values, aggregation);
}, [rows, column, aggregation, sortDirection]);
// Look up column format for format-aware display
const columnValueFormatter = useMemo(() => {
const columnMeta = columns.find((c) => c.name === column);
const formatType = (columnMeta?.format ?? columnMeta?.customRenderType) as
| ColumnFormatType
| undefined;
return createValueFormatter(formatType);
}, [columns, column]);
if (isLoading) {
return (
<div className="grid h-full place-items-center [container-type:size]">
@@ -142,6 +152,21 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
return <ChartBlankState icon={Hash} message="No data to display" />;
}
// Use format-aware formatter when available
if (columnValueFormatter) {
return (
<div className="h-full w-full [container-type:size]">
<div className="grid h-full w-full place-items-center">
<div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
{prefix && <span>{prefix}</span>}
<span>{columnValueFormatter(result)}</span>
{suffix && <span className="text-[0.4em] text-text-dimmed">{suffix}</span>}
</div>
</div>
</div>
);
}
const { displayValue, unitSuffix, decimalPlaces } = abbreviate
? abbreviateValue(result)
: { displayValue: result, unitSuffix: undefined, decimalPlaces: getDecimalPlaces(result) };
@@ -149,7 +174,7 @@ export function BigNumberCard({ rows, columns, config, isLoading = false }: BigN
return (
<div className="h-full w-full [container-type:size]">
<div className="grid h-full w-full place-items-center">
<div className="flex items-baseline gap-[0.15em] whitespace-nowrap font-normal tabular-nums leading-none text-text-bright text-[clamp(24px,12cqw,96px)]">
<div className="flex items-baseline gap-[0.15em] whitespace-nowrap text-[clamp(24px,12cqw,96px)] font-normal tabular-nums leading-none text-text-bright">
{prefix && <span>{prefix}</span>}
<AnimatedNumber value={displayValue} decimalPlaces={decimalPlaces} />
{(unitSuffix || suffix) && (
apps/webapp/app/components/primitives/charts/Chart.tsx
+7 -2
View File
@@ -104,6 +104,8 @@ const ChartTooltipContent = React.forwardRef<
indicator?: "line" | "dot" | "dashed";
nameKey?: string;
labelKey?: string;
/** Optional formatter for numeric values (e.g. bytes, duration) */
valueFormatter?: (value: number) => string;
}
>(
(
@@ -121,6 +123,7 @@ const ChartTooltipContent = React.forwardRef<
color,
nameKey,
labelKey,
valueFormatter,
},
ref
) => {
@@ -221,9 +224,11 @@ const ChartTooltipContent = React.forwardRef<
{itemConfig?.label || item.name}
</span>
</div>
{item.value && (
{item.value != null && (
<span className="text-foreground font-mono font-medium tabular-nums">
{item.value.toLocaleString()}
{valueFormatter && typeof item.value === "number"
? valueFormatter(item.value)
: item.value.toLocaleString()}
</span>
)}
</div>
apps/webapp/app/components/primitives/charts/ChartBar.tsx
+4 -1
View File
@@ -38,6 +38,8 @@ export type ChartBarRendererProps = {
referenceLine?: ReferenceLineProps;
/** Custom tooltip label formatter */
tooltipLabelFormatter?: (label: string, payload: any[]) => string;
/** Optional formatter for numeric tooltip values (e.g. bytes, duration) */
tooltipValueFormatter?: (value: number) => string;
/** Width injected by ResponsiveContainer */
width?: number;
/** Height injected by ResponsiveContainer */
@@ -62,6 +64,7 @@ export function ChartBarRenderer({
yAxisProps: yAxisPropsProp,
referenceLine,
tooltipLabelFormatter,
tooltipValueFormatter,
width,
height,
}: ChartBarRendererProps) {
@@ -159,7 +162,7 @@ export function ChartBarRenderer({
showLegend ? (
() => null
) : tooltipLabelFormatter ? (
<ChartTooltipContent />
<ChartTooltipContent valueFormatter={tooltipValueFormatter} />
) : (
<ZoomTooltip
isSelecting={zoom?.isSelecting}
apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx
+25 -4
View File
@@ -26,6 +26,8 @@ export type ChartLegendCompoundProps = {
totalLabel?: string;
/** Aggregation method – controls the header label and how totals are computed */
aggregation?: AggregationType;
/** Optional formatter for numeric values (e.g. bytes, duration) */
valueFormatter?: (value: number) => string;
/** Callback when "View all" button is clicked */
onViewAllLegendItems?: () => void;
/** When true, constrains legend to max 50% height with scrolling */
@@ -50,6 +52,7 @@ export function ChartLegendCompound({
className,
totalLabel,
aggregation,
valueFormatter,
onViewAllLegendItems,
scrollable = false,
}: ChartLegendCompoundProps) {
@@ -180,7 +183,11 @@ export function ChartLegendCompound({
<span className="font-medium">{currentTotalLabel}</span>
<span className="font-medium tabular-nums">
{currentTotal != null ? (
<AnimatedNumber value={currentTotal} duration={0.25} />
valueFormatter ? (
valueFormatter(currentTotal)
) : (
<AnimatedNumber value={currentTotal} duration={0.25} />
)
) : (
"\u2013"
)}
@@ -252,7 +259,11 @@ export function ChartLegendCompound({
)}
>
{total != null ? (
<AnimatedNumber value={total} duration={0.25} />
valueFormatter ? (
valueFormatter(total)
) : (
<AnimatedNumber value={total} duration={0.25} />
)
) : (
"\u2013"
)}
@@ -269,6 +280,7 @@ export function ChartLegendCompound({
item={legendItems.hoveredHiddenItem}
value={currentData[legendItems.hoveredHiddenItem.dataKey] ?? null}
remainingCount={legendItems.remaining - 1}
valueFormatter={valueFormatter}
/>
) : (
<ViewAllDataRow
@@ -315,9 +327,10 @@ type HoveredHiddenItemRowProps = {
item: { dataKey: string; color?: string; label: React.ReactNode };
value: number | null;
remainingCount: number;
valueFormatter?: (value: number) => string;
};
function HoveredHiddenItemRow({ item, value, remainingCount }: HoveredHiddenItemRowProps) {
function HoveredHiddenItemRow({ item, value, remainingCount, valueFormatter }: HoveredHiddenItemRowProps) {
return (
<div className="relative flex w-full items-center justify-between gap-2 rounded px-2 py-1">
{/* Active highlight background */}
@@ -339,7 +352,15 @@ function HoveredHiddenItemRow({ item, value, remainingCount }: HoveredHiddenItem
{remainingCount > 0 && <span className="text-text-dimmed">+{remainingCount} more</span>}
</div>
<span className="tabular-nums text-text-bright">
{value != null ? <AnimatedNumber value={value} duration={0.25} /> : "\u2013"}
{value != null ? (
valueFormatter ? (
valueFormatter(value)
) : (
<AnimatedNumber value={value} duration={0.25} />
)
) : (
"\u2013"
)}
</span>
</div>
</div>
apps/webapp/app/components/primitives/charts/ChartLine.tsx
+17 -2
View File
@@ -51,6 +51,8 @@ export type ChartLineRendererProps = {
stacked?: boolean;
/** Custom tooltip label formatter */
tooltipLabelFormatter?: (label: string, payload: any[]) => string;
/** Optional formatter for numeric tooltip values (e.g. bytes, duration) */
tooltipValueFormatter?: (value: number) => string;
/** Width injected by ResponsiveContainer */
width?: number;
/** Height injected by ResponsiveContainer */
@@ -75,6 +77,7 @@ export function ChartLineRenderer({
yAxisProps: yAxisPropsProp,
stacked = false,
tooltipLabelFormatter,
tooltipValueFormatter,
width,
height,
}: ChartLineRendererProps) {
@@ -157,7 +160,13 @@ export function ChartLineRenderer({
{/* When legend is shown below, render tooltip with cursor only (no content popup) */}
<ChartTooltip
cursor={{ stroke: "rgba(255, 255, 255, 0.1)", strokeWidth: 1 }}
content={showLegend ? () => null : <ChartTooltipContent indicator="line" />}
content={
showLegend ? (
() => null
) : (
<ChartTooltipContent indicator="line" valueFormatter={tooltipValueFormatter} />
)
}
labelFormatter={tooltipLabelFormatter}
/>
{/* Note: Legend is now rendered by ChartRoot outside the chart container */}
@@ -205,7 +214,13 @@ export function ChartLineRenderer({
{/* When legend is shown below, render tooltip with cursor only (no content popup) */}
<ChartTooltip
cursor={{ stroke: "rgba(255, 255, 255, 0.1)", strokeWidth: 1 }}
content={showLegend ? () => null : <ChartTooltipContent />}
content={
showLegend ? (
() => null
) : (
<ChartTooltipContent valueFormatter={tooltipValueFormatter} />
)
}
labelFormatter={tooltipLabelFormatter}
/>
{/* Note: Legend is now rendered by ChartRoot outside the chart container */}
apps/webapp/app/components/primitives/charts/ChartRoot.tsx
+7
View File
@@ -34,6 +34,8 @@ export type ChartRootProps = {
legendTotalLabel?: string;
/** Aggregation method used by the legend to compute totals (defaults to sum behavior) */
legendAggregation?: AggregationType;
/** Optional formatter for numeric legend values (e.g. bytes, duration) */
legendValueFormatter?: (value: number) => string;
/** Callback when "View all" legend button is clicked */
onViewAllLegendItems?: () => void;
/** When true, constrains legend to max 50% height with scrolling */
@@ -82,6 +84,7 @@ export function ChartRoot({
maxLegendItems = 5,
legendTotalLabel,
legendAggregation,
legendValueFormatter,
onViewAllLegendItems,
legendScrollable = false,
fillContainer = false,
@@ -108,6 +111,7 @@ export function ChartRoot({
maxLegendItems={maxLegendItems}
legendTotalLabel={legendTotalLabel}
legendAggregation={legendAggregation}
legendValueFormatter={legendValueFormatter}
onViewAllLegendItems={onViewAllLegendItems}
legendScrollable={legendScrollable}
fillContainer={fillContainer}
@@ -126,6 +130,7 @@ type ChartRootInnerProps = {
maxLegendItems?: number;
legendTotalLabel?: string;
legendAggregation?: AggregationType;
legendValueFormatter?: (value: number) => string;
onViewAllLegendItems?: () => void;
legendScrollable?: boolean;
fillContainer?: boolean;
@@ -140,6 +145,7 @@ function ChartRootInner({
maxLegendItems = 5,
legendTotalLabel,
legendAggregation,
legendValueFormatter,
onViewAllLegendItems,
legendScrollable = false,
fillContainer = false,
@@ -184,6 +190,7 @@ function ChartRootInner({
maxItems={maxLegendItems}
totalLabel={legendTotalLabel}
aggregation={legendAggregation}
valueFormatter={legendValueFormatter}
onViewAllLegendItems={onViewAllLegendItems}
scrollable={legendScrollable}
/>
apps/webapp/app/components/query/QueryEditor.tsx
+3 -3
View File
@@ -1175,9 +1175,9 @@ function QueryResultsCallouts({
<div className="flex flex-col gap-2 px-2 pt-2">
{hiddenColumns && hiddenColumns.length > 0 && (
<Callout variant="warning" className="shrink-0 text-sm">
<code>SELECT *</code> doesn't return all columns because it's slow. The following columns
are not shown: <span className="font-mono text-xs">{hiddenColumns.join(", ")}</span>.
Specify them explicitly to include them.
<code>SELECT *</code> returns core columns only. To include{" "}
<span className="font-mono text-xs">{hiddenColumns.join(", ")}</span>, add them to your
SELECT explicitly.
</Callout>
)}
{periodClipped && (
apps/webapp/app/env.server.ts
+10
View File
@@ -372,6 +372,7 @@ const EnvironmentSchema = z
// Development OTEL environment variables
DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
DEV_OTEL_METRICS_ENDPOINT: z.string().optional(),
// If this is set to 1, then the below variables are used to configure the batch processor for spans and logs
DEV_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"),
DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
@@ -382,6 +383,9 @@ const EnvironmentSchema = z
DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
DEV_OTEL_METRICS_EXPORT_INTERVAL_MILLIS: z.string().optional(),
DEV_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS: z.string().optional(),
DEV_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS: z.string().optional(),
PROD_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"),
PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
@@ -392,6 +396,9 @@ const EnvironmentSchema = z
PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
PROD_OTEL_METRICS_EXPORT_INTERVAL_MILLIS: z.string().optional(),
PROD_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS: z.string().optional(),
PROD_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS: z.string().optional(),
TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"),
TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"),
@@ -1229,6 +1236,9 @@ const EnvironmentSchema = z
EVENTS_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"),
EVENTS_CLICKHOUSE_BATCH_SIZE: z.coerce.number().int().default(1000),
EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
METRICS_CLICKHOUSE_BATCH_SIZE: z.coerce.number().int().default(10000),
METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
METRICS_CLICKHOUSE_MAX_CONCURRENCY: z.coerce.number().int().default(3),
EVENTS_CLICKHOUSE_INSERT_STRATEGY: z.enum(["insert", "insert_async"]).default("insert"),
EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT: z.string().default("1"),
EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: z.coerce.number().int().default(10485760),
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/AITabContent.tsx
+2
View File
@@ -30,6 +30,8 @@ export function AITabContent({
"Top 50 most expensive runs this week",
"Average execution duration by task this week",
"Run counts by tag in the past 7 days",
"CPU utilization over time by task",
"Peak memory usage per run",
];
return (
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx
+66 -1
View File
@@ -1,6 +1,9 @@
import { useState } from "react";
import { Header3 } from "~/components/primitives/Headers";
import { Paragraph } from "~/components/primitives/Paragraph";
import SegmentedControl from "~/components/primitives/SegmentedControl";
import type { QueryScope } from "~/services/queryService.server";
import { querySchemas } from "~/v3/querySchemas";
import { TryableCodeBlock } from "./TRQLGuideContent";
// Example queries for the Examples tab
@@ -9,6 +12,7 @@ export const exampleQueries: Array<{
description: string;
query: string;
scope: QueryScope;
table: string;
}> = [
{
title: "Failed runs by task (past 7 days)",
@@ -23,6 +27,7 @@ GROUP BY task_identifier
ORDER BY failed_count DESC
LIMIT 20`,
scope: "environment",
table: "runs",
},
{
title: "Execution duration p50 by task (past 7d)",
@@ -37,6 +42,7 @@ GROUP BY task_identifier
ORDER BY p50_duration_ms DESC
LIMIT 20`,
scope: "environment",
table: "runs",
},
{
title: "Runs over time",
@@ -50,6 +56,7 @@ GROUP BY timeBucket
ORDER BY timeBucket
LIMIT 1000`,
scope: "environment",
table: "runs",
},
{
title: "Most expensive 100 runs (past 7d)",
@@ -67,17 +74,75 @@ WHERE triggered_at > now() - INTERVAL 7 DAY
ORDER BY total_cost DESC
LIMIT 100`,
scope: "environment",
table: "runs",
},
{
title: "CPU utilization over time",
description: "Track process CPU utilization bucketed over time.",
query: `SELECT
timeBucket(),
avg(metric_value) AS avg_cpu
FROM metrics
WHERE metric_name = 'process.cpu.utilization'
GROUP BY timeBucket
ORDER BY timeBucket
LIMIT 1000`,
scope: "environment",
table: "metrics",
},
{
title: "Memory usage by task (past 7d)",
description: "Average memory usage per task identifier over the last 7 days.",
query: `SELECT
task_identifier,
avg(metric_value) AS avg_memory
FROM metrics
WHERE metric_name = 'system.memory.usage'
AND bucket_start > now() - INTERVAL 7 DAY
GROUP BY task_identifier
ORDER BY avg_memory DESC
LIMIT 20`,
scope: "environment",
table: "metrics",
},
{
title: "Available metric names",
description: "List all distinct metric names collected in your environment.",
query: `SELECT
metric_name,
count() AS sample_count
FROM metrics
GROUP BY metric_name
ORDER BY sample_count DESC
LIMIT 100`,
scope: "environment",
table: "metrics",
},
];
const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name }));
export function ExamplesContent({
onTryExample,
}: {
onTryExample: (query: string, scope: QueryScope) => void;
}) {
const [selectedTable, setSelectedTable] = useState(querySchemas[0].name);
const filtered = exampleQueries.filter((e) => e.table === selectedTable);
return (
<div className="space-y-6">
{exampleQueries.map((example) => (
<div className="sticky top-0 z-10 bg-background-bright pb-3">
<SegmentedControl
name="examples-table-selector"
value={selectedTable}
options={tableOptions}
variant="secondary/small"
fullWidth
onChange={setSelectedTable}
/>
</div>
{filtered.map((example) => (
<div key={example.title}>
<Header3 className="mb-1 text-text-bright">{example.title}</Header3>
<Paragraph variant="small" className="mb-2 text-text-dimmed">
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/TableSchemaContent.tsx
+29 -17
View File
@@ -1,8 +1,10 @@
import { useState } from "react";
import type { ColumnSchema } from "@internal/tsql";
import { Badge } from "~/components/primitives/Badge";
import { CopyableText } from "~/components/primitives/CopyableText";
import { Header3 } from "~/components/primitives/Headers";
import { Paragraph } from "~/components/primitives/Paragraph";
import SegmentedControl from "~/components/primitives/SegmentedControl";
import { querySchemas } from "~/v3/querySchemas";
function ColumnHelpItem({ col }: { col: ColumnSchema }) {
@@ -42,26 +44,36 @@ function ColumnHelpItem({ col }: { col: ColumnSchema }) {
);
}
const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name }));
export function TableSchemaContent() {
const [selectedTable, setSelectedTable] = useState(querySchemas[0].name);
const table = querySchemas.find((s) => s.name === selectedTable) ?? querySchemas[0];
return (
<div>
{querySchemas.map((table) => (
<div key={table.name} className="mb-6">
<div className="mb-2">
<Header3 className="font-mono text-text-bright">{table.name}</Header3>
{table.description && (
<Paragraph variant="small" className="mt-1 text-text-dimmed">
{table.description}
</Paragraph>
)}
</div>
<div className="flex flex-col gap-2 divide-y divide-grid-dimmed">
{Object.values(table.columns).map((col) => (
<ColumnHelpItem key={col.name} col={col} />
))}
</div>
</div>
))}
<div className="sticky top-0 z-10 bg-background-bright pb-3">
<SegmentedControl
name="table-schema-selector"
value={selectedTable}
options={tableOptions}
variant="secondary/small"
fullWidth
onChange={setSelectedTable}
/>
</div>
<div className="mb-2">
{table.description && (
<Paragraph variant="small" className="text-text-dimmed">
{table.description}
</Paragraph>
)}
</div>
<div className="flex flex-col gap-2 divide-y divide-grid-dimmed">
{Object.values(table.columns).map((col) => (
<ColumnHelpItem key={col.name} col={col} />
))}
</div>
</div>
);
}
apps/webapp/app/routes/deployments.$deploymentParam.ts
+70
View File
@@ -0,0 +1,70 @@
import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime";
import { z } from "zod";
import { prisma } from "~/db.server";
import { redirectWithErrorMessage } from "~/models/message.server";
import { requireUser } from "~/services/session.server";
import { rootPath, v3DeploymentPath } from "~/utils/pathBuilder";
const ParamsSchema = z.object({
deploymentParam: z.string(),
});
export async function loader({ params, request }: LoaderFunctionArgs) {
const user = await requireUser(request);
const { deploymentParam } = ParamsSchema.parse(params);
const deployment = await prisma.workerDeployment.findFirst({
where: {
friendlyId: deploymentParam,
project: {
organization: {
members: {
some: {
userId: user.id,
},
},
},
},
},
select: {
shortCode: true,
environment: {
select: {
slug: true,
},
},
project: {
select: {
slug: true,
organization: {
select: {
slug: true,
},
},
},
},
},
});
if (!deployment) {
return redirectWithErrorMessage(
rootPath(),
request,
"Deployment either doesn't exist or you don't have permission to view it",
{
ephemeral: false,
}
);
}
return redirect(
v3DeploymentPath(
{ slug: deployment.project.organization.slug },
{ slug: deployment.project.slug },
{ slug: deployment.environment.slug },
{ shortCode: deployment.shortCode },
0
)
);
}
apps/webapp/app/routes/login.mfa/route.tsx
+4 -5
View File
@@ -20,7 +20,7 @@ import { InputOTP, InputOTPGroup, InputOTPSlot } from "~/components/primitives/I
import { Paragraph } from "~/components/primitives/Paragraph";
import { Spinner } from "~/components/primitives/Spinner";
import { authenticator } from "~/services/auth.server";
import { commitSession, getUserSession, sessionStorage } from "~/services/sessionStorage.server";
import { commitSession, getUserSession } from "~/services/sessionStorage.server";
import { getSession as getMessageSession } from "~/models/message.server";
import { MultiFactorAuthenticationService } from "~/services/mfa/multiFactorAuthentication.server";
import { redirectWithErrorMessage, redirectBackWithErrorMessage } from "~/models/message.server";
@@ -152,9 +152,9 @@ export async function action({ request }: ActionFunctionArgs) {
}
async function completeLogin(request: Request, session: Session, userId: string) {
// Create a new authenticated session
const authSession = await sessionStorage.getSession(request.headers.get("Cookie"));
authSession.set(authenticator.sessionKey, { userId });
// Set the auth key on the same session object to avoid conflicting Set-Cookie headers
// (both authSession and session share the same __session cookie name)
session.set(authenticator.sessionKey, { userId });
// Get the redirect URL and clean up pending MFA data
const redirectTo = session.get("pending-mfa-redirect-to") ?? "/";
@@ -162,7 +162,6 @@ async function completeLogin(request: Request, session: Session, userId: string)
session.unset("pending-mfa-redirect-to");
const headers = new Headers();
headers.append("Set-Cookie", await sessionStorage.commitSession(authSession));
headers.append("Set-Cookie", await commitSession(session));
await trackAndClearReferralSource(request, userId, headers);
apps/webapp/app/routes/otel.v1.metrics.ts
+41
View File
@@ -0,0 +1,41 @@
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
import {
ExportMetricsServiceRequest,
ExportMetricsServiceResponse,
} from "@trigger.dev/otlp-importer";
import { otlpExporter } from "~/v3/otlpExporter.server";
export async function action({ request }: ActionFunctionArgs) {
try {
const contentType = request.headers.get("content-type")?.toLowerCase() ?? "";
if (contentType.startsWith("application/json")) {
const body = await request.json();
const exportResponse = await otlpExporter.exportMetrics(
body as ExportMetricsServiceRequest
);
return json(exportResponse, { status: 200 });
} else if (contentType.startsWith("application/x-protobuf")) {
const buffer = await request.arrayBuffer();
const exportRequest = ExportMetricsServiceRequest.decode(new Uint8Array(buffer));
const exportResponse = await otlpExporter.exportMetrics(exportRequest);
return new Response(ExportMetricsServiceResponse.encode(exportResponse).finish(), {
status: 200,
});
} else {
return new Response(
"Unsupported content type. Must be either application/x-protobuf or application/json",
{ status: 400 }
);
}
} catch (error) {
console.error(error);
return new Response("Internal Server Error", { status: 500 });
}
}
apps/webapp/app/routes/resources.incidents.tsx
+11 -3
View File
@@ -70,9 +70,17 @@ export function useIncidentStatus() {
};
}
export function IncidentStatusPanel({ isCollapsed = false }: { isCollapsed?: boolean }) {
const { title, hasIncident, isManagedCloud } = useIncidentStatus();
export function IncidentStatusPanel({
isCollapsed = false,
title,
hasIncident,
isManagedCloud,
}: {
isCollapsed?: boolean;
title: string | null;
hasIncident: boolean;
isManagedCloud: boolean;
}) {
if (!isManagedCloud || !hasIncident) {
return null;
}
apps/webapp/app/services/queryService.server.ts
+15 -8
View File
@@ -152,7 +152,14 @@ export async function executeQuery<TOut extends z.ZodSchema>(
return { success: false, error: new QueryError(errorMessage, { query: options.query }) };
}
// Build time filter fallback for triggered_at column
// Detect which table the query targets to determine the time column
// Each table schema declares its primary time column via timeConstraint
const matchedSchema = querySchemas.find((s) =>
new RegExp(`\\bFROM\\s+${s.name}\\b`, "i").test(options.query)
);
const timeColumn = matchedSchema?.timeConstraint ?? "triggered_at";
// Build time filter fallback for the table's time column
const defaultPeriod = await getDefaultPeriod(organizationId);
const timeFilter = timeFilters({
period: period ?? undefined,
@@ -173,15 +180,15 @@ export async function executeQuery<TOut extends z.ZodSchema>(
}
// Build the fallback WHERE condition based on what the user specified
let triggeredAtFallback: WhereClauseCondition;
let timeFallback: WhereClauseCondition;
if (timeFilter.from && timeFilter.to) {
triggeredAtFallback = { op: "between", low: timeFilter.from, high: timeFilter.to };
timeFallback = { op: "between", low: timeFilter.from, high: timeFilter.to };
} else if (timeFilter.from) {
triggeredAtFallback = { op: "gte", value: timeFilter.from };
timeFallback = { op: "gte", value: timeFilter.from };
} else if (timeFilter.to) {
triggeredAtFallback = { op: "lte", value: timeFilter.to };
timeFallback = { op: "lte", value: timeFilter.to };
} else {
triggeredAtFallback = { op: "gte", value: requestedFromDate! };
timeFallback = { op: "gte", value: requestedFromDate! };
}
const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30);
@@ -196,7 +203,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
project_id:
scope === "project" || scope === "environment" ? { op: "eq", value: projectId } : undefined,
environment_id: scope === "environment" ? { op: "eq", value: environmentId } : undefined,
triggered_at: { op: "gte", value: maxQueryPeriodDate },
[timeColumn]: { op: "gte", value: maxQueryPeriodDate },
// Optional filters for tasks and queues
task_identifier:
taskIdentifiers && taskIdentifiers.length > 0
@@ -238,7 +245,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
enforcedWhereClause,
fieldMappings,
whereClauseFallback: {
triggered_at: triggeredAtFallback,
[timeColumn]: timeFallback,
},
timeRange,
clickhouseSettings: {
apps/webapp/app/utils/columnFormat.ts
+70
View File
@@ -0,0 +1,70 @@
import type { ColumnFormatType } from "@internal/clickhouse";
import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
import { formatCurrencyAccurate } from "~/utils/numberFormatter";
/**
* Format a number as binary bytes (KiB, MiB, GiB, TiB)
*/
export function formatBytes(bytes: number): string {
if (bytes === 0) return "0 B";
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
const i = Math.min(
Math.max(0, Math.floor(Math.log(Math.abs(bytes)) / Math.log(1024))),
units.length - 1
);
return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 2)} ${units[i]}`;
}
/**
* Format a number as decimal bytes (KB, MB, GB, TB)
*/
export function formatDecimalBytes(bytes: number): string {
if (bytes === 0) return "0 B";
const units = ["B", "KB", "MB", "GB", "TB"];
const i = Math.min(
Math.max(0, Math.floor(Math.log(Math.abs(bytes)) / Math.log(1000))),
units.length - 1
);
return `${(bytes / Math.pow(1000, i)).toFixed(i === 0 ? 0 : 2)} ${units[i]}`;
}
/**
* Format a large number with human-readable suffix (K, M, B)
*/
export function formatQuantity(value: number): string {
const abs = Math.abs(value);
if (abs >= 1_000_000_000) return `${(value / 1_000_000_000).toFixed(2)}B`;
if (abs >= 1_000_000) return `${(value / 1_000_000).toFixed(2)}M`;
if (abs >= 1_000) return `${(value / 1_000).toFixed(2)}K`;
return value.toLocaleString();
}
/**
* Creates a value formatter function for a given column format type.
* Used by chart tooltips, legend values, and big number cards.
*/
export function createValueFormatter(
format?: ColumnFormatType
): ((value: number) => string) | undefined {
if (!format) return undefined;
switch (format) {
case "bytes":
return (v) => formatBytes(v);
case "decimalBytes":
return (v) => formatDecimalBytes(v);
case "percent":
return (v) => `${v.toFixed(2)}%`;
case "quantity":
return (v) => formatQuantity(v);
case "duration":
return (v) => formatDurationMilliseconds(v, { style: "short" });
case "durationSeconds":
return (v) => formatDurationMilliseconds(v * 1000, { style: "short" });
case "costInDollars":
return (v) => formatCurrencyAccurate(v);
case "cost":
return (v) => formatCurrencyAccurate(v / 100);
default:
return undefined;
}
}
apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts
+49
View File
@@ -956,6 +956,34 @@ async function resolveBuiltInDevVariables(runtimeEnvironment: RuntimeEnvironment
},
];
if (env.DEV_OTEL_METRICS_ENDPOINT) {
result.push({
key: "TRIGGER_OTEL_METRICS_ENDPOINT",
value: env.DEV_OTEL_METRICS_ENDPOINT,
});
}
if (env.DEV_OTEL_METRICS_EXPORT_INTERVAL_MILLIS) {
result.push({
key: "TRIGGER_OTEL_METRICS_EXPORT_INTERVAL_MILLIS",
value: env.DEV_OTEL_METRICS_EXPORT_INTERVAL_MILLIS,
});
}
if (env.DEV_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS) {
result.push({
key: "TRIGGER_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS",
value: env.DEV_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS,
});
}
if (env.DEV_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS) {
result.push({
key: "TRIGGER_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS",
value: env.DEV_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS,
});
}
if (env.DEV_OTEL_BATCH_PROCESSING_ENABLED === "1") {
result = result.concat([
{
@@ -1087,6 +1115,27 @@ async function resolveBuiltInProdVariables(
]);
}
if (env.PROD_OTEL_METRICS_EXPORT_INTERVAL_MILLIS) {
result.push({
key: "TRIGGER_OTEL_METRICS_EXPORT_INTERVAL_MILLIS",
value: env.PROD_OTEL_METRICS_EXPORT_INTERVAL_MILLIS,
});
}
if (env.PROD_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS) {
result.push({
key: "TRIGGER_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS",
value: env.PROD_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS,
});
}
if (env.PROD_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS) {
result.push({
key: "TRIGGER_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS",
value: env.PROD_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS,
});
}
if (env.PROD_OTEL_BATCH_PROCESSING_ENABLED === "1") {
result = result.concat([
{
apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts
+1
View File
@@ -266,6 +266,7 @@ export class ClickhouseEventRepository implements IEventRepository {
expires_at: convertDateToClickhouseDateTime(
new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) // 1 year
),
machine_id: event.machineId ?? "",
},
...this.spanEventsToTaskEventV1Input(event),
];
apps/webapp/app/v3/eventRepository/eventRepository.types.ts
+1
View File
@@ -56,6 +56,7 @@ export type CreateEventInput = Omit<
resourceProperties?: Attributes;
metadata: Attributes | undefined;
style: Attributes | undefined;
machineId?: string;
};
export type CreatableEventKind = TaskEventKind;
apps/webapp/app/v3/otlpExporter.server.ts
+245
View File
@@ -4,10 +4,13 @@ import {
AnyValue,
ExportLogsServiceRequest,
ExportLogsServiceResponse,
ExportMetricsServiceRequest,
ExportMetricsServiceResponse,
ExportTraceServiceRequest,
ExportTraceServiceResponse,
KeyValue,
ResourceLogs,
ResourceMetrics,
ResourceSpans,
SeverityNumber,
Span,
@@ -15,7 +18,10 @@ import {
Span_SpanKind,
Status_StatusCode,
} from "@trigger.dev/otlp-importer";
import type { MetricsV1Input } from "@internal/clickhouse";
import { logger } from "~/services/logger.server";
import { clickhouseClient } from "~/services/clickhouseInstance.server";
import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
import { ClickhouseEventRepository } from "./eventRepository/clickhouseEventRepository.server";
import {
clickhouseEventRepository,
@@ -42,6 +48,7 @@ class OTLPExporter {
private readonly _eventRepository: EventRepository,
private readonly _clickhouseEventRepository: ClickhouseEventRepository,
private readonly _clickhouseEventRepositoryV2: ClickhouseEventRepository,
private readonly _metricsFlushScheduler: DynamicFlushScheduler<MetricsV1Input>,
private readonly _verbose: boolean,
private readonly _spanAttributeValueLengthLimit: number
) {
@@ -66,6 +73,29 @@ class OTLPExporter {
});
}
async exportMetrics(
request: ExportMetricsServiceRequest
): Promise<ExportMetricsServiceResponse> {
return await startSpan(this._tracer, "exportMetrics", async (span) => {
const rows = this.#filterResourceMetrics(request.resourceMetrics).flatMap(
(resourceMetrics) => {
return convertMetricsToClickhouseRows(
resourceMetrics,
this._spanAttributeValueLengthLimit
);
}
);
span.setAttribute("metric_row_count", rows.length);
if (rows.length > 0) {
this._metricsFlushScheduler.addToBatch(rows);
}
return ExportMetricsServiceResponse.create();
});
}
async exportLogs(request: ExportLogsServiceRequest): Promise<ExportLogsServiceResponse> {
return await startSpan(this._tracer, "exportLogs", async (span) => {
this.#logExportLogsVerbose(request);
@@ -202,6 +232,18 @@ class OTLPExporter {
return isBoolValue(attribute.value) ? attribute.value.boolValue : false;
});
}
#filterResourceMetrics(resourceMetrics: ResourceMetrics[]): ResourceMetrics[] {
return resourceMetrics.filter((rm) => {
const triggerAttribute = rm.resource?.attributes.find(
(attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
);
if (!triggerAttribute) return false;
return isBoolValue(triggerAttribute.value) ? triggerAttribute.value.boolValue : false;
});
}
}
function convertLogsToCreateableEvents(
@@ -289,6 +331,7 @@ function convertLogsToCreateableEvents(
projectId: logProperties.projectId ?? resourceProperties.projectId ?? "unknown",
runId: logProperties.runId ?? resourceProperties.runId ?? "unknown",
taskSlug: logProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown",
machineId: logProperties.machineId ?? resourceProperties.machineId,
attemptNumber:
extractNumberAttribute(
log.attributes ?? [],
@@ -395,6 +438,7 @@ function convertSpansToCreateableEvents(
projectId: spanProperties.projectId ?? resourceProperties.projectId ?? "unknown",
runId: spanProperties.runId ?? resourceProperties.runId ?? "unknown",
taskSlug: spanProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown",
machineId: spanProperties.machineId ?? resourceProperties.machineId,
attemptNumber:
extractNumberAttribute(
span.attributes ?? [],
@@ -410,6 +454,194 @@ function convertSpansToCreateableEvents(
return { events, taskEventStore };
}
function floorToTenSecondBucket(timeUnixNano: bigint | number): string {
const epochMs = Number(BigInt(timeUnixNano) / BigInt(1_000_000));
const flooredMs = Math.floor(epochMs / 10_000) * 10_000;
const date = new Date(flooredMs);
// Format as ClickHouse DateTime: YYYY-MM-DD HH:MM:SS
return date.toISOString().replace("T", " ").replace(/\.\d{3}Z$/, "");
}
function convertMetricsToClickhouseRows(
resourceMetrics: ResourceMetrics,
spanAttributeValueLengthLimit: number
): MetricsV1Input[] {
const resourceAttributes = resourceMetrics.resource?.attributes ?? [];
const resourceProperties = extractEventProperties(resourceAttributes);
const organizationId = resourceProperties.organizationId ?? "unknown";
const projectId = resourceProperties.projectId ?? "unknown";
const environmentId = resourceProperties.environmentId ?? "unknown";
const resourceCtx = {
taskSlug: resourceProperties.taskSlug,
runId: resourceProperties.runId,
attemptNumber: resourceProperties.attemptNumber,
machineId: extractStringAttribute(resourceAttributes, SemanticInternalAttributes.MACHINE_ID),
workerId: extractStringAttribute(resourceAttributes, SemanticInternalAttributes.WORKER_ID),
workerVersion: extractStringAttribute(
resourceAttributes,
SemanticInternalAttributes.WORKER_VERSION
),
};
const rows: MetricsV1Input[] = [];
for (const scopeMetrics of resourceMetrics.scopeMetrics) {
for (const metric of scopeMetrics.metrics) {
const metricName = metric.name;
// Process gauge data points
if (metric.gauge) {
for (const dp of metric.gauge.dataPoints) {
const value: number =
dp.asDouble !== undefined ? dp.asDouble : dp.asInt !== undefined ? Number(dp.asInt) : 0;
const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
rows.push({
organization_id: organizationId,
project_id: projectId,
environment_id: environmentId,
metric_name: metricName,
metric_type: "gauge",
metric_subject: resolved.machineId ?? "unknown",
bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
value,
attributes: resolved.attributes,
});
}
}
// Process sum data points
if (metric.sum) {
for (const dp of metric.sum.dataPoints) {
const value: number =
dp.asDouble !== undefined ? dp.asDouble : dp.asInt !== undefined ? Number(dp.asInt) : 0;
const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
rows.push({
organization_id: organizationId,
project_id: projectId,
environment_id: environmentId,
metric_name: metricName,
metric_type: "sum",
metric_subject: resolved.machineId ?? "unknown",
bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
value,
attributes: resolved.attributes,
});
}
}
// Process histogram data points
if (metric.histogram) {
for (const dp of metric.histogram.dataPoints) {
const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
const count = Number(dp.count);
const sum = dp.sum ?? 0;
rows.push({
organization_id: organizationId,
project_id: projectId,
environment_id: environmentId,
metric_name: metricName,
metric_type: "histogram",
metric_subject: resolved.machineId ?? "unknown",
bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
value: count > 0 ? sum / count : 0,
attributes: resolved.attributes,
});
}
}
}
}
return rows;
}
// Prefixes injected by TaskContextMetricExporter — these are extracted into
// the nested `trigger` key and should not appear as top-level user attributes.
const INTERNAL_METRIC_ATTRIBUTE_PREFIXES = ["ctx.", "worker."];
interface ResourceContext {
taskSlug: string | undefined;
runId: string | undefined;
attemptNumber: number | undefined;
machineId: string | undefined;
workerId: string | undefined;
workerVersion: string | undefined;
}
function resolveDataPointContext(
dpAttributes: KeyValue[],
resourceCtx: ResourceContext
): {
machineId: string | undefined;
attributes: Record<string, unknown>;
} {
const runId =
resourceCtx.runId ??
extractStringAttribute(dpAttributes, SemanticInternalAttributes.RUN_ID);
const taskSlug =
resourceCtx.taskSlug ??
extractStringAttribute(dpAttributes, SemanticInternalAttributes.TASK_SLUG);
const attemptNumber =
resourceCtx.attemptNumber ??
extractNumberAttribute(dpAttributes, SemanticInternalAttributes.ATTEMPT_NUMBER);
const machineId =
resourceCtx.machineId ??
extractStringAttribute(dpAttributes, SemanticInternalAttributes.MACHINE_ID);
const workerId =
resourceCtx.workerId ??
extractStringAttribute(dpAttributes, SemanticInternalAttributes.WORKER_ID);
const workerVersion =
resourceCtx.workerVersion ??
extractStringAttribute(dpAttributes, SemanticInternalAttributes.WORKER_VERSION);
const machineName = extractStringAttribute(
dpAttributes,
SemanticInternalAttributes.MACHINE_PRESET_NAME
);
const environmentType = extractStringAttribute(
dpAttributes,
SemanticInternalAttributes.ENVIRONMENT_TYPE
);
// Build the trigger context object with only defined values
const trigger: Record<string, string | number> = {};
if (runId) trigger.run_id = runId;
if (taskSlug) trigger.task_slug = taskSlug;
if (attemptNumber !== undefined) trigger.attempt_number = attemptNumber;
if (machineId) trigger.machine_id = machineId;
if (machineName) trigger.machine_name = machineName;
if (workerId) trigger.worker_id = workerId;
if (workerVersion) trigger.worker_version = workerVersion;
if (environmentType) trigger.environment_type = environmentType;
// Build user attributes, filtering out internal ctx/worker keys
const result: Record<string, unknown> = {};
if (Object.keys(trigger).length > 0) {
result.trigger = trigger;
}
for (const attr of dpAttributes) {
if (INTERNAL_METRIC_ATTRIBUTE_PREFIXES.some((prefix) => attr.key.startsWith(prefix))) {
continue;
}
if (isStringValue(attr.value)) {
result[attr.key] = attr.value.stringValue;
} else if (isIntValue(attr.value)) {
result[attr.key] = Number(attr.value.intValue);
} else if (isDoubleValue(attr.value)) {
result[attr.key] = attr.value.doubleValue;
} else if (isBoolValue(attr.value)) {
result[attr.key] = attr.value.boolValue;
}
}
return { machineId, attributes: result };
}
function extractEventProperties(attributes: KeyValue[], prefix?: string) {
return {
metadata: convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]),
@@ -428,6 +660,7 @@ function extractEventProperties(attributes: KeyValue[], prefix?: string) {
SemanticInternalAttributes.ATTEMPT_NUMBER,
]),
taskSlug: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.TASK_SLUG]),
machineId: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.MACHINE_ID]),
};
}
@@ -891,10 +1124,22 @@ function hasUnpairedSurrogateAtEnd(str: string): boolean {
export const otlpExporter = singleton("otlpExporter", initializeOTLPExporter);
function initializeOTLPExporter() {
const metricsFlushScheduler = new DynamicFlushScheduler<MetricsV1Input>({
batchSize: env.METRICS_CLICKHOUSE_BATCH_SIZE,
flushInterval: env.METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS,
callback: async (_flushId, batch) => {
await clickhouseClient.metrics.insert(batch);
},
minConcurrency: 1,
maxConcurrency: env.METRICS_CLICKHOUSE_MAX_CONCURRENCY,
loadSheddingEnabled: false,
});
return new OTLPExporter(
eventRepository,
clickhouseEventRepository,
clickhouseEventRepositoryV2,
metricsFlushScheduler,
process.env.OTLP_EXPORTER_VERBOSE === "1",
process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT
? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10)
apps/webapp/app/v3/querySchemas.ts
+164 -2
View File
@@ -1,4 +1,4 @@
import { column, type TableSchema } from "@internal/tsql";
import { column, type BucketThreshold, type TableSchema } from "@internal/tsql";
import { z } from "zod";
import { autoFormatSQL } from "~/components/code/TSQLEditor";
import { runFriendlyStatus, runStatusTitleFromStatus } from "~/components/runs/v3/TaskRunStatus";
@@ -33,6 +33,7 @@ export const runsSchema: TableSchema = {
clickhouseName: "trigger_dev.task_runs_v2",
description: "Task runs - stores all task execution records",
timeConstraint: "triggered_at",
useFinal: true,
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
@@ -434,10 +435,171 @@ export const runsSchema: TableSchema = {
},
};
/**
* Schema definition for the metrics table (trigger_dev.metrics_v1)
*/
export const metricsSchema: TableSchema = {
name: "metrics",
clickhouseName: "trigger_dev.metrics_v1",
description: "Host and runtime metrics collected during task execution",
timeConstraint: "bucket_start",
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
columns: {
environment: {
name: "environment",
clickhouseName: "environment_id",
...column("String", { description: "The environment slug", example: "prod" }),
fieldMapping: "environment",
customRenderType: "environment",
},
project: {
name: "project",
clickhouseName: "project_id",
...column("String", {
description: "The project reference, they always start with `proj_`.",
example: "proj_howcnaxbfxdmwmxazktx",
}),
fieldMapping: "project",
customRenderType: "project",
},
metric_name: {
name: "metric_name",
...column("LowCardinality(String)", {
description: "The name of the metric (e.g. process.cpu.utilization, system.memory.usage)",
example: "process.cpu.utilization",
coreColumn: true,
}),
},
metric_type: {
name: "metric_type",
...column("LowCardinality(String)", {
description: "The type of metric",
allowedValues: ["gauge", "sum", "histogram"],
example: "gauge",
}),
},
machine_id: {
name: "machine_id",
clickhouseName: "metric_subject",
...column("String", {
description: "The machine ID that produced this metric",
example: "machine-abc123",
}),
},
bucket_start: {
name: "bucket_start",
...column("DateTime", {
description: "The start of the 10-second aggregation bucket",
example: "2024-01-15 09:30:00",
coreColumn: true,
}),
},
metric_value: {
name: "metric_value",
clickhouseName: "value",
...column("Float64", {
description: "The metric value",
example: "0.75",
coreColumn: true,
}),
},
// Attributes (JSON column for user-defined and system attributes)
attributes: {
name: "attributes",
...column("JSON", {
description: "JSON attributes attached to the metric data point.",
example: '{"region": "us-east-1"}',
}),
},
// Trigger context columns (from attributes.trigger.* JSON subpaths)
run_id: {
name: "run_id",
...column("String", {
description: "The run ID associated with this metric",
customRenderType: "runId",
example: "run_cm1a2b3c4d5e6f7g8h9i",
coreColumn: true,
}),
expression: "attributes.trigger.run_id",
},
task_identifier: {
name: "task_identifier",
...column("String", {
description: "Task identifier/slug",
example: "my-background-task",
coreColumn: true,
}),
expression: "attributes.trigger.task_slug",
},
attempt_number: {
name: "attempt_number",
...column("UInt64", {
description: "The attempt number for this metric",
example: "1",
}),
expression: "attributes.trigger.attempt_number",
},
machine_name: {
name: "machine_name",
...column("String", {
description: "The machine preset used for execution",
allowedValues: [...MACHINE_PRESETS],
example: "small-1x",
}),
expression: "attributes.trigger.machine_name",
},
environment_type: {
name: "environment_type",
...column("String", {
description: "Environment type",
allowedValues: [...ENVIRONMENT_TYPES],
customRenderType: "environmentType",
example: "PRODUCTION",
}),
expression: "attributes.trigger.environment_type",
},
worker_id: {
name: "worker_id",
...column("String", {
description: "The worker ID that produced this metric",
customRenderType: "deploymentId",
example: "deployment_cm1a2b3c4d5e",
}),
expression: "attributes.trigger.worker_id",
},
worker_version: {
name: "worker_version",
...column("String", {
description: "The worker version that produced this metric",
example: "20240115.1",
}),
expression: "attributes.trigger.worker_version",
},
},
timeBucketThresholds: [
// Metrics are pre-aggregated into 10-second buckets, so 10s is the most granular interval.
// All thresholds are shifted coarser compared to the runs table defaults.
{ maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } },
{ maxRangeSeconds: 12 * 60 * 60, interval: { value: 1, unit: "MINUTE" } },
{ maxRangeSeconds: 2 * 24 * 60 * 60, interval: { value: 5, unit: "MINUTE" } },
{ maxRangeSeconds: 7 * 24 * 60 * 60, interval: { value: 15, unit: "MINUTE" } },
{ maxRangeSeconds: 30 * 24 * 60 * 60, interval: { value: 1, unit: "HOUR" } },
{ maxRangeSeconds: 90 * 24 * 60 * 60, interval: { value: 6, unit: "HOUR" } },
{ maxRangeSeconds: 180 * 24 * 60 * 60, interval: { value: 1, unit: "DAY" } },
{ maxRangeSeconds: 365 * 24 * 60 * 60, interval: { value: 1, unit: "WEEK" } },
] satisfies BucketThreshold[],
};
/**
* All available schemas for the query editor
*/
export const querySchemas: TableSchema[] = [runsSchema];
export const querySchemas: TableSchema[] = [runsSchema, metricsSchema];
/**
* Default query for the query editor
apps/webapp/app/v3/services/aiQueryService.server.ts
+62 -8
View File
@@ -55,7 +55,7 @@ export class AIQueryService {
constructor(
private readonly tableSchema: TableSchema[],
private readonly model: LanguageModelV1 = openai("gpt-4o-mini")
private readonly model: LanguageModelV1 = openai("gpt-4.1-mini")
) {}
/**
@@ -65,7 +65,7 @@ export class AIQueryService {
private buildSetTimeFilterTool() {
return tool({
description:
"Set the time filter for the query page UI instead of adding triggered_at conditions to the query. ALWAYS use this tool when the user wants to filter by time (e.g., 'last 7 days', 'past hour', 'yesterday'). The UI will apply this filter automatically. Do NOT add triggered_at to the WHERE clause - use this tool instead.",
"Set the time filter for the query page UI instead of adding time conditions to the query. ALWAYS use this tool when the user wants to filter by time (e.g., 'last 7 days', 'past hour', 'yesterday'). The UI will apply this filter automatically using the table's time column (triggered_at for runs, bucket_start for metrics). Do NOT add triggered_at or bucket_start to the WHERE clause for time filtering - use this tool instead.",
parameters: z.object({
period: z
.string()
@@ -366,7 +366,7 @@ export class AIQueryService {
* Build the system prompt for the AI
*/
private buildSystemPrompt(schemaDescription: string): string {
return `You are an expert SQL assistant that generates TSQL queries for a task run analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
return `You are an expert SQL assistant that generates TSQL queries for a task analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
## Your Task
Convert natural language requests into valid TSQL SELECT queries. Always validate your queries using the validateTSQLQuery tool before returning them.
@@ -374,6 +374,13 @@ Convert natural language requests into valid TSQL SELECT queries. Always validat
## Available Schema
${schemaDescription}
## Choosing the Right Table
- **runs** — Task run records (status, timing, cost, output, etc.). Use for questions about runs, tasks, failures, durations, costs, queues.
- **metrics** — Host and runtime metrics collected during task execution (CPU, memory). Use for questions about resource usage, CPU utilization, memory consumption, or performance monitoring. Each row is a 10-second aggregation bucket tied to a specific run.
When the user mentions "CPU", "memory", "utilization", "resource usage", or similar terms, query the \`metrics\` table. When they mention "runs", "tasks", "failures", "status", "duration", or "cost", query the \`runs\` table.
## TSQL Syntax Guide
TSQL supports standard SQL syntax with some ClickHouse-specific features:
@@ -437,16 +444,51 @@ LIMIT 1000
Only use explicit \`toStartOfHour\`/\`toStartOfDay\` etc. if the user specifically requests a particular bucket size (e.g., "group by hour", "bucket by day").
### Common Patterns
#### Runs table
- Status filter: WHERE status = 'Failed' or WHERE status IN ('Failed', 'Crashed')
- Time filtering: Use the \`setTimeFilter\` tool (NOT triggered_at in WHERE clause)
- Time filtering: Use the \`setTimeFilter\` tool (NOT triggered_at/bucket_start in WHERE clause)
#### Metrics table
- Filter by metric name: WHERE metric_name = 'process.cpu.utilization'
- Filter by run: WHERE run_id = 'run_abc123'
- Filter by task: WHERE task_identifier = 'my-task'
- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p95, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
- Use \`metric_value\` — the metric's observed value
- Use prettyFormat(expr, 'bytes') to tell the UI to format values as bytes (e.g., "1.50 GiB") — keeps values numeric for charts
- Use prettyFormat(expr, 'percent') for percentage values
- prettyFormat does NOT change the SQL — it only adds a display hint
- Available format types: bytes, decimalBytes, percent, quantity, duration, durationSeconds, costInDollars
- For memory metrics (including nodejs.heap.*), always use prettyFormat with 'bytes'
- For CPU utilization, consider prettyFormat with 'percent'
\`\`\`sql
-- CPU utilization over time for a task
SELECT timeBucket(), task_identifier, prettyFormat(avg(metric_value), 'percent') AS avg_cpu
FROM metrics
WHERE metric_name = 'process.cpu.utilization'
GROUP BY timeBucket, task_identifier
ORDER BY timeBucket
LIMIT 1000
\`\`\`
\`\`\`sql
-- Peak memory usage per run
SELECT run_id, task_identifier, prettyFormat(max(metric_value), 'bytes') AS peak_memory
FROM metrics
WHERE metric_name = 'process.memory.usage'
GROUP BY run_id, task_identifier
ORDER BY peak_memory DESC
LIMIT 100
\`\`\`
## Important Rules
1. NEVER use SELECT * - ClickHouse is a columnar database where SELECT * has very poor performance
2. Always select only the specific columns needed for the request
3. When column selection is ambiguous, use the core columns marked [CORE] in the schema
4. **TIME FILTERING**: When the user wants to filter by time (e.g., "last 7 days", "past hour", "yesterday"), ALWAYS use the \`setTimeFilter\` tool instead of adding \`triggered_at\` conditions to the query. The UI has a time filter that will apply this automatically.
5. Do NOT add \`triggered_at\` to WHERE clauses - use \`setTimeFilter\` tool instead. If the user doesn't specify a time period, do NOT add any time filter (the UI defaults to 7 days).
4. **TIME FILTERING**: When the user wants to filter by time (e.g., "last 7 days", "past hour", "yesterday"), ALWAYS use the \`setTimeFilter\` tool instead of adding time conditions to the WHERE clause. The UI has a time filter that will apply this automatically. This applies to both the \`runs\` table (triggered_at) and the \`metrics\` table (bucket_start).
5. Do NOT add \`triggered_at\` or \`bucket_start\` to WHERE clauses for time filtering - use \`setTimeFilter\` tool instead. If the user doesn't specify a time period, do NOT add any time filter (the UI defaults to 7 days).
6. **TIME BUCKETING**: When the user wants to see data over time or in time buckets, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
7. ALWAYS use the validateTSQLQuery tool to check your query before returning it
8. If validation fails, fix the issues and try again (up to 3 attempts)
@@ -472,7 +514,7 @@ If you cannot generate a valid query, explain why briefly.`;
* Build the system prompt for edit mode
*/
private buildEditSystemPrompt(schemaDescription: string): string {
return `You are an expert SQL assistant that modifies existing TSQL queries for a task run analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
return `You are an expert SQL assistant that modifies existing TSQL queries for a task analytics system. TSQL is a SQL dialect similar to ClickHouse SQL.
## Your Task
Modify the provided TSQL query according to the user's instructions. Make only the changes requested - preserve the existing query structure where possible.
@@ -480,6 +522,11 @@ Modify the provided TSQL query according to the user's instructions. Make only t
## Available Schema
${schemaDescription}
## Choosing the Right Table
- **runs** — Task run records (status, timing, cost, output, etc.). Use for questions about runs, tasks, failures, durations, costs, queues.
- **metrics** — Host and runtime metrics collected during task execution (CPU, memory). Use for questions about resource usage, CPU utilization, memory consumption, or performance monitoring. Each row is a 10-second aggregation bucket tied to a specific run.
## TSQL Syntax Guide
TSQL supports standard SQL syntax with some ClickHouse-specific features:
@@ -539,11 +586,18 @@ ORDER BY timeBucket
LIMIT 1000
\`\`\`
### Common Metrics Patterns
- Filter by metric: WHERE metric_name = 'process.cpu.utilization'
- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p50, nodejs.event_loop.delay.p99, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
- Use \`metric_value\` — the metric's observed value
- Use prettyFormat(expr, 'bytes') for memory metrics (including nodejs.heap.*), prettyFormat(expr, 'percent') for CPU utilization
- prettyFormat does NOT change the SQL — it only adds a display hint for the UI
## Important Rules
1. NEVER use SELECT * - ClickHouse is a columnar database where SELECT * has very poor performance
2. If the existing query uses SELECT *, replace it with specific columns (use core columns marked [CORE] as defaults)
3. **TIME FILTERING**: When the user wants to change time filtering (e.g., "change to last 30 days"), use the \`setTimeFilter\` tool instead of modifying \`triggered_at\` conditions. If the existing query has \`triggered_at\` in WHERE, consider removing it and using \`setTimeFilter\` instead.
3. **TIME FILTERING**: When the user wants to change time filtering (e.g., "change to last 30 days"), use the \`setTimeFilter\` tool instead of modifying time column conditions. If the existing query has \`triggered_at\` or \`bucket_start\` in WHERE for time filtering, consider removing it and using \`setTimeFilter\` instead.
4. **TIME BUCKETING**: When adding time-series grouping, use \`timeBucket()\` in SELECT and reference it as \`timeBucket\` in GROUP BY / ORDER BY. Only use manual bucketing functions (toStartOfHour, toStartOfDay, etc.) when the user explicitly requests a specific bucket size.
5. ALWAYS use the validateTSQLQuery tool to check your modified query before returning it
6. If validation fails, fix the issues and try again (up to 3 attempts)
apps/webapp/package.json
+1 -1
View File
@@ -70,7 +70,7 @@
"@opentelemetry/exporter-logs-otlp-http": "0.203.0",
"@opentelemetry/exporter-metrics-otlp-proto": "0.203.0",
"@opentelemetry/exporter-trace-otlp-http": "0.203.0",
"@opentelemetry/host-metrics": "^0.36.0",
"@opentelemetry/host-metrics": "^0.37.0",
"@opentelemetry/instrumentation": "0.203.0",
"@opentelemetry/instrumentation-aws-sdk": "^0.57.0",
"@opentelemetry/instrumentation-express": "^0.52.0",
docs/config/config-file.mdx
+35 -6
View File
@@ -154,21 +154,30 @@ Some ones we recommend:
### Telemetry Exporters
You can also configure custom telemetry exporters to send your traces and logs to other external services. For example, you can send your logs to [Axiom](https://axiom.co/docs/guides/opentelemetry-nodejs#exporter-instrumentation-ts). First, add the opentelemetry exporter packages to your package.json file:
You can also configure custom telemetry exporters to send your traces, logs, and metrics to other external services. For example, you can send your logs to [Axiom](https://axiom.co/docs/guides/opentelemetry-nodejs#exporter-instrumentation-ts). First, add the opentelemetry exporter packages to your package.json file:
```json package.json
"dependencies": {
"@opentelemetry/exporter-logs-otlp-http": "0.52.1",
"@opentelemetry/exporter-trace-otlp-http": "0.52.1"
"@opentelemetry/exporter-trace-otlp-http": "0.52.1",
"@opentelemetry/exporter-metrics-otlp-proto": "0.52.1"
}
```
<Note>
Axiom's `/v1/metrics` endpoint only supports protobuf (`application/x-protobuf`), not JSON. Use
`@opentelemetry/exporter-metrics-otlp-proto` instead of
`@opentelemetry/exporter-metrics-otlp-http` for metrics. Traces and logs work fine with the
`-http` (JSON) exporters.
</Note>
Then, configure the exporters in your `trigger.config.ts` file:
```ts trigger.config.ts
import { defineConfig } from "@trigger.dev/sdk";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
// Initialize OTLP trace exporter with the endpoint URL and headers;
export default defineConfig({
@@ -196,18 +205,28 @@ export default defineConfig({
},
}),
],
metricExporters: [
new OTLPMetricExporter({
url: "https://api.axiom.co/v1/metrics",
headers: {
Authorization: `Bearer ${process.env.AXIOM_API_TOKEN}`,
"x-axiom-metrics-dataset": process.env.AXIOM_METRICS_DATASET,
},
}),
],
},
});
```
Make sure to set the `AXIOM_API_TOKEN` and `AXIOM_DATASET` environment variables in your project.
Make sure to set the `AXIOM_API_TOKEN`, `AXIOM_DATASET`, and `AXIOM_METRICS_DATASET` environment variables in your project. Axiom requires a separate, dedicated dataset for metrics — you cannot reuse the same dataset for traces/logs and metrics.
It's important to note that you cannot configure exporters using `OTEL_*` environment variables, as they would conflict with our internal telemetry. Instead you should configure the exporters via passing in arguments to the `OTLPTraceExporter` and `OTLPLogExporter` constructors. For example, here is how you can configure exporting to Honeycomb:
It's important to note that you cannot configure exporters using `OTEL_*` environment variables, as they would conflict with our internal telemetry. Instead you should configure the exporters via passing in arguments to the `OTLPTraceExporter`, `OTLPLogExporter`, and `OTLPMetricExporter` constructors. For example, here is how you can configure exporting to Honeycomb:
```ts trigger.config.ts
import { defineConfig } from "@trigger.dev/sdk";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
// Initialize OTLP trace exporter with the endpoint URL and headers;
export default defineConfig({
@@ -235,6 +254,15 @@ export default defineConfig({
},
}),
],
metricExporters: [
new OTLPMetricExporter({
url: "https://api.honeycomb.io/v1/metrics",
headers: {
"x-honeycomb-team": process.env.HONEYCOMB_API_KEY,
"x-honeycomb-dataset": process.env.HONEYCOMB_DATASET,
},
}),
],
},
});
```
@@ -465,8 +493,9 @@ export default defineConfig({
```
<Note>
Any packages that install or build a native binary or use WebAssembly (WASM) should be added to external, as they
cannot be bundled. For example, `re2`, `sharp`, `sqlite3`, and WASM packages should be added to external.
Any packages that install or build a native binary or use WebAssembly (WASM) should be added to
external, as they cannot be bundled. For example, `re2`, `sharp`, `sqlite3`, and WASM packages
should be added to external.
</Note>
### JSX
docs/docs.json
+20 -5
View File
@@ -179,6 +179,10 @@
}
]
},
{
"group": "Insights",
"pages": ["insights/query", "insights/metrics"]
},
{
"group": "Using the Dashboard",
"pages": ["run-tests", "troubleshooting-alerts", "replaying", "bulk-actions"]
@@ -239,10 +243,7 @@
},
{
"group": "Batches API",
"pages": [
"management/batches/create",
"management/batches/stream-items"
]
"pages": ["management/batches/create", "management/batches/stream-items"]
},
{
"group": "Runs API",
@@ -255,6 +256,16 @@
"management/runs/update-metadata"
]
},
{
"group": "Queues API",
"pages": [
"management/queues/list",
"management/queues/retrieve",
"management/queues/pause",
"management/queues/concurrency-override",
"management/queues/concurrency-reset"
]
},
{
"group": "Schedules API",
"pages": [
@@ -286,6 +297,10 @@
"management/deployments/get-latest",
"management/deployments/promote"
]
},
{
"group": "Query API",
"pages": ["management/query/execute"]
}
]
},
@@ -685,4 +700,4 @@
"destination": "/migrating-from-v3"
}
]
}
}
docs/images/metrics-built-in.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 711 KiB

docs/images/query-chart-usage-percentiles.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 532 KiB

docs/insights/metrics.mdx
+102
View File
@@ -0,0 +1,102 @@
---
title: "Metrics dashboards"
description: "Create custom dashboards with real-time metrics powered by TRQL queries."
---
## Overview
In the Trigger.dev dashboard we have built-in dashboards and you can create your own.
Metrics dashboards are powered by [TRQL queries](/insights/query) with widgets that can be displayed as charts, tables, or single values. They automatically refresh to show the latest data.
### Available metrics data
Trigger.dev automatically collects process metrics (CPU, memory) and Node.js runtime metrics (event loop, heap) for all deployed tasks -- no configuration needed. Requires SDK version **4.4.1 or later**. You can also create custom metrics using the `otel.metrics` API from the SDK.
All of this data is available in the `metrics` table for use in dashboard widgets. See [Logging, tracing & metrics](/logging#metrics) for the full list of automatic metrics and how to create custom ones, or the [Query page](/insights/query#metrics-table-columns) for the `metrics` table schema.
![The built-in Metrics dashboard](/images/metrics-built-in.png)
### Visualization types
- **Line chart** - Show trends over time
- **Bar chart** - Compare values across categories
- **Area chart** - Display cumulative trends
- **Table** - Show detailed data in rows
- **Single value** - Display a single metric (count, sum, average, etc.)
You can also add Titles to your dashboard.
## Filtering and time ranges
All widgets on a dashboard use the time range filter applied to the dashboard.
You can also filter the data by:
- Scope: Environment, Project, Organization
- Tasks
- Queues
## Creating custom metrics dashboards
1. In the sidebar click the + icon next to "Metrics".
2. Name your custom dashboard.
3. From the top-right you can "Add chart" or "Add title".
4. For charts you write [TRQL queries](/insights/query) and choose a visualization type.
5. You can resize and reposition widgets on your dashboards.
## Performance considerations
### Optimize queries for metrics
1. **Use time bucketing** - `timeBucket()` automatically groups by appropriate intervals
2. **Limit result size** - Add `LIMIT` clauses, especially for table widgets
3. **Use approximate functions** - `uniq()` instead of `uniqExact()` for faster approximate counts
## Exporting metric data
Export data from any metric widget:
1. Click the widget menu (three dots)
2. Select "Copy JSON" or "Copy CSV"
## Best practices
1. **Start simple** - Begin with basic metrics and iterate based on insights
2. **Use meaningful names** - Give widgets clear, descriptive titles
3. **Group related metrics** - Organize dashboards by theme (performance, costs, errors)
4. **Test queries first** - Use the Query page to develop and test before adding to dashboards
## Troubleshooting
### Widget shows "No data"
- Check that your query returns results in the Query page
- Verify time filters include the period with data
- Ensure task/queue filters match existing runs
### Widget is slow to load
- Add time range filters to your query
- Use `LIMIT` clauses
- Simplify aggregations
- Check query execution time in Query page
### Chart displays incorrectly
- Verify column names match visualization config
- Check data types (numbers for charts, dates for time series)
- Ensure `timeBucket()` is used for time-series charts
- Review that series columns exist in query results
## Limits
Metrics is powered by Query so have [the same limits](/insights/query#limits) as Query.
There is a separate concurrency limits for metric widgets.
| Limit | Details |
| :------------------------ | :------------- |
| Concurrent widget queries | 30 per project |
See [Limits](/limits) for details.
docs/insights/query.mdx
+581
View File
@@ -0,0 +1,581 @@
---
title: "Query"
description: "Query allows you to write custom queries against your data using TRQL (Trigger.dev Query Language), a SQL-style language based on ClickHouse SQL. You can query your data through the dashboard, SDK, or REST API."
---
### Available tables
- `runs`: contains all task run data including status, timing, costs, and metadata
- `metrics`: contains metrics data for your runs including CPU, memory, and your custom metrics
### `metrics` table columns
| Column | Type | Description |
| :--- | :--- | :--- |
| `metric_name` | string | Metric identifier (e.g., `process.cpu.utilization`) |
| `metric_type` | string | `gauge`, `sum`, or `histogram` |
| `value` | number | The observed value |
| `bucket_start` | datetime | 10-second aggregation bucket start time |
| `run_id` | string | Associated run ID |
| `task_identifier` | string | Task slug |
| `attempt_number` | number | Attempt number |
| `machine_id` | string | Machine that produced the metric |
| `machine_name` | string | Machine preset (e.g., `small-1x`) |
| `worker_version` | string | Worker version |
| `environment_type` | string | `PRODUCTION`, `STAGING`, `DEVELOPMENT`, `PREVIEW` |
| `attributes` | json | Raw JSON attributes for custom data |
See [Logging, tracing & metrics](/logging#automatic-system-and-runtime-metrics) for the full list of automatically collected metrics and how to create custom metrics.
### `prettyFormat()`
Use `prettyFormat()` to format metric values for display:
```sql
SELECT
timeBucket(),
prettyFormat(avg(value), 'bytes') AS avg_memory
FROM metrics
WHERE metric_name = 'process.memory.usage'
GROUP BY timeBucket
ORDER BY timeBucket
LIMIT 1000
```
Available format types: `bytes`, `percent`, `duration`, `durationSeconds`, `quantity`, `costInDollars`.
## Using the Query dashboard
Navigate to the Query page to write and execute queries. The dashboard provides:
- **AI-powered query generation** - Describe what you want in natural language
- **Syntax highlighting** - SQL syntax highlighting for better readability
- **Query history** - Access your previous queries
- **Interactive help** - Built-in documentation for TRQL syntax and functions
- **Export options** - Download results as JSON or CSV
![The Query dashboard](/images/query-chart-usage-percentiles.png)
## Querying from the SDK
Use `query.execute()` to run TRQL queries programmatically from your backend code:
```typescript
import { query } from "@trigger.dev/sdk";
// Basic query with defaults (environment scope, json format)
const result = await query.execute("SELECT run_id, status FROM runs LIMIT 10");
console.log(result.results); // Array<Record<string, any>>
```
### Type-safe queries
Use the `QueryTable` type for nice inferred types in your query results:
```typescript
import { query, type QueryTable } from "@trigger.dev/sdk";
// Type-safe query using QueryTable with specific columns
const typedResult = await query.execute<QueryTable<"runs", "run_id" | "status" | "triggered_at">>(
"SELECT run_id, status, triggered_at FROM runs LIMIT 10"
);
typedResult.results.forEach((row) => {
console.log(row.run_id, row.status); // Fully typed!
});
```
### Query options
```typescript
import { query } from "@trigger.dev/sdk";
const result = await query.execute("SELECT COUNT(*) as count FROM runs", {
// Scope: "environment" (default), "project", or "organization"
scope: "project",
// Time period using shorthand (e.g., "7d", "30d", "1h")
period: "7d",
// Or use explicit time range
// from: new Date("2024-01-01"),
// to: new Date("2024-01-31"),
// Response format: "json" (default) or "csv"
format: "json",
});
```
### CSV export
Export query results as CSV by setting `format: "csv"`:
```typescript
const csvResult = await query.execute("SELECT run_id, status, triggered_at FROM runs", {
format: "csv",
period: "7d",
});
const lines = csvResult.results.split("\n");
console.log(lines[0]); // CSV header row
```
## Querying from the REST API
Execute queries via HTTP POST to `/api/v1/query`:
```sh
curl -X POST https://api.trigger.dev/api/v1/query \
-H "Authorization: Bearer YOUR_SECRET_KEY" \
-H "Content-Type: application/json" \
-d '{
"query": "SELECT run_id, status FROM runs LIMIT 10",
"scope": "environment",
"period": "7d",
"format": "json"
}'
```
See the [API reference](/management/query/execute) for full details.
## TRQL syntax guide
### Basic queries
Select columns from a table:
```sql
SELECT run_id, task_identifier, status
FROM runs
LIMIT 10
```
Alias columns with `AS`:
```sql
SELECT task_identifier AS task, count() AS total
FROM runs
GROUP BY task
```
### Using \*
Note that when you use `SELECT *` we don't return all the columns, we only return the core columns. This is for performance reasons (the underlying ClickHouse database is columnar and selecting lots of columns isn't efficient).
You should specify the columns you want to return.
### Filtering with WHERE
Use comparison operators:
```sql
SELECT run_id, task_identifier FROM runs
WHERE status = 'Failed'
```
Available operators:
```sql
-- Comparison operators
WHERE status = 'Failed' -- Equal
WHERE status != 'Completed' -- Not equal
WHERE attempt_count > 3 -- Greater than
WHERE attempt_count >= 3 -- Greater than or equal
WHERE attempt_count < 5 -- Less than
WHERE attempt_count <= 5 -- Less than or equal
-- IN for multiple values
WHERE status IN ('Failed', 'Crashed')
-- LIKE for pattern matching (% = wildcard)
WHERE task_identifier LIKE 'email%'
-- ILIKE for case-insensitive matching
WHERE task_identifier ILIKE '%send%'
-- BETWEEN for ranges
WHERE triggered_at BETWEEN '2024-01-01' AND '2024-01-31'
-- NULL checks
WHERE completed_at IS NOT NULL
WHERE completed_at IS NULL
-- Array column checks
WHERE has(tags, 'user_12345')
WHERE notEmpty(tags)
WHERE hasAny(tags, array('user_12345', 'user_67890'))
WHERE hasAll(tags, array('user_12345', 'user_67890'))
WHERE indexOf(tags, 'user_12345') > 0
WHERE arrayElement(tags, 1) = 'user_12345'
```
### Sorting and limiting
Sort results with `ORDER BY`:
```sql
SELECT run_id, compute_cost, triggered_at
FROM runs
ORDER BY compute_cost DESC, triggered_at ASC
LIMIT 50
```
### Grouping and aggregation
Use `GROUP BY` with aggregate functions:
```sql
SELECT
task_identifier,
avg(value) AS avg_memory
FROM metrics
WHERE metric_name = 'process.memory.usage'
GROUP BY task_identifier
ORDER BY avg_memory DESC
LIMIT 20
```
## Available functions
TRQL provides a rich set of functions for data analysis.
### Aggregate functions
- `count()` - Count rows
- `countIf(col, cond)` - Count rows matching condition
- `countDistinct(col)` - Count unique values
- `sum(col)` - Sum of values
- `sumIf(col, cond)` - Sum values matching condition
- `avg(col)` - Average of values
- `min(col)` - Minimum value
- `max(col)` - Maximum value
- `median(col)` - Median value (50th percentile)
- `quantile(p)(col)` - Value at percentile p (0-1)
- `stddevPop(col)` - Population standard deviation
- `stddevSamp(col)` - Sample standard deviation
Example:
```sql
SELECT
task_identifier,
count() AS total_runs,
avg(usage_duration) AS avg_duration_ms,
median(usage_duration) AS median_duration_ms,
quantile(0.95)(usage_duration) AS p95_duration_ms
FROM runs
GROUP BY task_identifier
```
### Date/time functions
**Time bucketing:**
```sql
-- Auto-bucket by time period based on query's time range
SELECT timeBucket(), count() AS runs
FROM runs
GROUP BY timeBucket()
```
**Date extraction:**
```sql
SELECT
toYear(triggered_at) AS year,
toMonth(triggered_at) AS month,
toDayOfWeek(triggered_at) AS day_of_week,
toHour(triggered_at) AS hour
FROM runs
```
**Date truncation:**
```sql
SELECT
toStartOfDay(triggered_at) AS day,
count() AS runs_per_day
FROM runs
GROUP BY day
ORDER BY day DESC
```
**Date arithmetic:**
```sql
-- Add/subtract time
SELECT dateAdd('day', 7, triggered_at) AS week_later
FROM runs
-- Calculate differences
SELECT dateDiff('minute', executed_at, completed_at) AS duration_minutes
FROM runs
WHERE completed_at IS NOT NULL
```
Common date functions:
- `now()` - Current date and time
- `today()` - Current date
- `toDate(dt)` - Convert to date
- `toStartOfDay(dt)`, `toStartOfHour(dt)`, `toStartOfMonth(dt)` - Truncate to start of period
- `formatDateTime(dt, format)` - Format datetime as string
### String functions
```sql
SELECT
lower(status) AS status_lower,
upper(status) AS status_upper,
concat(task_identifier, '-', status) AS combined,
substring(run_id, 1, 8) AS short_id,
length(task_identifier) AS name_length
FROM runs
```
Common string functions:
- `length(s)` - String length
- `lower(s)`, `upper(s)` - Case conversion
- `concat(s1, s2, ...)` - Concatenate strings
- `substring(s, offset, len)` - Extract substring
- `trim(s)` - Remove whitespace
- `replace(s, from, to)` - Replace occurrences
- `startsWith(s, prefix)`, `endsWith(s, suffix)` - Check prefixes/suffixes
### Conditional functions
```sql
SELECT
run_id,
if(status = 'Failed', 1, 0) AS is_failed,
multiIf(
status = 'Completed', 'ok',
status = 'Failed', 'bad',
'other'
) AS status_category,
coalesce(completed_at, triggered_at) AS end_time
FROM runs
```
- `if(cond, then, else)` - Conditional expression
- `multiIf(c1, t1, c2, t2, ..., else)` - Multiple conditions (like CASE)
- `coalesce(a, b, ...)` - First non-null value
### Math functions
```sql
SELECT
round(compute_cost, 4) AS cost_rounded,
ceil(usage_duration / 1000) AS duration_seconds_up,
floor(usage_duration / 1000) AS duration_seconds_down,
abs(compute_cost) AS cost_abs
FROM runs
```
### Array functions
Useful for working with tags and other array columns:
```sql
SELECT
run_id,
tags,
length(tags) AS tag_count,
has(tags, 'user_12345') AS is_production,
arrayJoin(tags) AS individual_tag -- Expand array to rows
FROM runs
WHERE notEmpty(tags)
```
### JSON functions
Extract data from JSON columns (like runs.output, runs.error, metrics.attributes, etc.):
```sql
SELECT
run_id,
output.message AS output_message,
output.count AS count,
output.error != NULL AS has_error
FROM runs
WHERE output IS NOT NULL
```
## Query scopes
Control what data your query can access:
- **`environment`** (default) - Query runs in the current environment only
- **`project`** - Query runs across all environments in the project
- **`organization`** - Query runs across all projects in the organization
```typescript
// Query across all environments in a project
const result = await query.execute("SELECT environment, count() FROM runs GROUP BY environment", {
scope: "project",
});
```
## Time ranges
We recommend avoiding adding `triggered_at` in the actual TRQL query. The dashboard, API, and SDK have a time filter that is applied automatically and is easier to work with. It means the queries can be executed with multiple periods easily.
### Using period shorthand
```typescript
await query.execute("SELECT count() FROM runs", {
period: "4d", // Last 4 days
});
// Supported periods: "1h", "6h", "12h", "1d", "7d", "30d", "90d", etc.
```
### Using explicit dates
```typescript
await query.execute("SELECT count() FROM runs", {
from: new Date("2024-01-01"),
to: new Date("2024-01-31"),
});
// Or use Unix timestamps
await query.execute("SELECT count() FROM runs", {
from: Date.now() - 7 * 24 * 60 * 60 * 1000, // 7 days ago
to: Date.now(),
});
```
## Example queries
### Failed runs (in the last 24 hours)
```sql
SELECT
task_identifier,
run_id,
error,
triggered_at
FROM runs
WHERE status = 'Failed'
ORDER BY triggered_at DESC
```
With the time filter set to 24h.
### Task success rate by day
```sql
SELECT
toDate(triggered_at) AS day,
task_identifier,
countIf(status = 'Completed') AS completed,
countIf(status = 'Failed') AS failed,
round(completed / (completed + failed) * 100, 2) AS success_rate_pct
FROM runs
WHERE status IN ('Completed', 'Failed')
GROUP BY day, task_identifier
ORDER BY day DESC, task_identifier
```
### Top 10 most expensive runs
```sql
SELECT
run_id,
task_identifier,
compute_cost,
usage_duration,
triggered_at
FROM runs
WHERE compute_cost > 0
ORDER BY compute_cost DESC
LIMIT 10
```
### Average compute duration over time
```sql
SELECT
timeBucket() AS time,
task_identifier,
avg(usage_duration) AS avg_duration_ms,
count() AS run_count
FROM runs
WHERE usage_duration IS NOT NULL
GROUP BY time, task_identifier
ORDER BY time ASC
```
### Runs by queue and machine
```sql
SELECT
queue,
machine,
count() AS run_count,
countIf(status = 'Completed') AS completed,
countIf(status = 'Failed') AS failed
FROM runs
GROUP BY queue, machine
ORDER BY queue, machine
```
### CPU utilization over time
Track process CPU utilization bucketed over time.
```sql
SELECT
timeBucket(),
avg(value) AS avg_cpu
FROM metrics
WHERE metric_name = 'process.cpu.utilization'
GROUP BY timeBucket
ORDER BY timeBucket
LIMIT 1000
```
### Memory usage by task (past 7d)
Average process memory usage per task identifier over the last 7 days.
```sql
SELECT
task_identifier,
avg(value) AS avg_memory
FROM metrics
WHERE metric_name = 'process.memory.usage'
GROUP BY task_identifier
ORDER BY avg_memory DESC
LIMIT 20
```
### Available metric names
List all distinct metric names collected in your environment.
```sql
SELECT
metric_name,
count() AS sample_count
FROM metrics
GROUP BY metric_name
ORDER BY sample_count DESC
LIMIT 100
```
## Best practices
1. **Use the built-in time filtering** - The dashboard, API, and SDK have a time filter that is applied automatically and is easier to work with. It means the queries can be executed with multiple periods easily.
2. **Use LIMIT** - Add a `LIMIT` clause to reduce the rows returned if you don't need everything.
3. **Use appropriate aggregations** - For large datasets, use `uniq()` instead of `uniqExact()` for approximate but faster counts
## Limits
We have several limits to prevent abuse and ensure performance:
- **Concurrency limit**: We limit the number of concurrent queries per organization.
- **Row limit**: We limit the number of rows returned to 10k.
- **Time restrictions**: We limit the time period you can query.
- **Time/Memory limit**: We limit the memory a query can use and the time it can run for. As well as other limits like AST complexity.
See [Limits](/limits) for current quota details.
docs/limits.mdx
+35 -3
View File
@@ -57,8 +57,8 @@ If you're creating schedules for your user you will definitely need to request m
## Projects
| Pricing tier | Limit |
| :----------- | :----------------- |
| Pricing tier | Limit |
| :----------- | :------------------ |
| All tiers | 10 per organization |
Each project receives its own concurrency allocation. If you need to support multiple tenants with the same codebase but different environment variables, see the [Multi-tenant applications](/deploy-environment-variables#multi-tenant-applications) section for a recommended workaround.
@@ -112,7 +112,9 @@ Batch triggering uses a token bucket algorithm to rate limit the number of runs
**How it works**: You can burst up to your bucket size, then tokens refill at the specified rate. For example, a Free user can trigger 1,200 runs immediately, then must wait for tokens to refill (100 runs become available every 10 seconds).
<Note>
When you hit batch rate limits, the SDK throws a `BatchTriggerError` with `isRateLimited: true`. See [Handling batch trigger errors](/triggering#handling-batch-trigger-errors) for how to detect and react to rate limits in your code.
When you hit batch rate limits, the SDK throws a `BatchTriggerError` with `isRateLimited: true`.
See [Handling batch trigger errors](/triggering#handling-batch-trigger-errors) for how to detect
and react to rate limits in your code.
</Note>
## Batch processing concurrency
@@ -186,6 +188,36 @@ An alert destination is a single email address, Slack channel, or webhook URL th
If you're on the Pro plan and need more than the plan limit, you can request more by contacting us via [email](https://trigger.dev/contact) or [Discord](https://trigger.dev/discord).
## Query
Query execution is subject to the following limits:
| Limit | Details |
| :----------------- | :-------------------- |
| Max execution time | 10 seconds per query |
| Max result rows | 10,000 rows per query |
| Concurrent queries | 3 per project |
### Query lookback period
The maximum time range a query can look back is based on your plan:
| Pricing tier | Limit |
| :----------- | :------ |
| Free | 1 day |
| Hobby | 7 days |
| Pro | 30 days |
If your query's time range exceeds your plan's lookback limit, it will be automatically clipped to the maximum allowed period.
## Metric widget concurrency
The number of metric widgets that can be queried concurrently per project.
| Limit | Details |
| :------------------------ | :------------- |
| Concurrent widget queries | 30 per project |
## Machines
The default machine is `small-1x` which has 0.5 vCPU and 0.5 GB of RAM. You can optionally configure a higher spec machine which will increase the cost of running the task but can also improve the performance of the task if it is CPU or memory bound.
docs/logging.mdx
+118 -2
View File
@@ -1,6 +1,6 @@
---
title: "Logging and tracing"
description: "How to use the built-in logging and tracing system."
title: "Logging, tracing & metrics"
description: "How to use the built-in logging, tracing, and metrics system."
---
![The run log](/images/run-log.png)
@@ -77,3 +77,119 @@ export const customTrace = task({
},
});
```
## Metrics
Trigger.dev collects system and runtime metrics automatically for deployed tasks, and provides an API for recording custom metrics using OpenTelemetry.
You can view metrics in the [Metrics dashboards](/insights/metrics), query them with [TRQL](/insights/query), and export them to external services via [telemetry exporters](/config/config-file#telemetry-exporters).
### Custom metrics API
Import `otel` from `@trigger.dev/sdk` and use the standard OpenTelemetry Metrics API to create custom instruments.
Create instruments **at module level** (outside the task `run` function) so they are reused across runs:
```ts /trigger/metrics.ts
import { task, logger, otel } from "@trigger.dev/sdk";
// Create a meter — instruments are created once at module level
const meter = otel.metrics.getMeter("my-app");
const itemsProcessed = meter.createCounter("items.processed", {
description: "Total number of items processed",
unit: "items",
});
const itemDuration = meter.createHistogram("item.duration", {
description: "Time spent processing each item",
unit: "ms",
});
const queueDepth = meter.createUpDownCounter("queue.depth", {
description: "Current queue depth",
unit: "items",
});
export const processQueue = task({
id: "process-queue",
run: async (payload: { items: string[] }) => {
queueDepth.add(payload.items.length);
for (const item of payload.items) {
const start = performance.now();
// ... process item ...
const elapsed = performance.now() - start;
itemsProcessed.add(1, { "item.type": "order" });
itemDuration.record(elapsed, { "item.type": "order" });
queueDepth.add(-1);
}
logger.info("Queue processed", { count: payload.items.length });
},
});
```
#### Available instrument types
| Instrument | Method | Use case |
| :--- | :--- | :--- |
| Counter | `meter.createCounter()` | Monotonically increasing values (items processed, requests sent) |
| Histogram | `meter.createHistogram()` | Distributions of values (durations, sizes) |
| UpDownCounter | `meter.createUpDownCounter()` | Values that go up and down (queue depth, active connections) |
All instruments accept optional attributes when recording values. Attributes let you break down metrics by dimension (e.g., by item type, status, or region).
### Automatic system and runtime metrics
Trigger.dev automatically collects the following metrics for deployed tasks. No configuration is needed. Requires SDK version **4.4.1 or later**.
| Metric name | Type | Unit | Description |
| :--- | :--- | :--- | :--- |
| `process.cpu.utilization` | gauge | ratio | Process CPU usage (0-1) |
| `process.cpu.time` | counter | seconds | CPU time consumed |
| `process.memory.usage` | gauge | bytes | Process memory usage |
| `nodejs.event_loop.utilization` | gauge | ratio | Event loop utilization (0-1) |
| `nodejs.event_loop.delay.p95` | gauge | seconds | Event loop delay p95 |
| `nodejs.event_loop.delay.max` | gauge | seconds | Event loop delay max |
| `nodejs.heap.used` | gauge | bytes | V8 heap used |
| `nodejs.heap.total` | gauge | bytes | V8 heap total |
<Note>
In dev mode (`trigger dev`), only `process.*` and custom metrics are available.
</Note>
### Context attributes
All metrics (both automatic and custom) are tagged with run context so you can filter and group them:
- `run_id` — the run that produced the metric
- `task_identifier` — the task slug
- `attempt_number` — the attempt number
- `machine_name` — the machine preset (e.g., `small-1x`)
- `worker_version` — the deployed worker version
- `environment_type` — `PRODUCTION`, `STAGING`, `DEVELOPMENT`, or `PREVIEW`
### Querying metrics
Use [TRQL](/insights/query) to query metrics data. For example, to see average CPU utilization over time:
```sql
SELECT
timeBucket(),
avg(value) AS avg_cpu
FROM metrics
WHERE metric_name = 'process.cpu.utilization'
GROUP BY timeBucket
ORDER BY timeBucket
LIMIT 1000
```
See the [Query page](/insights/query#metrics-table-columns) for the full `metrics` table schema.
### Exporting metrics
You can send metrics to external observability services (Axiom, Honeycomb, Datadog, etc.) by configuring [telemetry exporters](/config/config-file#telemetry-exporters) in your `trigger.config.ts`.
docs/management/query/execute.mdx
+11
View File
@@ -0,0 +1,11 @@
---
title: "Execute a query"
openapi: "v3-openapi POST /api/v1/query"
---
See the [Query documentation](/insights/query#example-queries) for comprehensive examples including:
- Failed runs analysis
- Task success rates over time
- Cost tracking and optimization
- Performance metrics and percentiles
docs/management/queues/concurrency-override.mdx
+4
View File
@@ -0,0 +1,4 @@
---
title: "Override Concurrency Limit"
openapi: "v3-openapi POST /api/v1/queues/{queueParam}/concurrency/override"
---
docs/management/queues/concurrency-reset.mdx
+4
View File
@@ -0,0 +1,4 @@
---
title: "Reset Concurrency Limit"
openapi: "v3-openapi POST /api/v1/queues/{queueParam}/concurrency/reset"
---
docs/management/queues/list.mdx
+4
View File
@@ -0,0 +1,4 @@
---
title: "List Queues"
openapi: "v3-openapi GET /api/v1/queues"
---
docs/management/queues/pause.mdx
+4
View File
@@ -0,0 +1,4 @@
---
title: "Pause or Resume Queue"
openapi: "v3-openapi POST /api/v1/queues/{queueParam}/pause"
---
docs/management/queues/retrieve.mdx
+4
View File
@@ -0,0 +1,4 @@
---
title: "Retrieve Queue"
openapi: "v3-openapi GET /api/v1/queues/{queueParam}"
---
docs/migrating-from-v3.mdx
+18
View File
@@ -6,6 +6,20 @@ description: "What's new in v4, how to migrate, and breaking changes."
import NodeVersions from "/snippets/node-versions.mdx";
import MigrateV4UsingAi from "/snippets/migrate-v4-using-ai.mdx";
<Warning>
**Action required: Trigger.dev v3 deprecation**
We're retiring Trigger.dev v3. **New v3 deploys will stop working from 1 April 2026.** Trigger.dev v4 is stable, fully supported, and recommended for all users.
**Key dates:**
- **1 April 2026** — New v3 deploys will no longer work. Existing v3 runs will continue to execute.
- **1 July 2026** — v3 will be fully shut down. All v3 runs will stop executing.
**What you need to do:** Migrate to v4 before April to avoid disruption to your task executions. The migration takes about 2 minutes — follow the steps on this page below. If you have questions or need help, [contact us](https://trigger.dev/contact) or reach out in our [Discord](https://trigger.dev/discord).
</Warning>
## What's new in v4?
| Feature | Description |
@@ -48,6 +62,10 @@ Note that between steps 4 and 5, runs triggered with the v4 package will continu
concurrency will return to normal.
</Warning>
<Note>
When migrating from v3 to v4, our infrastructure IPs may change. If you use IP allowlisting (e.g. for databases or APIs), [update your allowlists](https://trigger.dev/changelog/static-ips) with the current static IPs before or immediately after switching to v4 to avoid connectivity issues or downtime.
</Note>
## Migrate using AI
Use the prompt in the accordion below to help you migrate your v3 tasks to v4. The prompt gives good results when using Claude 4 Sonnet. You’ll need a relatively large token limit.
docs/queue-concurrency.mdx
+126
View File
@@ -226,3 +226,129 @@ export const subtask = task({
```
When the parent task reaches the `triggerAndWait` call, it checkpoints and transitions to the `WAITING` state, releasing its concurrency slot back to both its queue and the environment. Once the subtask completes, the parent task will resume and re-acquire a concurrency slot.
## Managing queues with the SDK
The SDK provides a `queues` namespace that allows you to manage queues programmatically. You can list, retrieve, pause, resume, and modify concurrency limits for queues.
<Note>
Import from `@trigger.dev/sdk`:
```ts
import { queues } from "@trigger.dev/sdk";
```
</Note>
### Listing queues
You can list all queues in your environment with pagination support:
```ts
import { queues } from "@trigger.dev/sdk";
// List all queues (returns paginated results)
const allQueues = await queues.list();
// With pagination options
const pagedQueues = await queues.list({
page: 1,
perPage: 20,
});
```
### Retrieving a queue
You can retrieve a specific queue by its ID, or by its type and name:
```ts
import { queues } from "@trigger.dev/sdk";
// Using queue ID (starts with "queue_")
const queueById = await queues.retrieve("queue_1234");
// Using type and name for a task's default queue
const taskQueue = await queues.retrieve({
type: "task",
name: "my-task-id",
});
// Using type and name for a custom queue
const customQueue = await queues.retrieve({
type: "custom",
name: "my-custom-queue",
});
```
The queue object contains useful information about the queue state:
```ts
{
id: "queue_1234", // Queue ID
name: "my-task-id", // Queue name
type: "task", // "task" or "custom"
running: 5, // Currently executing runs
queued: 10, // Runs waiting to execute
paused: false, // Whether the queue is paused
concurrencyLimit: 10, // Current concurrency limit
concurrency: {
current: 10, // Effective limit
base: 10, // Default limit from code
override: null, // Override value (if set)
overriddenAt: null, // When override was applied
overriddenBy: null, // Who applied the override
}
}
```
### Pausing and resuming queues
You can pause a queue to prevent new runs from starting. Runs that are currently executing will continue to completion.
```ts
import { queues } from "@trigger.dev/sdk";
// Pause a queue using its ID
await queues.pause("queue_1234");
// Or using type and name
await queues.pause({ type: "task", name: "my-task-id" });
await queues.pause({ type: "custom", name: "my-custom-queue" });
```
To resume a paused queue and allow new runs to start:
```ts
import { queues } from "@trigger.dev/sdk";
// Resume a queue using its ID
await queues.resume("queue_1234");
// Or using type and name
await queues.resume({ type: "task", name: "my-task-id" });
await queues.resume({ type: "custom", name: "my-custom-queue" });
```
### Overriding concurrency limits
You can temporarily override a queue's concurrency limit. This is useful for scaling up or down based on demand:
```ts
import { queues } from "@trigger.dev/sdk";
// Set concurrency limit to 5
await queues.overrideConcurrencyLimit("queue_1234", 5);
// Or using type and name
await queues.overrideConcurrencyLimit({ type: "task", name: "my-task-id" }, 20);
```
To reset the concurrency limit back to the base value defined in your code:
```ts
import { queues } from "@trigger.dev/sdk";
// Reset concurrency limit to the base value
await queues.resetConcurrencyLimit("queue_1234");
// Or using type and name
await queues.resetConcurrencyLimit({ type: "task", name: "my-task-id" });
```
docs/snippets/migrate-v4-using-ai.mdx
+11 -5
View File
@@ -151,7 +151,7 @@ await myTask.trigger({ foo: "bar" });
await myTask.trigger({ foo: "bar" }, { queue: "my-queue" });
**Lifecycle hooks**: Function signatures have changed to use a single object parameter instead of separate parameters. This is the old version:
**Lifecycle hooks**: Function signatures have changed to use a single object parameter instead of separate parameters. Prefer `onStartAttempt` over the deprecated `onStart` when you need code to run before each attempt. This is the old version:
// Old v3 way
@@ -171,10 +171,10 @@ This is the new version:
// New v4 way - single object parameter for hooks
export const myTask = task({
id: "my-task",
onStart: ({ payload, ctx }) => {},
onSuccess: ({ payload, output, ctx }) => {},
onFailure: ({ payload, error, ctx }) => {},
catchError: ({ payload, ctx, error, retry }) => {},
onStartAttempt: ({ payload, ctx }) => {}, // prefer over deprecated onStart
onSuccess: ({ payload, ctx, task, output }) => {},
onFailure: ({ payload, ctx, task, error }) => {},
catchError: ({ payload, ctx, task, error, retry, retryAt, retryDelayInMs }) => {},
run: async (payload, { ctx }) => {}, // run function unchanged
});
@@ -204,6 +204,12 @@ const batch = await batch.retrieve(batchHandle.batchId); // Use batch.retrieve()
console.log(batch.runs);
**triggerAndWait / batchTriggerAndWait**: In v4 these return a Result object, not the raw output. Use `if (result.ok) { ... result.output }` or call `.unwrap()` to get the output (throws if the run failed). Do not wrap `triggerAndWait` or `batchTriggerAndWait` in `Promise.all` — this is not supported.
**Context (ctx) changes**: `ctx.attempt.id` and `ctx.attempt.status` have been removed; use `ctx.attempt.number` where needed. `ctx.task.exportName` has been removed.
Can you help me convert the following code from v3 to v4? Please include the full converted code in the answer, do not truncate it anywhere.
```
docs/triggering.mdx
+48 -1
View File
@@ -867,8 +867,9 @@ await myTask.trigger({ updated: "data" }, { debounce: { key: "user-123", delay:
The `debounce` option accepts:
- `key` - A unique string to identify the debounce group (scoped to the task)
- `delay` - Duration string specifying how long to delay (e.g., "5s", "1m", "30s")
- `delay` - Duration string specifying how long to delay. Supported units: `s` (seconds), `m` (minutes), `h`/`hr` (hours), `d` (days), `w` (weeks). Minimum is 1 second. Examples: `"5s"`, `"1m"`, `"2h30m"`
- `mode` - Optional. Controls which trigger's data is used: `"leading"` (default) or `"trailing"`
- `maxDelay` - Optional. Maximum total time from the first trigger before the run must execute. Uses the same duration format as `delay`
**How it works:**
@@ -877,6 +878,52 @@ The `debounce` option accepts:
3. Once no new triggers occur within the delay duration, the run executes
4. After the run starts executing, a new trigger with the same key will create a new run
**Limiting total delay with `maxDelay`:**
By default, continuous triggers can delay execution indefinitely. The `maxDelay` option sets an upper bound on the total delay from the first trigger, ensuring the run eventually executes even with constant activity.
```ts
await summarizeChat.trigger(
{ conversationId: "123" },
{
debounce: {
key: "conversation-123",
delay: "10s", // Wait 10s after each message
maxDelay: "5m", // But always run within 5 minutes of first trigger
},
}
);
```
This is useful for scenarios like:
- Summarizing AI chat threads that need periodic updates even during active conversations
- Syncing data that should happen regularly despite continuous changes
- Any case where you want debouncing but also guarantee timely execution
**Timeline example with `maxDelay`:**
Consider `delay: "5s"` and `maxDelay: "30s"` with triggers arriving every 2 seconds:
| Time | Event | Result |
| :--- | :--- | :--- |
| 0s | Trigger 1 | Run A created, scheduled for 5s |
| 2s | Trigger 2 | Run A rescheduled to 7s |
| 4s | Trigger 3 | Run A rescheduled to 9s |
| ... | ... | ... |
| 26s | Trigger 14 | Run A rescheduled to 31s |
| 28s | Trigger 15 | Would reschedule to 33s, but exceeds maxDelay (30s). Run A executes, Run B created |
| 30s | Trigger 16 | Run B rescheduled to 35s |
Without `maxDelay`, continuous triggers would prevent the run from ever executing. With `maxDelay: "30s"`, execution is guaranteed within 30 seconds of the first trigger.
<Note>
The `maxDelay` value is evaluated from each trigger call, not stored with the original run. This
means if you pass different `maxDelay` values for the same debounce key, each trigger uses its own
`maxDelay` to check against the original run's creation time. For consistent behavior, use the
same `maxDelay` value for all triggers with the same debounce key.
</Note>
**Leading vs Trailing mode:**
By default, debounce uses **leading mode** - the run executes with data from the **first** trigger.
docs/v3-openapi.yaml
+582 -2
View File
@@ -530,7 +530,17 @@ paths:
description: The deployment ID
status:
type: string
enum: ["PENDING", "INSTALLING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED", "TIMED_OUT"]
enum:
[
"PENDING",
"INSTALLING",
"BUILDING",
"DEPLOYING",
"DEPLOYED",
"FAILED",
"CANCELED",
"TIMED_OUT",
]
description: The current status of the deployment
contentHash:
type: string
@@ -622,7 +632,17 @@ paths:
description: The deployment ID
status:
type: string
enum: ["PENDING", "INSTALLING", "BUILDING", "DEPLOYING", "DEPLOYED", "FAILED", "CANCELED", "TIMED_OUT"]
enum:
[
"PENDING",
"INSTALLING",
"BUILDING",
"DEPLOYING",
"DEPLOYED",
"FAILED",
"CANCELED",
"TIMED_OUT",
]
description: The current status of the deployment
contentHash:
type: string
@@ -733,6 +753,122 @@ paths:
-H "Authorization: Bearer tr_dev_1234" \
-H "Content-Type: application/json"
"/api/v1/query":
post:
operationId: execute_query_v1
summary: Execute a TRQL query
description: Execute a TRQL (Trigger.dev Query Language) query against your run data. TRQL is a SQL-style query language that allows you to analyze runs, calculate metrics, and export data.
requestBody:
required: true
content:
application/json:
schema:
"$ref": "#/components/schemas/ExecuteQueryRequestBody"
responses:
"200":
description: Query executed successfully
content:
application/json:
schema:
"$ref": "#/components/schemas/ExecuteQueryResponse"
"400":
description: Invalid query or request parameters
content:
application/json:
schema:
type: object
properties:
error:
type: string
description: Error message describing the query error
"401":
description: Unauthorized - API key is missing or invalid
"500":
description: Internal server error during query execution
tags:
- query
security:
- secretKey: []
x-codeSamples:
- lang: typescript
label: SDK - Basic query
source: |-
import { query } from "@trigger.dev/sdk";
// Basic query with defaults (environment scope, json format)
const result = await query.execute(
"SELECT run_id, status FROM runs LIMIT 10"
);
console.log(result.results);
- lang: typescript
label: SDK - Type-safe query
source: |-
import { query, type QueryTable } from "@trigger.dev/sdk";
// Type-safe query using QueryTable
const result = await query.execute<
QueryTable<"runs", "run_id" | "status" | "triggered_at">
>(
"SELECT run_id, status, triggered_at FROM runs LIMIT 10"
);
result.results.forEach(row => {
console.log(row.run_id, row.status); // Fully typed!
});
- lang: typescript
label: SDK - With options
source: |-
import { query } from "@trigger.dev/sdk";
const result = await query.execute(
"SELECT COUNT(*) as count FROM runs WHERE status = 'Failed'",
{
scope: "project", // Query across all environments
period: "7d", // Last 7 days
format: "json"
}
);
- lang: typescript
label: SDK - CSV export
source: |-
import { query } from "@trigger.dev/sdk";
const csvResult = await query.execute(
"SELECT run_id, status, triggered_at FROM runs",
{
format: "csv",
period: "30d"
}
);
// csvResult.results is a CSV string
const lines = csvResult.results.split('\n');
- lang: curl
label: cURL - Basic query
source: |-
curl -X POST "https://api.trigger.dev/api/v1/query" \
-H "Authorization: Bearer tr_dev_1234" \
-H "Content-Type: application/json" \
-d '{
"query": "SELECT run_id, status FROM runs LIMIT 10",
"scope": "environment",
"period": "7d",
"format": "json"
}'
- lang: curl
label: cURL - Aggregation query
source: |-
curl -X POST "https://api.trigger.dev/api/v1/query" \
-H "Authorization: Bearer tr_dev_1234" \
-H "Content-Type: application/json" \
-d '{
"query": "SELECT task_identifier, count() as runs, countIf(status = '\''Failed'\'') as failures FROM runs GROUP BY task_identifier",
"scope": "environment",
"from": "2024-01-01T00:00:00Z",
"to": "2024-01-31T23:59:59Z",
"format": "json"
}'
"/api/v1/runs/{runId}/reschedule":
parameters:
- $ref: "#/components/parameters/runId"
@@ -1618,6 +1754,301 @@ paths:
]
}'
"/api/v1/queues":
get:
operationId: list_queues_v1
summary: List all queues
description: List all queues in your environment with pagination support.
parameters:
- in: query
name: page
schema:
type: integer
required: false
description: Page number of the queue listing (1-based)
- in: query
name: perPage
schema:
type: integer
required: false
description: Number of queues per page
responses:
"200":
description: Successful request
content:
application/json:
schema:
"$ref": "#/components/schemas/ListQueuesResult"
"401":
description: Unauthorized request
tags:
- queues
security:
- secretKey: []
x-codeSamples:
- lang: typescript
source: |-
import { queues } from "@trigger.dev/sdk";
// List all queues
const allQueues = await queues.list();
// With pagination
const pagedQueues = await queues.list({
page: 1,
perPage: 20,
});
"/api/v1/queues/{queueParam}":
get:
operationId: retrieve_queue_v1
summary: Retrieve a queue
description: Get a queue by its ID, or by type and name.
parameters:
- in: path
name: queueParam
required: true
schema:
type: string
description: The queue ID (e.g., `queue_1234`), or the name of the queue when using the `type` query parameter.
example: queue_1234
- in: query
name: type
schema:
type: string
enum: [id, task, custom]
default: id
required: false
description: |
How to interpret the `queueParam` path parameter:
- `id`: Treat as a queue ID (default)
- `task`: Treat as a task ID to get the task's default queue
- `custom`: Treat as a custom queue name
responses:
"200":
description: Successful request
content:
application/json:
schema:
"$ref": "#/components/schemas/QueueObject"
"401":
description: Unauthorized request
"404":
description: Queue not found
tags:
- queues
security:
- secretKey: []
x-codeSamples:
- lang: typescript
source: |-
import { queues } from "@trigger.dev/sdk";
// Using queue ID
const queue = await queues.retrieve("queue_1234");
// Using type and name for a task queue
const taskQueue = await queues.retrieve({
type: "task",
name: "my-task-id",
});
// Using type and name for a custom queue
const customQueue = await queues.retrieve({
type: "custom",
name: "my-custom-queue",
});
"/api/v1/queues/{queueParam}/pause":
post:
operationId: pause_queue_v1
summary: Pause or resume a queue
description: Pause a queue to prevent new runs from starting, or resume a paused queue. Runs that are currently executing will continue to completion.
parameters:
- in: path
name: queueParam
required: true
schema:
type: string
description: The queue ID (e.g., `queue_1234`), or the name of the queue when using the `type` body parameter.
example: queue_1234
requestBody:
required: true
content:
application/json:
schema:
type: object
required: ["action"]
properties:
type:
type: string
enum: [id, task, custom]
default: id
description: |
How to interpret the `queueParam` path parameter:
- `id`: Treat as a queue ID (default)
- `task`: Treat as a task ID to get the task's default queue
- `custom`: Treat as a custom queue name
action:
type: string
enum: [pause, resume]
description: Whether to pause or resume the queue
responses:
"200":
description: Queue paused or resumed successfully
content:
application/json:
schema:
"$ref": "#/components/schemas/QueueObject"
"400":
description: Invalid request parameters
"401":
description: Unauthorized request
"404":
description: Queue not found
tags:
- queues
security:
- secretKey: []
x-codeSamples:
- lang: typescript
source: |-
import { queues } from "@trigger.dev/sdk";
// Pause a queue
await queues.pause("queue_1234");
await queues.pause({ type: "task", name: "my-task-id" });
// Resume a queue
await queues.resume("queue_1234");
await queues.resume({ type: "task", name: "my-task-id" });
"/api/v1/queues/{queueParam}/concurrency/override":
post:
operationId: override_queue_concurrency_v1
summary: Override queue concurrency limit
description: Override the concurrency limit of a queue. This is useful for temporarily scaling up or down based on demand.
parameters:
- in: path
name: queueParam
required: true
schema:
type: string
description: The queue ID (e.g., `queue_1234`), or the name of the queue when using the `type` body parameter.
example: queue_1234
requestBody:
required: true
content:
application/json:
schema:
type: object
required: ["concurrencyLimit"]
properties:
type:
type: string
enum: [id, task, custom]
default: id
description: |
How to interpret the `queueParam` path parameter:
- `id`: Treat as a queue ID (default)
- `task`: Treat as a task ID to get the task's default queue
- `custom`: Treat as a custom queue name
concurrencyLimit:
type: integer
minimum: 0
maximum: 100000
description: The new concurrency limit to set for the queue
responses:
"200":
description: Concurrency limit overridden successfully
content:
application/json:
schema:
"$ref": "#/components/schemas/QueueObject"
"400":
description: Invalid request parameters
"401":
description: Unauthorized request
"404":
description: Queue not found
tags:
- queues
security:
- secretKey: []
x-codeSamples:
- lang: typescript
source: |-
import { queues } from "@trigger.dev/sdk";
// Override concurrency limit to 5
await queues.overrideConcurrencyLimit("queue_1234", 5);
// Using type and name
await queues.overrideConcurrencyLimit(
{ type: "task", name: "my-task-id" },
20
);
"/api/v1/queues/{queueParam}/concurrency/reset":
post:
operationId: reset_queue_concurrency_v1
summary: Reset queue concurrency limit
description: Reset the concurrency limit of a queue back to its base value defined in code.
parameters:
- in: path
name: queueParam
required: true
schema:
type: string
description: The queue ID (e.g., `queue_1234`), or the name of the queue when using the `type` body parameter.
example: queue_1234
requestBody:
required: false
content:
application/json:
schema:
type: object
properties:
type:
type: string
enum: [id, task, custom]
default: id
description: |
How to interpret the `queueParam` path parameter:
- `id`: Treat as a queue ID (default)
- `task`: Treat as a task ID to get the task's default queue
- `custom`: Treat as a custom queue name
responses:
"200":
description: Concurrency limit reset successfully
content:
application/json:
schema:
"$ref": "#/components/schemas/QueueObject"
"400":
description: Queue is not overridden or invalid request parameters
"401":
description: Unauthorized request
"404":
description: Queue not found
tags:
- queues
security:
- secretKey: []
x-codeSamples:
- lang: typescript
source: |-
import { queues } from "@trigger.dev/sdk";
// Reset concurrency limit to the base value
await queues.resetConcurrencyLimit("queue_1234");
// Using type and name
await queues.resetConcurrencyLimit({
type: "task",
name: "my-task-id",
});
components:
parameters:
taskIdentifier:
@@ -1763,6 +2194,97 @@ components:
minimum: 0
maximum: 1000
description: An optional property that specifies the maximum number of concurrent run executions. If this property is omitted, the task can potentially use up the full concurrency of an environment.
QueueObject:
type: object
required: ["id", "name", "type", "running", "queued", "paused"]
properties:
id:
type: string
description: The queue ID, e.g., `queue_1234`
example: queue_1234
name:
type: string
description: The queue name. For task queues, this is the task ID. For custom queues, this is the name you specified.
example: my-task-id
type:
type: string
enum: [task, custom]
description: |
The type of queue:
- `task`: Created automatically for each task
- `custom`: Created explicitly in your code using `queue()`
example: task
running:
type: integer
description: The number of runs currently executing
example: 5
queued:
type: integer
description: The number of runs currently queued
example: 10
paused:
type: boolean
description: Whether the queue is paused. When paused, no new runs will start.
example: false
concurrencyLimit:
type: integer
nullable: true
description: The current concurrency limit of the queue
example: 10
concurrency:
type: object
description: Detailed concurrency information
properties:
current:
type: integer
nullable: true
description: The effective/current concurrency limit
example: 10
base:
type: integer
nullable: true
description: The base concurrency limit defined in code
example: 10
override:
type: integer
nullable: true
description: The override concurrency limit (if set)
example: null
overriddenAt:
type: string
format: date-time
nullable: true
description: When the concurrency limit was overridden
example: null
overriddenBy:
type: string
nullable: true
description: Who overrode the concurrency limit (null if via API)
example: null
ListQueuesResult:
type: object
required: ["data", "pagination"]
properties:
data:
type: array
items:
"$ref": "#/components/schemas/QueueObject"
description: An array of queue objects
pagination:
type: object
properties:
currentPage:
type: integer
description: The current page number
example: 1
totalPages:
type: integer
description: The total number of pages
example: 5
count:
type: integer
description: The total number of queues
example: 50
BatchTriggerRequestBody:
type: object
properties:
@@ -2945,3 +3467,61 @@ components:
stackTrace:
type: string
example: "Error: Something went wrong"
ExecuteQueryRequestBody:
type: object
required:
- query
properties:
query:
type: string
description: The TRQL query to execute
example: "SELECT run_id, status, triggered_at FROM runs WHERE status = 'Failed' LIMIT 10"
scope:
type: string
enum: ["environment", "project", "organization"]
default: "environment"
description: The scope of data to query - environment (default), project, or organization
period:
type: string
nullable: true
description: Time period shorthand (e.g., "7d", "30d", "1h"). Cannot be used with from/to.
example: "7d"
from:
type: string
format: date-time
nullable: true
description: Start of time range as ISO 8601 timestamp. Must be used with 'to'.
example: "2024-01-01T00:00:00Z"
to:
type: string
format: date-time
nullable: true
description: End of time range as ISO 8601 timestamp. Must be used with 'from'.
example: "2024-01-31T23:59:59Z"
format:
type: string
enum: ["json", "csv"]
default: "json"
description: Response format - "json" returns structured data (default), "csv" returns CSV string
ExecuteQueryResponse:
oneOf:
- type: object
description: JSON format response
properties:
format:
type: string
enum: ["json"]
results:
type: array
items:
type: object
description: Array of result rows
- type: object
description: CSV format response
properties:
format:
type: string
enum: ["csv"]
results:
type: string
description: CSV-formatted results
docs/vercel-integration.mdx
+45 -18
View File
@@ -10,7 +10,8 @@ The Vercel integration connects your Vercel project to your Trigger.dev project
This eliminates the need to manually run the `trigger.dev deploy` command or maintain custom CI/CD workflows for Vercel-based projects.
<Note>
The Vercel integration requires the [GitHub integration](/github-integration) to be connected as well, since Trigger.dev builds your tasks from your GitHub repository.
The Vercel integration requires the [GitHub integration](/github-integration) to be connected as
well, since Trigger.dev builds your tasks from your GitHub repository.
</Note>
## Installation
@@ -22,7 +23,8 @@ You can connect Vercel from two entry points:
<Steps>
<Step title="Connect Vercel">
Go to your project's **Settings** page and click **Connect Vercel**. This will redirect you to Vercel to authorize the Trigger.dev app.
Go to your project's **Settings** page and click **Connect Vercel**. This will redirect you to
Vercel to authorize the Trigger.dev app.
</Step>
<Step title="Select a Vercel project">
@@ -30,15 +32,18 @@ You can connect Vercel from two entry points:
</Step>
<Step title="Map environments">
If your Vercel project has custom environments, choose which one maps to your Trigger.dev staging environment.
If your Vercel project has custom environments, choose which one maps to your Trigger.dev staging
environment.
</Step>
<Step title="Sync environment variables">
Review the environment variables that will be pulled from Vercel into Trigger.dev. You can deselect any variables you don't want to sync.
Review the environment variables that will be pulled from Vercel into Trigger.dev. You can
deselect any variables you don't want to sync.
</Step>
<Step title="Configure build settings">
Optionally adjust [build settings](#build-settings) for atomic deployments, env var pulling, and new env var discovery.
<Step title="Configure build options">
Optionally adjust [build options](#build-options) for atomic deployments, env var pulling, and new
env var discovery.
</Step>
<Step title="Connect GitHub">
@@ -52,11 +57,14 @@ You can connect Vercel from two entry points:
<Steps>
<Step title="Install the integration">
Find Trigger.dev on the Vercel Marketplace and install it. This will redirect you to Trigger.dev to complete setup.
Install the [Trigger.dev integration from the Vercel
Marketplace](https://vercel.com/marketplace/trigger). This will redirect you to Trigger.dev to
complete setup.
</Step>
<Step title="Select your Trigger.dev organization and project">
Choose which Trigger.dev organization and project to connect. If you're new to Trigger.dev, you'll be guided through creating an organization and project.
Choose which Trigger.dev organization and project to connect. If you're new to Trigger.dev, you'll
be guided through creating an organization and project.
</Step>
<Step title="Connect GitHub">
@@ -66,9 +74,21 @@ You can connect Vercel from two entry points:
</Steps>
<Note>
When installing from the Vercel Marketplace, default build settings are applied automatically. You can adjust them later in your project settings.
When installing from the Vercel Marketplace, default Build options are applied automatically. You
can adjust them later in your project settings.
</Note>
<Warning>
**Vercel Root Directory:** If your Vercel project uses a **Root Directory** (e.g. you deploy a
single subfolder such as `app` or `web`), you may see "The specified Root Directory does not
exist" after connecting the integration. If you see this error, try using the **repository root**
(leave Root Directory empty) in your Vercel project settings. If your Vercel frontend build
requires a Root Directory (e.g. in a monorepo), keep that setting in Vercel and instead point
Trigger.dev to the subfolder by setting the **Trigger config file** path (and other [Build
options](#build-options)) in your Trigger.dev project configuration. Trigger.dev always builds
from the repo root.
</Warning>
## Environment variable sync
The integration syncs environment variables in both directions:
@@ -78,13 +98,15 @@ The integration syncs environment variables in both directions:
**Trigger.dev → Vercel**: Trigger.dev syncs API keys (like `TRIGGER_SECRET_KEY`) to your Vercel project so your app can communicate with Trigger.dev.
The following variables are excluded from the Vercel → Trigger.dev sync:
- `TRIGGER_SECRET_KEY`, `TRIGGER_VERSION`, `TRIGGER_PREVIEW_BRANCH` (managed by Trigger.dev)
- Sensitive/secret-type variables (Vercel API limitation)
You can control sync behavior per-variable from your project's Vercel settings. Deselecting a variable prevents its value from being updated during future syncs.
<Tip>
For dynamic environment variables (e.g., from NeonDB branching), use the `syncEnvVars` build extension instead. Learn more about [environment variables](/deploy-environment-variables).
For dynamic environment variables (e.g., from NeonDB branching), use the `syncEnvVars` build
extension instead. Learn more about [environment variables](/deploy-environment-variables).
</Tip>
## Atomic deployments
@@ -94,7 +116,9 @@ Atomic deployments ensure your Vercel app and Trigger.dev tasks are deployed in
Atomic deployments are enabled for the production environment by default.
<Note>
When atomic deployments are enabled, the integration automatically disables `Auto-assign Custom Production Domains` on your Vercel project. This is required so that Vercel doesn't promote a deployment before the Trigger.dev build is ready.
When atomic deployments are enabled, the integration automatically disables `Auto-assign Custom
Production Domains` on your Vercel project. This is required so that Vercel doesn't promote a
deployment before the Trigger.dev build is ready.
</Note>
Previously, setting up atomic deployments with Vercel required custom GitHub Actions workflows. The Vercel integration automates this entirely. For more details on how atomic deployments work, see [Atomic deploys](/deployment/atomic-deployment).
@@ -103,20 +127,21 @@ Previously, setting up atomic deployments with Vercel required custom GitHub Act
The integration maps Vercel environments to Trigger.dev environments:
| Vercel environment | Trigger.dev environment |
| --- | --- |
| Production | Production |
| Vercel environment | Trigger.dev environment |
| ------------------ | ------------------------------ |
| Production | Production |
| Custom environment | Staging (you choose which one) |
| Preview | Preview |
| Development | Development |
| Preview | Preview |
| Development | Development |
If your Vercel project has a custom environment, you can select which one maps to your Trigger.dev staging environment during setup or in your project settings.
<Note>
Preview deployments require the preview environment to be enabled on your project. Learn more about [preview branches](/deployment/preview-branches).
Preview deployments require the preview environment to be enabled on your project. Learn more
about [preview branches](/deployment/preview-branches).
</Note>
## Build settings
## Build options
You can configure the following settings per-environment from your project's Vercel settings:
@@ -124,6 +149,8 @@ You can configure the following settings per-environment from your project's Ver
- **Pull env vars before build**: When enabled, Trigger.dev pulls the latest environment variables from Vercel before each build. Enabled for production, staging, and preview by default.
- **Discover new env vars**: When enabled, new environment variables found in Vercel that don't yet exist in Trigger.dev are created automatically during builds. Only available for environments that also have env var pulling enabled. Enabled for production, staging, and preview by default.
To change build options that would normally go in `trigger.config.ts` (such as [extensions](/config/config-file#extensions) or other build configuration), use **Build options** on your project's configuration page in the Trigger.dev dashboard.
## Disconnecting
You can disconnect the Vercel integration from either side:
internal-packages/clickhouse/schema/019_create_metrics_v1.sql
+44
View File
@@ -0,0 +1,44 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS trigger_dev.metrics_v1
(
organization_id LowCardinality(String),
project_id LowCardinality(String),
environment_id String CODEC(ZSTD(1)),
metric_name LowCardinality(String),
metric_type LowCardinality(String),
metric_subject String CODEC(ZSTD(1)),
bucket_start DateTime CODEC(Delta(4), ZSTD(1)),
value Float64 DEFAULT 0 CODEC(ZSTD(1)),
attributes JSON(
`trigger.run_id` String,
`trigger.task_slug` String,
`trigger.attempt_number` Int64,
`trigger.environment_type` LowCardinality(String),
`trigger.machine_id` String,
`trigger.machine_name` LowCardinality(String),
`trigger.worker_id` String,
`trigger.worker_version` String,
`system.cpu.logical_number` String,
`system.cpu.state` LowCardinality(String),
`system.memory.state` LowCardinality(String),
`system.device` String,
`system.filesystem.type` LowCardinality(String),
`system.filesystem.mountpoint` String,
`system.filesystem.mode` LowCardinality(String),
`system.filesystem.state` LowCardinality(String),
`disk.io.direction` LowCardinality(String),
`process.cpu.state` LowCardinality(String),
`network.io.direction` LowCardinality(String),
max_dynamic_paths=8
),
INDEX idx_run_id attributes.trigger.run_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_task_slug attributes.trigger.task_slug TYPE bloom_filter(0.001) GRANULARITY 1
)
ENGINE = MergeTree()
PARTITION BY toDate(bucket_start)
ORDER BY (organization_id, project_id, environment_id, metric_name, metric_subject, bucket_start)
TTL bucket_start + INTERVAL 60 DAY
SETTINGS ttl_only_drop_parts = 1;
-- +goose Down
DROP TABLE IF EXISTS trigger_dev.metrics_v1;
internal-packages/clickhouse/schema/020_add_machine_id_to_task_events_v2.sql
+7
View File
@@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE trigger_dev.task_events_v2
ADD COLUMN machine_id String DEFAULT '' CODEC(ZSTD(1));
-- +goose Down
ALTER TABLE trigger_dev.task_events_v2
DROP COLUMN machine_id;
internal-packages/clickhouse/src/index.ts
+9 -1
View File
@@ -26,12 +26,14 @@ import {
getLogDetailQueryBuilderV2,
getLogsSearchListQueryBuilder,
} from "./taskEvents.js";
import { insertMetrics } from "./metrics.js";
import { Logger, type LogLevel } from "@trigger.dev/core/logger";
import type { Agent as HttpAgent } from "http";
import type { Agent as HttpsAgent } from "https";
export type * from "./taskRuns.js";
export type * from "./taskEvents.js";
export type * from "./metrics.js";
export type * from "./client/queryBuilder.js";
// Re-export column constants, indices, and type-safe accessors
@@ -56,7 +58,7 @@ export {
type FieldMappings,
type WhereClauseCondition,
} from "./client/tsql.js";
export type { OutputColumnMetadata } from "@internal/tsql";
export type { ColumnFormatType, OutputColumnMetadata } from "@internal/tsql";
// Errors
export { QueryError } from "./client/errors.js";
@@ -206,6 +208,12 @@ export class ClickHouse {
};
}
get metrics() {
return {
insert: insertMetrics(this.writer),
};
}
get taskEventsV2() {
return {
insert: insertTaskEventsV2(this.writer),
internal-packages/clickhouse/src/metrics.ts
+29
View File
@@ -0,0 +1,29 @@
import { z } from "zod";
import { ClickhouseWriter } from "./client/types.js";
export const MetricsV1Input = z.object({
organization_id: z.string(),
project_id: z.string(),
environment_id: z.string(),
metric_name: z.string(),
metric_type: z.string(),
metric_subject: z.string(),
bucket_start: z.string(),
value: z.number(),
attributes: z.unknown(),
});
export type MetricsV1Input = z.input<typeof MetricsV1Input>;
export function insertMetrics(ch: ClickhouseWriter) {
return ch.insertUnsafe<MetricsV1Input>({
name: "insertMetrics",
table: "trigger_dev.metrics_v1",
settings: {
enable_json_type: 1,
type_json_skip_duplicated_paths: 1,
input_format_json_throw_on_bad_escape_sequence: 0,
input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
},
});
}
internal-packages/clickhouse/src/taskEvents.ts
+2
View File
@@ -19,6 +19,7 @@ export const TaskEventV1Input = z.object({
attributes: z.unknown(),
metadata: z.string(),
expires_at: z.string(),
machine_id: z.string().optional(),
});
export type TaskEventV1Input = z.input<typeof TaskEventV1Input>;
@@ -153,6 +154,7 @@ export const TaskEventV2Input = z.object({
attributes: z.unknown(),
metadata: z.string(),
expires_at: z.string(),
machine_id: z.string().optional(),
// inserted_at has a default value in the table, so it's optional for inserts
inserted_at: z.string().optional(),
});
internal-packages/otlp-importer/src/index.ts
+39
View File
@@ -10,6 +10,12 @@ import {
ExportLogsServiceResponse,
} from "./generated/opentelemetry/proto/collector/logs/v1/logs_service";
import {
ExportMetricsPartialSuccess,
ExportMetricsServiceRequest,
ExportMetricsServiceResponse,
} from "./generated/opentelemetry/proto/collector/metrics/v1/metrics_service";
import type {
AnyValue,
KeyValue,
@@ -33,6 +39,21 @@ import {
Status,
Status_StatusCode,
} from "./generated/opentelemetry/proto/trace/v1/trace";
import {
ResourceMetrics,
ScopeMetrics,
Metric,
Gauge,
Sum,
Histogram,
ExponentialHistogram,
Summary,
NumberDataPoint,
HistogramDataPoint,
ExponentialHistogramDataPoint,
SummaryDataPoint,
AggregationTemporality,
} from "./generated/opentelemetry/proto/metrics/v1/metrics";
export {
LogRecord,
@@ -57,3 +78,21 @@ export {
export { ExportTracePartialSuccess, ExportTraceServiceRequest, ExportTraceServiceResponse };
export { ExportLogsPartialSuccess, ExportLogsServiceRequest, ExportLogsServiceResponse };
export { ExportMetricsPartialSuccess, ExportMetricsServiceRequest, ExportMetricsServiceResponse };
export {
ResourceMetrics,
ScopeMetrics,
Metric,
Gauge,
Sum,
Histogram,
ExponentialHistogram,
Summary,
NumberDataPoint,
HistogramDataPoint,
ExponentialHistogramDataPoint,
SummaryDataPoint,
AggregationTemporality,
};
internal-packages/run-engine/src/engine/tests/getSnapshotsSince.test.ts
+11 -1
View File
@@ -191,7 +191,17 @@ describe("RunEngine getSnapshotsSince", () => {
organizationId: authenticatedEnvironment.organization.id,
});
// Wait for waitpoint completion
// Poll until the waitpoint is completed by the background worker
for (let i = 0; i < 50; i++) {
await setTimeout(100);
const wp = await prisma.waitpoint.findFirst({
where: { id: waitpoint.id },
select: { status: true },
});
if (wp?.status === "COMPLETED") break;
}
// Allow time for the snapshot to be created after waitpoint completion
await setTimeout(200);
// Get all snapshots
internal-packages/tsql/src/index.ts
+3
View File
@@ -109,6 +109,7 @@ export {
type ClickHouseType,
type ColumnSchema,
type FieldMappings,
type ColumnFormatType,
type OutputColumnMetadata,
type RequiredFilter,
type SchemaRegistry,
@@ -133,7 +134,9 @@ export {
// Re-export time bucket utilities
export {
BUCKET_THRESHOLDS,
calculateTimeBucketInterval,
type BucketThreshold,
type TimeBucketInterval,
} from "./query/time_buckets.js";
internal-packages/tsql/src/query/printer.test.ts
+206 -1
View File
@@ -2,7 +2,13 @@ import { describe, it, expect, beforeEach } from "vitest";
import { parseTSQLSelect, parseTSQLExpr, compileTSQL } from "../index.js";
import { ClickHousePrinter, printToClickHouse, type PrintResult } from "./printer.js";
import { createPrinterContext, PrinterContext } from "./printer_context.js";
import { createSchemaRegistry, column, type TableSchema, type SchemaRegistry } from "./schema.js";
import {
createSchemaRegistry,
column,
type TableSchema,
type SchemaRegistry,
} from "./schema.js";
import type { BucketThreshold } from "./time_buckets.js";
import { QueryError, SyntaxError } from "./errors.js";
/**
@@ -2335,16 +2341,19 @@ describe("Basic column metadata", () => {
name: "status",
type: "LowCardinality(String)",
customRenderType: "runStatus",
format: "runStatus",
});
expect(columns[1]).toEqual({
name: "usage_duration_ms",
type: "UInt32",
customRenderType: "duration",
format: "duration",
});
expect(columns[2]).toEqual({
name: "cost_in_cents",
type: "Float64",
customRenderType: "cost",
format: "cost",
});
});
@@ -2687,6 +2696,133 @@ describe("Basic column metadata", () => {
expect(columns[2].name).toBe("avg");
});
});
describe("prettyFormat()", () => {
it("should strip prettyFormat from SQL and attach format to column metadata", () => {
const ctx = createMetadataTestContext();
const { sql, columns } = printQuery(
"SELECT prettyFormat(usage_duration_ms, 'bytes') AS memory FROM runs",
ctx
);
// SQL should not contain prettyFormat
expect(sql).not.toContain("prettyFormat");
expect(sql).toContain("usage_duration_ms");
expect(columns).toHaveLength(1);
expect(columns[0].name).toBe("memory");
expect(columns[0].format).toBe("bytes");
});
it("should work with aggregation wrapping", () => {
const ctx = createMetadataTestContext();
const { sql, columns } = printQuery(
"SELECT prettyFormat(avg(usage_duration_ms), 'bytes') AS avg_memory FROM runs",
ctx
);
expect(sql).not.toContain("prettyFormat");
expect(sql).toContain("avg(usage_duration_ms)");
expect(columns).toHaveLength(1);
expect(columns[0].name).toBe("avg_memory");
expect(columns[0].format).toBe("bytes");
expect(columns[0].type).toBe("Float64");
});
it("should work without explicit alias", () => {
const ctx = createMetadataTestContext();
const { sql, columns } = printQuery(
"SELECT prettyFormat(usage_duration_ms, 'percent') FROM runs",
ctx
);
expect(sql).not.toContain("prettyFormat");
expect(columns).toHaveLength(1);
expect(columns[0].name).toBe("usage_duration_ms");
expect(columns[0].format).toBe("percent");
});
it("should throw for invalid format type", () => {
const ctx = createMetadataTestContext();
expect(() => {
printQuery(
"SELECT prettyFormat(usage_duration_ms, 'invalid') FROM runs",
ctx
);
}).toThrow(QueryError);
expect(() => {
printQuery(
"SELECT prettyFormat(usage_duration_ms, 'invalid') FROM runs",
ctx
);
}).toThrow(/Unknown format type/);
});
it("should throw for wrong argument count", () => {
const ctx = createMetadataTestContext();
expect(() => {
printQuery("SELECT prettyFormat(usage_duration_ms) FROM runs", ctx);
}).toThrow(QueryError);
expect(() => {
printQuery("SELECT prettyFormat(usage_duration_ms) FROM runs", ctx);
}).toThrow(/requires exactly 2 arguments/);
});
it("should throw when second argument is not a string literal", () => {
const ctx = createMetadataTestContext();
expect(() => {
printQuery(
"SELECT prettyFormat(usage_duration_ms, 123) FROM runs",
ctx
);
}).toThrow(QueryError);
expect(() => {
printQuery(
"SELECT prettyFormat(usage_duration_ms, 123) FROM runs",
ctx
);
}).toThrow(/must be a string literal/);
});
it("should override schema-level customRenderType", () => {
const ctx = createMetadataTestContext();
const { columns } = printQuery(
"SELECT prettyFormat(usage_duration_ms, 'bytes') AS mem FROM runs",
ctx
);
expect(columns).toHaveLength(1);
// prettyFormat's format should take precedence
expect(columns[0].format).toBe("bytes");
// customRenderType from schema should NOT be set since prettyFormat overrides
// The source column had customRenderType: "duration" but prettyFormat replaces it
});
it("should auto-populate format from customRenderType when not explicitly set", () => {
const ctx = createMetadataTestContext();
const { columns } = printQuery(
"SELECT usage_duration_ms, cost_in_cents FROM runs",
ctx
);
expect(columns).toHaveLength(2);
// customRenderType should auto-populate format
expect(columns[0].customRenderType).toBe("duration");
expect(columns[0].format).toBe("duration");
expect(columns[1].customRenderType).toBe("cost");
expect(columns[1].format).toBe("cost");
});
it("should not set format when column has no customRenderType", () => {
const ctx = createMetadataTestContext();
const { columns } = printQuery("SELECT run_id FROM runs", ctx);
expect(columns).toHaveLength(1);
expect(columns[0].format).toBeUndefined();
expect(columns[0].customRenderType).toBeUndefined();
});
});
});
describe("Unknown column blocking", () => {
@@ -3570,4 +3706,73 @@ describe("timeBucket()", () => {
expect(Object.values(params)).toContain("org_test123");
});
});
describe("per-table timeBucketThresholds", () => {
const customThresholds: BucketThreshold[] = [
// 10-second minimum granularity (e.g., for pre-aggregated metrics)
{ maxRangeSeconds: 10 * 60, interval: { value: 10, unit: "SECOND" } },
{ maxRangeSeconds: 30 * 60, interval: { value: 30, unit: "SECOND" } },
{ maxRangeSeconds: 2 * 60 * 60, interval: { value: 1, unit: "MINUTE" } },
];
const schemaWithCustomThresholds: TableSchema = {
...timeBucketSchema,
name: "metrics",
timeBucketThresholds: customThresholds,
};
it("should use custom thresholds when defined on the table schema", () => {
// 3-minute range: global default would give 5 SECOND, custom gives 10 SECOND
const threeMinuteRange = {
from: new Date("2024-01-01T00:00:00Z"),
to: new Date("2024-01-01T00:03:00Z"),
};
const schema = createSchemaRegistry([schemaWithCustomThresholds]);
const ctx = createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test123" },
project_id: { op: "eq", value: "proj_test456" },
environment_id: { op: "eq", value: "env_test789" },
},
timeRange: threeMinuteRange,
});
const ast = parseTSQLSelect(
"SELECT timeBucket(), count() FROM metrics GROUP BY timeBucket"
);
const { sql } = printToClickHouse(ast, ctx);
// Custom thresholds: under 10 min → 10 SECOND (not the global 5 SECOND)
expect(sql).toContain("toStartOfInterval(created_at, INTERVAL 10 SECOND)");
});
it("should fall back to global defaults when no custom thresholds are defined", () => {
// 3-minute range with standard schema (no custom thresholds)
const threeMinuteRange = {
from: new Date("2024-01-01T00:00:00Z"),
to: new Date("2024-01-01T00:03:00Z"),
};
const schema = createSchemaRegistry([timeBucketSchema]);
const ctx = createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test123" },
project_id: { op: "eq", value: "proj_test456" },
environment_id: { op: "eq", value: "env_test789" },
},
timeRange: threeMinuteRange,
});
const ast = parseTSQLSelect(
"SELECT timeBucket(), count() FROM runs GROUP BY timeBucket"
);
const { sql } = printToClickHouse(ast, ctx);
// Global default: under 5 min → 5 SECOND
expect(sql).toContain("toStartOfInterval(created_at, INTERVAL 5 SECOND)");
});
});
});
internal-packages/tsql/src/query/printer.ts
+82 -7
View File
@@ -59,6 +59,7 @@ import {
ClickHouseType,
hasFieldMapping,
getInternalValueFromMappingCaseInsensitive,
type ColumnFormatType,
} from "./schema";
/**
@@ -739,6 +740,14 @@ export class ClickHousePrinter {
metadata.description = sourceColumn.description;
}
// Set format hint from prettyFormat() or auto-populate from customRenderType
const sourceWithFormat = sourceColumn as (Partial<ColumnSchema> & { format?: ColumnFormatType }) | null;
if (sourceWithFormat?.format) {
metadata.format = sourceWithFormat.format;
} else if (sourceColumn?.customRenderType) {
metadata.format = sourceColumn.customRenderType as ColumnFormatType;
}
this.outputColumns.push(metadata);
}
@@ -932,6 +941,53 @@ export class ClickHousePrinter {
};
}
// Handle prettyFormat(expr, 'formatType') — metadata-only wrapper
if ((col as Call).expression_type === "call") {
const call = col as Call;
if (call.name.toLowerCase() === "prettyformat") {
if (call.args.length !== 2) {
throw new QueryError(
"prettyFormat() requires exactly 2 arguments: prettyFormat(expression, 'formatType')"
);
}
const formatArg = call.args[1];
if (
(formatArg as Constant).expression_type !== "constant" ||
typeof (formatArg as Constant).value !== "string"
) {
throw new QueryError(
"prettyFormat() second argument must be a string literal format type"
);
}
const formatType = (formatArg as Constant).value as string;
const validFormats = [
"bytes",
"decimalBytes",
"quantity",
"percent",
"duration",
"durationSeconds",
"costInDollars",
"cost",
];
if (!validFormats.includes(formatType)) {
throw new QueryError(
`Unknown format type '${formatType}'. Valid types: ${validFormats.join(", ")}`
);
}
const innerAnalysis = this.analyzeSelectColumn(call.args[0]);
return {
outputName: innerAnalysis.outputName,
sourceColumn: {
...(innerAnalysis.sourceColumn ?? {}),
type: innerAnalysis.sourceColumn?.type ?? innerAnalysis.inferredType ?? undefined,
format: formatType as ColumnFormatType,
} as Partial<ColumnSchema> & { format?: ColumnFormatType },
inferredType: innerAnalysis.inferredType,
};
}
}
// Handle Call (function/aggregation) - infer type from function
if ((col as Call).expression_type === "call") {
const call = col as Call;
@@ -1559,13 +1615,20 @@ export class ClickHousePrinter {
joinStrings.push(`AS ${this.printIdentifier(node.alias)}`);
}
// Always add FINAL for direct table references to ensure deduplicated results
// from ReplacingMergeTree tables in ClickHouse
// Add FINAL for direct table references to ReplacingMergeTree tables
// to ensure deduplicated results. Only applied when the table schema
// opts in via `useFinal: true` (not needed for plain MergeTree tables).
if (node.table) {
const tableExpr = node.table;
const isDirectTable = (tableExpr as Field).expression_type === "field";
if (isDirectTable) {
joinStrings.push("FINAL");
if ((tableExpr as Field).expression_type === "field") {
const field = tableExpr as Field;
const tableName = field.chain[0];
if (typeof tableName === "string") {
const tableSchema = this.lookupTable(tableName);
if (tableSchema.useFinal) {
joinStrings.push("FINAL");
}
}
}
}
@@ -2802,6 +2865,14 @@ export class ClickHousePrinter {
private visitCall(node: Call): string {
const name = node.name;
// Handle prettyFormat() — strip wrapper, only emit the inner expression
if (name.toLowerCase() === "prettyformat") {
if (node.args.length !== 2) {
throw new QueryError("prettyFormat() requires exactly 2 arguments");
}
return this.visit(node.args[0]);
}
// Handle timeBucket() - special TSQL function for automatic time bucketing
if (name.toLowerCase() === "timebucket") {
return this.visitTimeBucket(node);
@@ -2978,8 +3049,12 @@ export class ClickHousePrinter {
);
}
// Calculate the appropriate interval
const interval = calculateTimeBucketInterval(timeRange.from, timeRange.to);
// Calculate the appropriate interval (use table-specific thresholds if defined)
const interval = calculateTimeBucketInterval(
timeRange.from,
timeRange.to,
tableSchema.timeBucketThresholds
);
// Emit toStartOfInterval(column, INTERVAL N UNIT)
return `toStartOfInterval(${escapeClickHouseIdentifier(clickhouseColumnName)}, INTERVAL ${interval.value} ${interval.unit})`;
internal-packages/tsql/src/query/schema.ts
+51
View File
@@ -2,6 +2,7 @@
// Defines allowed tables, columns, and tenant isolation configuration
import { QueryError } from "./errors";
import type { BucketThreshold } from "./time_buckets";
/**
* ClickHouse data types supported by TSQL
@@ -269,6 +270,33 @@ export interface ColumnSchema {
*/
export type FieldMappings = Record<string, Record<string, string>>;
/**
* Display format types for column values.
*
* These tell the UI how to render values without changing the underlying data type.
* Includes both existing custom render types and new format hint types.
*/
export type ColumnFormatType =
// Existing custom render types
| "runId"
| "runStatus"
| "duration"
| "durationSeconds"
| "costInDollars"
| "cost"
| "machine"
| "environment"
| "environmentType"
| "project"
| "queue"
| "tags"
| "number"
// Format hint types (used by prettyFormat())
| "bytes"
| "decimalBytes"
| "quantity"
| "percent";
/**
* Metadata for a column in query results.
*
@@ -290,6 +318,16 @@ export interface OutputColumnMetadata {
* Only present for columns or virtual columns defined in the table schema.
*/
description?: string;
/**
* Display format hint — tells the UI how to render numeric values.
*
* Set by `prettyFormat(expr, 'formatType')` in TSQL queries.
* The underlying value remains numeric (for charts), but the UI uses this
* hint for axis labels, table cells, and tooltips.
*
* Also auto-populated from `customRenderType` when not explicitly set.
*/
format?: ColumnFormatType;
}
/**
@@ -354,6 +392,19 @@ export interface TableSchema {
* ```
*/
timeConstraint?: string;
/**
* Custom time bucket thresholds for this table.
* When set, timeBucket() uses these instead of the global defaults.
* Useful when the table's time granularity differs from the standard (e.g., metrics
* pre-aggregated into 10-second buckets shouldn't go below 10-second intervals).
*/
timeBucketThresholds?: BucketThreshold[];
/**
* Whether to add the FINAL keyword when querying this table.
* This should be set to `true` for ReplacingMergeTree tables where deduplication
* is needed to get correct results. Not needed for plain MergeTree tables.
*/
useFinal?: boolean;
}
/**
internal-packages/tsql/src/query/time_buckets.ts
+18 -4
View File
@@ -17,13 +17,23 @@ export interface TimeBucketInterval {
}
/**
* Time bucket thresholds: each entry defines a maximum time range duration (in seconds)
* A threshold mapping a maximum time range duration to a bucket interval.
*/
export interface BucketThreshold {
/** Maximum range duration in seconds for this threshold to apply */
maxRangeSeconds: number;
/** The bucket interval to use when the range is under maxRangeSeconds */
interval: TimeBucketInterval;
}
/**
* Default time bucket thresholds: each entry defines a maximum time range duration (in seconds)
* and the corresponding bucket interval to use.
*
* The intervals are chosen to produce roughly 50-100 data points for the given range.
* Entries are ordered from smallest to largest range.
*/
const BUCKET_THRESHOLDS: Array<{ maxRangeSeconds: number; interval: TimeBucketInterval }> = [
export const BUCKET_THRESHOLDS: BucketThreshold[] = [
// Under 5 minutes → 5 second buckets (max 60 buckets)
{ maxRangeSeconds: 5 * 60, interval: { value: 5, unit: "SECOND" } },
// Under 30 minutes → 30 second buckets (max 60 buckets)
@@ -73,10 +83,14 @@ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" };
* ); // { value: 6, unit: "HOUR" }
* ```
*/
export function calculateTimeBucketInterval(from: Date, to: Date): TimeBucketInterval {
export function calculateTimeBucketInterval(
from: Date,
to: Date,
thresholds?: BucketThreshold[]
): TimeBucketInterval {
const rangeSeconds = Math.abs(to.getTime() - from.getTime()) / 1000;
for (const threshold of BUCKET_THRESHOLDS) {
for (const threshold of thresholds ?? BUCKET_THRESHOLDS) {
if (rangeSeconds < threshold.maxRangeSeconds) {
return threshold.interval;
}
packages/build/CHANGELOG.md
+14
View File
@@ -1,5 +1,19 @@
# @trigger.dev/build
## 4.4.1
### Patch Changes
- Updated dependencies:
- `@trigger.dev/core@4.4.1`
## 4.4.0
### Patch Changes
- Updated dependencies:
- `@trigger.dev/core@4.4.0`
## 4.3.3
### Patch Changes
packages/build/package.json
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/build",
"version": "4.3.3",
"version": "4.4.1",
"description": "trigger.dev build extensions",
"license": "MIT",
"publishConfig": {
@@ -78,7 +78,7 @@
},
"dependencies": {
"@prisma/config": "^6.10.0",
"@trigger.dev/core": "workspace:4.3.3",
"@trigger.dev/core": "workspace:4.4.1",
"mlly": "^1.7.1",
"pkg-types": "^1.1.3",
"resolve": "^1.22.8",
packages/cli-v3/CHANGELOG.md
+22
View File
@@ -1,5 +1,27 @@
# trigger.dev
## 4.4.1
### Patch Changes
- Add OTEL metrics pipeline for task workers. Workers collect process CPU/memory, Node.js runtime metrics (event loop utilization, event loop delay, heap usage), and user-defined custom metrics via `otel.metrics.getMeter()`. Metrics are exported to ClickHouse with 10-second aggregation buckets and 1m/5m rollups, and are queryable through the dashboard query engine with typed attribute columns, `prettyFormat()` for human-readable values, and AI query support. ([#3061](https://github.com/triggerdotdev/trigger.dev/pull/3061))
- Updated dependencies:
- `@trigger.dev/build@4.4.1`
- `@trigger.dev/core@4.4.1`
- `@trigger.dev/schema-to-json@4.4.1`
## 4.4.0
### Patch Changes
- Fix runner getting stuck indefinitely when `execute()` is called on a dead child process. ([#2978](https://github.com/triggerdotdev/trigger.dev/pull/2978))
- Add optional `timeoutInSeconds` parameter to the `wait_for_run_to_complete` MCP tool. Defaults to 60 seconds. If the run doesn't complete within the timeout, the current state of the run is returned instead of waiting indefinitely. ([#3035](https://github.com/triggerdotdev/trigger.dev/pull/3035))
- Fixed a minor issue in the deployment command on distinguishing between local builds for the cloud vs local builds for self-hosting setups. ([#3070](https://github.com/triggerdotdev/trigger.dev/pull/3070))
- Updated dependencies:
- `@trigger.dev/core@4.4.0`
- `@trigger.dev/build@4.4.0`
- `@trigger.dev/schema-to-json@4.4.0`
## 4.3.3
### Patch Changes
packages/cli-v3/package.json
+4 -4
View File
@@ -1,6 +1,6 @@
{
"name": "trigger.dev",
"version": "4.3.3",
"version": "4.4.1",
"description": "A Command-Line Interface for Trigger.dev projects",
"type": "module",
"license": "MIT",
@@ -93,9 +93,9 @@
"@opentelemetry/sdk-trace-node": "2.0.1",
"@opentelemetry/semantic-conventions": "1.36.0",
"@s2-dev/streamstore": "^0.17.6",
"@trigger.dev/build": "workspace:4.3.3",
"@trigger.dev/core": "workspace:4.3.3",
"@trigger.dev/schema-to-json": "workspace:4.3.3",
"@trigger.dev/build": "workspace:4.4.1",
"@trigger.dev/core": "workspace:4.4.1",
"@trigger.dev/schema-to-json": "workspace:4.4.1",
"ansi-escapes": "^7.0.0",
"braces": "^3.0.3",
"c12": "^1.11.1",
packages/cli-v3/src/dev/taskRunProcessPool.ts
+48
View File
@@ -2,6 +2,7 @@ import {
MachinePresetResources,
ServerBackgroundWorker,
WorkerManifest,
generateFriendlyId,
} from "@trigger.dev/core/v3";
import { TaskRunProcess } from "../executions/taskRunProcess.js";
import { logger } from "../utilities/logger.js";
@@ -23,6 +24,8 @@ export class TaskRunProcessPool {
private readonly maxExecutionsPerProcess: number;
private readonly executionCountsPerProcess: Map<number, number> = new Map();
private readonly deprecatedVersions: Set<string> = new Set();
private readonly idleTimers: Map<TaskRunProcess, NodeJS.Timeout> = new Map();
private static readonly IDLE_TIMEOUT_MS = 30_000;
constructor(options: TaskRunProcessPoolOptions) {
this.options = options;
@@ -38,6 +41,7 @@ export class TaskRunProcessPool {
const versionProcesses = this.availableProcessesByVersion.get(version) || [];
const processesToKill = versionProcesses.filter((process) => !process.isExecuting());
processesToKill.forEach((process) => this.clearIdleTimer(process));
Promise.all(processesToKill.map((process) => this.killProcess(process))).then(() => {
this.availableProcessesByVersion.delete(version);
});
@@ -71,6 +75,7 @@ export class TaskRunProcessPool {
version,
availableProcesses.filter((p) => p !== reusableProcess)
);
this.clearIdleTimer(reusableProcess);
if (!this.busyProcessesByVersion.has(version)) {
this.busyProcessesByVersion.set(version, new Set());
@@ -106,6 +111,7 @@ export class TaskRunProcessPool {
env: {
...this.options.env,
...env,
TRIGGER_MACHINE_ID: generateFriendlyId("machine"),
},
serverWorker,
machineResources,
@@ -154,6 +160,7 @@ export class TaskRunProcessPool {
this.availableProcessesByVersion.set(version, []);
}
this.availableProcessesByVersion.get(version)!.push(process);
this.startIdleTimer(process, version);
} catch (error) {
logger.debug("[TaskRunProcessPool] Failed to cleanup process for reuse, killing it", {
error,
@@ -213,7 +220,42 @@ export class TaskRunProcessPool {
return process.isHealthy;
}
private startIdleTimer(process: TaskRunProcess, version: string): void {
this.clearIdleTimer(process);
const timer = setTimeout(() => {
// Synchronously remove from available pool before async kill to prevent race with getProcess()
const available = this.availableProcessesByVersion.get(version);
if (available) {
const index = available.indexOf(process);
if (index !== -1) {
available.splice(index, 1);
}
}
this.idleTimers.delete(process);
logger.debug("[TaskRunProcessPool] Idle timeout reached, killing process", {
pid: process.pid,
version,
});
this.killProcess(process);
}, TaskRunProcessPool.IDLE_TIMEOUT_MS);
this.idleTimers.set(process, timer);
}
private clearIdleTimer(process: TaskRunProcess): void {
const timer = this.idleTimers.get(process);
if (timer) {
clearTimeout(timer);
this.idleTimers.delete(process);
}
}
private async killProcess(process: TaskRunProcess): Promise<void> {
this.clearIdleTimer(process);
if (!process.isHealthy) {
logger.debug("[TaskRunProcessPool] Process is not healthy, skipping cleanup", {
processId: process.pid,
@@ -245,6 +287,12 @@ export class TaskRunProcessPool {
versions: Array.from(this.availableProcessesByVersion.keys()),
});
// Clear all idle timers
for (const timer of this.idleTimers.values()) {
clearTimeout(timer);
}
this.idleTimers.clear();
// Kill all available processes across all versions
const allAvailableProcesses = Array.from(this.availableProcessesByVersion.values()).flat();
await Promise.all(allAvailableProcesses.map((process) => this.killProcess(process)));
packages/cli-v3/src/entryPoints/dev-run-controller.ts
+1 -1
View File
@@ -596,7 +596,7 @@ export class DevRunController {
const { taskRunProcess, isReused } = await this.opts.taskRunProcessPool.getProcess(
this.opts.worker.manifest,
{
id: "unmanaged",
id: this.opts.worker.serverWorker.id,
contentHash: this.opts.worker.build.contentHash,
version: this.opts.worker.serverWorker?.version,
engine: "V2",
packages/cli-v3/src/entryPoints/dev-run-worker.ts
+10 -1
View File
@@ -204,12 +204,18 @@ async function doBootstrap() {
const tracingSDK = new TracingSDK({
url: env.TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
metricsUrl: env.TRIGGER_OTEL_METRICS_ENDPOINT,
instrumentations: config.telemetry?.instrumentations ?? config.instrumentations ?? [],
exporters: config.telemetry?.exporters ?? [],
logExporters: config.telemetry?.logExporters ?? [],
metricExporters: config.telemetry?.metricExporters ?? [],
metricReaders: config.telemetry?.metricReaders ?? [],
diagLogLevel: (env.TRIGGER_OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
resource: config.telemetry?.resource,
hostMetrics: true,
hostMetricGroups: ["process.cpu", "process.memory"],
nodejsRuntimeMetrics: true,
});
const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);
@@ -619,8 +625,11 @@ const zodIpc = new ZodIpcConnection({
}
await flushAll(timeoutInMs);
},
FLUSH: async ({ timeoutInMs }) => {
FLUSH: async ({ timeoutInMs, disableContext }) => {
await flushAll(timeoutInMs);
if (disableContext) {
taskContext.disable();
}
},
RESOLVE_WAITPOINT: async ({ waitpoint }) => {
_sharedWorkerRuntime?.resolveWaitpoints([waitpoint]);
packages/cli-v3/src/entryPoints/managed-run-worker.ts
+16 -2
View File
@@ -183,12 +183,23 @@ async function doBootstrap() {
const tracingSDK = new TracingSDK({
url: env.TRIGGER_OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
instrumentations: config.instrumentations ?? [],
metricsUrl: env.TRIGGER_OTEL_METRICS_ENDPOINT,
instrumentations: config.telemetry?.instrumentations ?? config.instrumentations ?? [],
diagLogLevel: (env.TRIGGER_OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
forceFlushTimeoutMillis: 30_000,
exporters: config.telemetry?.exporters ?? [],
logExporters: config.telemetry?.logExporters ?? [],
metricExporters: config.telemetry?.metricExporters ?? [],
metricReaders: config.telemetry?.metricReaders ?? [],
resource: config.telemetry?.resource,
hostMetrics: true,
hostMetricGroups:
getEnvVar("TRIGGER_SYSTEM_METRICS_ENABLED") === "1"
? undefined
: ["process.cpu", "process.memory"],
nodejsRuntimeMetrics: true,
filesystemMetrics: getEnvVar("TRIGGER_SYSTEM_METRICS_ENABLED") === "1",
diskIoMetrics: getEnvVar("TRIGGER_SYSTEM_METRICS_ENABLED") === "1",
});
const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);
@@ -607,8 +618,11 @@ const zodIpc = new ZodIpcConnection({
}
await flushAll(timeoutInMs);
},
FLUSH: async ({ timeoutInMs }) => {
FLUSH: async ({ timeoutInMs, disableContext }) => {
await flushAll(timeoutInMs);
if (disableContext) {
taskContext.disable();
}
},
RESOLVE_WAITPOINT: async ({ waitpoint }) => {
_sharedWorkerRuntime?.resolveWaitpoints([waitpoint]);
packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts
+4 -2
View File
@@ -1,4 +1,4 @@
import { WorkerManifest } from "@trigger.dev/core/v3";
import { WorkerManifest, generateFriendlyId } from "@trigger.dev/core/v3";
import { TaskRunProcess } from "../../executions/taskRunProcess.js";
import { RunnerEnv } from "./env.js";
import { RunLogger, SendDebugLogOptions } from "./logger.js";
@@ -22,6 +22,7 @@ export class TaskRunProcessProvider {
private readonly logger: RunLogger;
private readonly processKeepAliveEnabled: boolean;
private readonly processKeepAliveMaxExecutionCount: number;
private readonly machineId = generateFriendlyId("machine");
// Process keep-alive state
private persistentProcess: TaskRunProcess | null = null;
@@ -250,7 +251,7 @@ export class TaskRunProcessProvider {
workerManifest: this.workerManifest,
env: processEnv,
serverWorker: {
id: "managed",
id: this.env.TRIGGER_DEPLOYMENT_ID,
contentHash: this.env.TRIGGER_CONTENT_HASH,
version: this.env.TRIGGER_DEPLOYMENT_VERSION,
engine: "V2",
@@ -269,6 +270,7 @@ export class TaskRunProcessProvider {
return {
...taskRunEnv,
...this.env.gatherProcessEnv(),
TRIGGER_MACHINE_ID: this.machineId,
HEARTBEAT_INTERVAL_MS: String(this.env.TRIGGER_HEARTBEAT_INTERVAL_SECONDS * 1000),
};
}
packages/cli-v3/src/executions/taskRunProcess.ts
+3 -3
View File
@@ -126,7 +126,7 @@ export class TaskRunProcess {
return;
}
await tryCatch(this.#flush());
await tryCatch(this.#flush({ disableContext: !kill }));
if (kill) {
await this.#gracefullyTerminate(this.options.gracefulTerminationTimeoutInMs);
@@ -240,10 +240,10 @@ export class TaskRunProcess {
return this;
}
async #flush(timeoutInMs: number = 5_000) {
async #flush({ timeoutInMs = 5_000, disableContext = false } = {}) {
logger.debug("flushing task run process", { pid: this.pid });
await this._ipc?.sendWithAck("FLUSH", { timeoutInMs }, timeoutInMs + 1_000);
await this._ipc?.sendWithAck("FLUSH", { timeoutInMs, disableContext }, timeoutInMs + 1_000);
}
async #cancel(timeoutInMs: number = 30_000) {
packages/core/CHANGELOG.md
+25
View File
@@ -1,5 +1,30 @@
# internal-platform
## 4.4.1
## 4.4.0
### Patch Changes
- Add `maxDelay` option to debounce feature. This allows setting a maximum time limit for how long a debounced run can be delayed, ensuring execution happens within a specified window even with continuous triggers. ([#2984](https://github.com/triggerdotdev/trigger.dev/pull/2984))
```typescript
await myTask.trigger(payload, {
debounce: {
key: "my-key",
delay: "5s",
maxDelay: "30m", // Execute within 30 minutes regardless of continuous triggers
},
});
```
- Fixed a minor issue in the deployment command on distinguishing between local builds for the cloud vs local builds for self-hosting setups. ([#3070](https://github.com/triggerdotdev/trigger.dev/pull/3070))
- fix: vendor superjson to fix ESM/CJS compatibility ([#2949](https://github.com/triggerdotdev/trigger.dev/pull/2949))
Bundle superjson during build to avoid `ERR_REQUIRE_ESM` errors on Node.js versions that don't support `require(ESM)` by default (< 22.12.0) and AWS Lambda which intentionally disables it.
- Add Vercel integration support to API schemas: `commitSHA` and `integrationDeployments` on deployment responses, and `source` field for environment variable imports. ([#2994](https://github.com/triggerdotdev/trigger.dev/pull/2994))
## 4.3.3
### Patch Changes
packages/core/package.json
+4 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/core",
"version": "4.3.3",
"version": "4.4.1",
"description": "Core code used across the Trigger.dev SDK and platform",
"license": "MIT",
"publishConfig": {
@@ -176,10 +176,13 @@
"@opentelemetry/api-logs": "0.203.0",
"@opentelemetry/core": "2.0.1",
"@opentelemetry/exporter-logs-otlp-http": "0.203.0",
"@opentelemetry/exporter-metrics-otlp-http": "0.203.0",
"@opentelemetry/host-metrics": "^0.37.0",
"@opentelemetry/exporter-trace-otlp-http": "0.203.0",
"@opentelemetry/instrumentation": "0.203.0",
"@opentelemetry/resources": "2.0.1",
"@opentelemetry/sdk-logs": "0.203.0",
"@opentelemetry/sdk-metrics": "2.0.1",
"@opentelemetry/sdk-trace-base": "2.0.1",
"@opentelemetry/sdk-trace-node": "2.0.1",
"@opentelemetry/semantic-conventions": "1.36.0",
packages/core/src/v3/config.ts
+15
View File
@@ -1,5 +1,6 @@
import type { Instrumentation } from "@opentelemetry/instrumentation";
import type { SpanExporter } from "@opentelemetry/sdk-trace-base";
import type { MetricReader, PushMetricExporter } from "@opentelemetry/sdk-metrics";
import type { BuildExtension } from "./build/extensions.js";
import type {
AnyOnFailureHookFunction,
@@ -109,6 +110,20 @@ export type TriggerConfig = {
*/
logExporters?: Array<LogRecordExporter>;
/**
* Metric exporters to use for OpenTelemetry. This is useful if you want to export metrics to external services.
* Each exporter is automatically wrapped in a PeriodicExportingMetricReader.
*
* For more control over the reader configuration, use `metricReaders` instead.
*/
metricExporters?: Array<PushMetricExporter>;
/**
* Metric readers for OpenTelemetry. Add custom metric readers to export
* metrics to external services alongside the default Trigger.dev exporter.
*/
metricReaders?: Array<MetricReader>;
/**
* Resource to use for OpenTelemetry. This is useful if you want to add custom resources to your tasks.
*
packages/core/src/v3/index.ts
+1
View File
@@ -49,6 +49,7 @@ export {
NULL_SENTINEL,
} from "./utils/flattenAttributes.js";
export { omit } from "./utils/omit.js";
export { generateFriendlyId, fromFriendlyId } from "./isomorphic/friendlyId.js";
export {
calculateNextRetryDelay,
calculateResetAt,
packages/core/src/v3/otel/diskIoMetrics.ts
+92
View File
@@ -0,0 +1,92 @@
import { type MeterProvider } from "@opentelemetry/sdk-metrics";
import * as fs from "node:fs";
import * as fsPromises from "node:fs/promises";
const SECTOR_SIZE = 512;
const FILTERED_DEVICE_PREFIXES = ["loop", "ram", "dm-"];
type DiskStats = {
device: string;
readsCompleted: number;
sectorsRead: number;
writesCompleted: number;
sectorsWritten: number;
};
function parseProcDiskstats(content: string): DiskStats[] {
const entries: DiskStats[] = [];
for (const line of content.split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
const fields = trimmed.split(/\s+/);
if (fields.length < 14) continue;
const device = fields[2]!;
if (FILTERED_DEVICE_PREFIXES.some((prefix) => device.startsWith(prefix))) {
continue;
}
entries.push({
device,
readsCompleted: parseInt(fields[3]!, 10),
sectorsRead: parseInt(fields[5]!, 10),
writesCompleted: parseInt(fields[7]!, 10),
sectorsWritten: parseInt(fields[9]!, 10),
});
}
return entries;
}
export function startDiskIoMetrics(meterProvider: MeterProvider) {
try {
fs.accessSync("/proc/diskstats", fs.constants.R_OK);
} catch {
return;
}
const meter = meterProvider.getMeter("system-disk", "1.0.0");
const ioCounter = meter.createObservableCounter("system.disk.io", {
description: "Disk I/O bytes read and written per device",
unit: "By",
});
const opsCounter = meter.createObservableCounter("system.disk.operations", {
description: "Disk read/write operation counts per device",
unit: "{operation}",
});
meter.addBatchObservableCallback(
async (obs) => {
try {
const content = await fsPromises.readFile("/proc/diskstats", "utf-8");
const stats = parseProcDiskstats(content);
for (const entry of stats) {
const readAttrs = {
"system.device": entry.device,
"disk.io.direction": "read",
};
const writeAttrs = {
"system.device": entry.device,
"disk.io.direction": "write",
};
obs.observe(ioCounter, entry.sectorsRead * SECTOR_SIZE, readAttrs);
obs.observe(ioCounter, entry.sectorsWritten * SECTOR_SIZE, writeAttrs);
obs.observe(opsCounter, entry.readsCompleted, readAttrs);
obs.observe(opsCounter, entry.writesCompleted, writeAttrs);
}
} catch {
// Skip entire cycle on failure
}
},
[ioCounter, opsCounter]
);
}
packages/core/src/v3/otel/filesystemMetrics.ts
+134
View File
@@ -0,0 +1,134 @@
import { type MeterProvider } from "@opentelemetry/sdk-metrics";
import * as fs from "node:fs";
import * as fsPromises from "node:fs/promises";
const VIRTUAL_FS_TYPES = new Set([
"proc",
"sysfs",
"devpts",
"tmpfs",
"devtmpfs",
"cgroup",
"cgroup2",
"squashfs",
"autofs",
"debugfs",
"securityfs",
"pstore",
"bpf",
"tracefs",
"hugetlbfs",
"mqueue",
"fusectl",
"configfs",
"binfmt_misc",
]);
type MountEntry = {
device: string;
mountpoint: string;
fsType: string;
options: string;
};
function parseProcMounts(content: string): MountEntry[] {
const entries: MountEntry[] = [];
for (const line of content.split("\n")) {
if (!line.trim()) continue;
const parts = line.split(" ");
if (parts.length < 4) continue;
const fsType = parts[2]!;
if (VIRTUAL_FS_TYPES.has(fsType)) continue;
entries.push({
device: parts[0]!,
mountpoint: unescapeMountPath(parts[1]!),
fsType,
options: parts[3]!,
});
}
return entries;
}
function unescapeMountPath(path: string): string {
return path.replace(/\\040/g, " ").replace(/\\011/g, "\t");
}
export function startFilesystemMetrics(meterProvider: MeterProvider) {
try {
fs.accessSync("/proc/mounts", fs.constants.R_OK);
} catch {
return;
}
if (typeof fsPromises.statfs !== "function") {
return;
}
const meter = meterProvider.getMeter("system-filesystem", "1.0.0");
const usageCounter = meter.createObservableUpDownCounter("system.filesystem.usage", {
description: "Filesystem bytes used, free, and reserved per mountpoint",
unit: "By",
});
const utilizationGauge = meter.createObservableGauge("system.filesystem.utilization", {
description: "Fraction of filesystem space used (0-1)",
unit: "1",
});
meter.addBatchObservableCallback(
async (obs) => {
try {
const mountsContent = await fsPromises.readFile("/proc/mounts", "utf-8");
const mounts = parseProcMounts(mountsContent);
for (const mount of mounts) {
try {
const stats = await fsPromises.statfs(mount.mountpoint);
const bsize = stats.bsize;
const total = stats.blocks * bsize;
const free = stats.bavail * bsize;
const reserved = (stats.bfree - stats.bavail) * bsize;
const used = total - stats.bfree * bsize;
const mode = mount.options.startsWith("ro") ? "ro" : "rw";
const baseAttrs = {
"system.device": mount.device,
"system.filesystem.type": mount.fsType,
"system.filesystem.mountpoint": mount.mountpoint,
"system.filesystem.mode": mode,
};
obs.observe(usageCounter, used, {
...baseAttrs,
"system.filesystem.state": "used",
});
obs.observe(usageCounter, free, {
...baseAttrs,
"system.filesystem.state": "free",
});
obs.observe(usageCounter, reserved, {
...baseAttrs,
"system.filesystem.state": "reserved",
});
if (total > 0) {
obs.observe(utilizationGauge, used / total, baseAttrs);
}
} catch {
// Skip this mount on statfs failure
}
}
} catch {
// Skip entire cycle on failure
}
},
[usageCounter, utilizationGauge]
);
}
packages/core/src/v3/otel/machineId.ts
+4
View File
@@ -0,0 +1,4 @@
import { generateFriendlyId } from "../isomorphic/friendlyId.js";
import { getEnvVar } from "../utils/getEnv.js";
export const machineId = getEnvVar("TRIGGER_MACHINE_ID") ?? generateFriendlyId("machine");
packages/core/src/v3/otel/nodejsRuntimeMetrics.ts
+81
View File
@@ -0,0 +1,81 @@
import { type MeterProvider } from "@opentelemetry/sdk-metrics";
import type { ObservableGauge } from "@opentelemetry/api";
import { performance, monitorEventLoopDelay } from "node:perf_hooks";
function tryMonitorEventLoopDelay() {
try {
const eld = monitorEventLoopDelay({ resolution: 20 });
eld.enable();
return eld;
} catch {
// monitorEventLoopDelay is not implemented in Bun
return undefined;
}
}
export function startNodejsRuntimeMetrics(meterProvider: MeterProvider) {
const meter = meterProvider.getMeter("nodejs-runtime", "1.0.0");
// Event loop utilization (diff between collection intervals)
let lastElu = performance.eventLoopUtilization();
const eluGauge = meter.createObservableGauge("nodejs.event_loop.utilization", {
description: "Event loop utilization over the last collection interval",
unit: "1",
});
// Event loop delay histogram (from perf_hooks) — not available in Bun
const eld = tryMonitorEventLoopDelay();
const observables: ObservableGauge[] = [eluGauge];
let eldP95: ObservableGauge | undefined;
let eldMax: ObservableGauge | undefined;
if (eld) {
eldP95 = meter.createObservableGauge("nodejs.event_loop.delay.p95", {
description: "p95 event loop delay",
unit: "s",
});
eldMax = meter.createObservableGauge("nodejs.event_loop.delay.max", {
description: "Max event loop delay",
unit: "s",
});
observables.push(eldP95, eldMax);
}
// Heap metrics
const heapUsed = meter.createObservableGauge("nodejs.heap.used", {
description: "V8 heap used",
unit: "By",
});
const heapTotal = meter.createObservableGauge("nodejs.heap.total", {
description: "V8 heap total allocated",
unit: "By",
});
observables.push(heapUsed, heapTotal);
// Single batch callback for all metrics
meter.addBatchObservableCallback(
(obs) => {
// ELU
const currentElu = performance.eventLoopUtilization();
const diff = performance.eventLoopUtilization(currentElu, lastElu);
lastElu = currentElu;
obs.observe(eluGauge, diff.utilization);
// Event loop delay (nanoseconds -> seconds)
if (eld && eldP95 && eldMax) {
obs.observe(eldP95, eld.percentile(95) / 1e9);
obs.observe(eldMax, eld.max / 1e9);
eld.reset();
}
// Heap
const mem = process.memoryUsage();
obs.observe(heapUsed, mem.heapUsed);
obs.observe(heapTotal, mem.heapTotal);
},
observables
);
}
packages/core/src/v3/otel/tracingSDK.ts
+120 -7
View File
@@ -4,11 +4,14 @@ import {
TraceFlags,
TracerProvider,
diag,
metrics,
} from "@opentelemetry/api";
import { logs } from "@opentelemetry/api-logs";
import { TraceState } from "@opentelemetry/core";
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
import { HostMetrics } from "@opentelemetry/host-metrics";
import { registerInstrumentations, type Instrumentation } from "@opentelemetry/instrumentation";
import {
detectResources,
@@ -24,6 +27,13 @@ import {
ReadableLogRecord,
SimpleLogRecordProcessor,
} from "@opentelemetry/sdk-logs";
import {
AggregationType,
MeterProvider,
PeriodicExportingMetricReader,
type MetricReader,
type PushMetricExporter,
} from "@opentelemetry/sdk-metrics";
import { RandomIdGenerator, SpanProcessor } from "@opentelemetry/sdk-trace-base";
import {
BatchSpanProcessor,
@@ -32,7 +42,6 @@ import {
SimpleSpanProcessor,
SpanExporter,
} from "@opentelemetry/sdk-trace-node";
import { SemanticResourceAttributes, SEMATTRS_HTTP_URL } from "@opentelemetry/semantic-conventions";
import { VERSION } from "../../version.js";
import {
OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT,
@@ -47,11 +56,17 @@ import {
import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
import { taskContext } from "../task-context-api.js";
import {
BufferingMetricExporter,
TaskContextLogProcessor,
TaskContextMetricExporter,
TaskContextSpanProcessor,
} from "../taskContext/otelProcessors.js";
import { traceContext } from "../trace-context-api.js";
import { getEnvVar } from "../utils/getEnv.js";
import { machineId } from "./machineId.js";
import { startDiskIoMetrics } from "./diskIoMetrics.js";
import { startFilesystemMetrics } from "./filesystemMetrics.js";
import { startNodejsRuntimeMetrics } from "./nodejsRuntimeMetrics.js";
export type TracingDiagnosticLogLevel =
| "none"
@@ -64,12 +79,26 @@ export type TracingDiagnosticLogLevel =
export type TracingSDKConfig = {
url: string;
metricsUrl?: string;
forceFlushTimeoutMillis?: number;
instrumentations?: Instrumentation[];
exporters?: SpanExporter[];
logExporters?: LogRecordExporter[];
metricExporters?: PushMetricExporter[];
metricReaders?: MetricReader[];
diagLogLevel?: TracingDiagnosticLogLevel;
resource?: Resource;
hostMetrics?: boolean;
/** Limit host metrics collection to specific groups (e.g. ["process.cpu", "process.memory"]) */
hostMetricGroups?: string[];
/** Enable Node.js runtime metrics (event loop utilization, heap usage, etc.) */
nodejsRuntimeMetrics?: boolean;
/** Enable filesystem metrics (Linux only, reads /proc/mounts + fs.statfs) */
filesystemMetrics?: boolean;
/** Enable disk I/O metrics (Linux only, reads /proc/diskstats) */
diskIoMetrics?: boolean;
/** Metric instrument name patterns to drop (supports wildcards, e.g. "system.cpu.*") */
droppedMetrics?: string[];
};
const idGenerator = new RandomIdGenerator();
@@ -78,6 +107,7 @@ export class TracingSDK {
private readonly _logProvider: LoggerProvider;
private readonly _spanExporter: SpanExporter;
private readonly _traceProvider: NodeTracerProvider;
private readonly _meterProvider: MeterProvider;
public readonly getLogger: LoggerProvider["getLogger"];
public readonly getTracer: TracerProvider["getTracer"];
@@ -99,13 +129,13 @@ export class TracingSDK {
})
.merge(
resourceFromAttributes({
[SemanticResourceAttributes.CLOUD_PROVIDER]: "trigger.dev",
[SemanticResourceAttributes.SERVICE_NAME]:
getEnvVar("TRIGGER_OTEL_SERVICE_NAME") ?? "trigger.dev",
"cloud.provider": "trigger.dev",
"service.name": getEnvVar("TRIGGER_OTEL_SERVICE_NAME") ?? "trigger.dev",
[SemanticInternalAttributes.TRIGGER]: true,
[SemanticInternalAttributes.CLI_VERSION]: VERSION,
[SemanticInternalAttributes.SDK_VERSION]: VERSION,
[SemanticInternalAttributes.SDK_LANGUAGE]: "typescript",
[SemanticInternalAttributes.MACHINE_ID]: machineId,
})
)
.merge(resourceFromAttributes(envResourceAttributes))
@@ -259,16 +289,99 @@ export class TracingSDK {
logs.setGlobalLoggerProvider(loggerProvider);
// Metrics setup
const metricsUrl =
config.metricsUrl ??
getEnvVar("TRIGGER_OTEL_METRICS_ENDPOINT") ??
`${config.url}/v1/metrics`;
const rawMetricExporter = new OTLPMetricExporter({
url: metricsUrl,
timeoutMillis: config.forceFlushTimeoutMillis,
});
const collectionIntervalMs = parseInt(
getEnvVar("TRIGGER_OTEL_METRICS_COLLECTION_INTERVAL_MILLIS") ?? "10000"
);
const exportIntervalMs = parseInt(
getEnvVar("TRIGGER_OTEL_METRICS_EXPORT_INTERVAL_MILLIS") ?? "30000"
);
// Chain: PeriodicReader(10s) → TaskContextMetricExporter → BufferingMetricExporter(30s) → OTLP
const bufferingExporter = new BufferingMetricExporter(rawMetricExporter, exportIntervalMs);
const metricExporter = new TaskContextMetricExporter(bufferingExporter);
const exportTimeoutMillis = parseInt(
getEnvVar("TRIGGER_OTEL_METRICS_EXPORT_TIMEOUT_MILLIS") ?? "30000"
);
const metricReaders: MetricReader[] = [
new PeriodicExportingMetricReader({
exporter: metricExporter,
exportIntervalMillis: collectionIntervalMs,
exportTimeoutMillis: Math.min(exportTimeoutMillis, collectionIntervalMs),
}),
...(config.metricExporters ?? []).map(
(exporter) =>
new PeriodicExportingMetricReader({
exporter,
exportIntervalMillis: collectionIntervalMs,
exportTimeoutMillis: Math.min(exportTimeoutMillis, collectionIntervalMs),
})
),
...(config.metricReaders ?? []),
];
const meterProvider = new MeterProvider({
resource: commonResources,
readers: metricReaders,
views: (config.droppedMetrics ?? []).map((pattern) => ({
instrumentName: pattern,
aggregation: { type: AggregationType.DROP },
})),
});
this._meterProvider = meterProvider;
metrics.setGlobalMeterProvider(meterProvider);
if (config.hostMetrics) {
const hostMetrics = new HostMetrics({
meterProvider,
metricGroups: config.hostMetricGroups,
});
hostMetrics.start();
}
if (config.nodejsRuntimeMetrics) {
startNodejsRuntimeMetrics(meterProvider);
}
if (config.filesystemMetrics) {
startFilesystemMetrics(meterProvider);
}
if (config.diskIoMetrics) {
startDiskIoMetrics(meterProvider);
}
this.getLogger = loggerProvider.getLogger.bind(loggerProvider);
this.getTracer = traceProvider.getTracer.bind(traceProvider);
}
public async flush() {
await Promise.all([this._traceProvider.forceFlush(), this._logProvider.forceFlush()]);
await Promise.all([
this._traceProvider.forceFlush(),
this._logProvider.forceFlush(),
this._meterProvider.forceFlush(),
]);
}
public async shutdown() {
await Promise.all([this._traceProvider.shutdown(), this._logProvider.shutdown()]);
await Promise.all([
this._traceProvider.shutdown(),
this._logProvider.shutdown(),
this._meterProvider.shutdown(),
]);
}
}
@@ -465,7 +578,7 @@ function isSpanInternalOnly(span: ReadableSpan): boolean {
return true;
}
const httpUrl = span.attributes[SEMATTRS_HTTP_URL] ?? span.attributes["url.full"];
const httpUrl = span.attributes["http.url"] ?? span.attributes["url.full"];
const url = safeParseUrl(httpUrl);
packages/core/src/v3/schemas/messages.ts
+1
View File
@@ -213,6 +213,7 @@ export const WorkerToExecutorMessageCatalog = {
FLUSH: {
message: z.object({
timeoutInMs: z.number(),
disableContext: z.boolean().optional(),
}),
callback: z.void(),
},
packages/core/src/v3/semanticInternalAttributes.ts
+2
View File
@@ -19,6 +19,7 @@ export const SemanticInternalAttributes = {
TASK_EXPORT_NAME: "ctx.task.exportName",
QUEUE_NAME: "ctx.queue.name",
QUEUE_ID: "ctx.queue.id",
MACHINE_ID: "ctx.machine.id",
MACHINE_PRESET_NAME: "ctx.machine.name",
MACHINE_PRESET_CPU: "ctx.machine.cpu",
MACHINE_PRESET_MEMORY: "ctx.machine.memory",
@@ -65,4 +66,5 @@ export const SemanticInternalAttributes = {
WARM_START: "warm_start",
ATTEMPT_EXECUTION_COUNT: "$trigger.executionCount",
TASK_EVENT_STORE: "$trigger.taskEventStore",
RUN_TAGS: "ctx.run.tags",
};
packages/core/src/v3/taskContext/index.ts
+9 -3
View File
@@ -1,13 +1,14 @@
import { Attributes } from "@opentelemetry/api";
import { ServerBackgroundWorker, TaskRunContext } from "../schemas/index.js";
import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
import { getGlobal, registerGlobal, unregisterGlobal } from "../utils/globals.js";
import { getGlobal, registerGlobal } from "../utils/globals.js";
import { TaskContext } from "./types.js";
const API_NAME = "task-context";
export class TaskContextAPI {
private static _instance?: TaskContextAPI;
private _runDisabled = false;
private constructor() {}
@@ -23,6 +24,10 @@ export class TaskContextAPI {
return this.#getTaskContext() !== undefined;
}
get isRunDisabled(): boolean {
return this._runDisabled;
}
get ctx(): TaskRunContext | undefined {
return this.#getTaskContext()?.ctx;
}
@@ -98,11 +103,12 @@ export class TaskContextAPI {
}
public disable() {
unregisterGlobal(API_NAME);
this._runDisabled = true;
}
public setGlobalTaskContext(taskContext: TaskContext): boolean {
return registerGlobal(API_NAME, taskContext);
this._runDisabled = false;
return registerGlobal(API_NAME, taskContext, true);
}
#getTaskContext(): TaskContext | undefined {
packages/core/src/v3/taskContext/otelProcessors.ts
+202 -1
View File
@@ -1,5 +1,15 @@
import { Context, trace, Tracer } from "@opentelemetry/api";
import { Attributes, Context, trace, Tracer } from "@opentelemetry/api";
import { ExportResult, ExportResultCode } from "@opentelemetry/core";
import { LogRecordProcessor, SdkLogRecord } from "@opentelemetry/sdk-logs";
import type {
AggregationOption,
AggregationTemporality,
InstrumentType,
MetricData,
PushMetricExporter,
ResourceMetrics,
ScopeMetrics,
} from "@opentelemetry/sdk-metrics";
import { Span, SpanProcessor } from "@opentelemetry/sdk-trace-base";
import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
import { taskContext } from "../task-context-api.js";
@@ -104,3 +114,194 @@ export class TaskContextLogProcessor implements LogRecordProcessor {
return this._innerProcessor.shutdown();
}
}
export class TaskContextMetricExporter implements PushMetricExporter {
selectAggregationTemporality?: (instrumentType: InstrumentType) => AggregationTemporality;
selectAggregation?: (instrumentType: InstrumentType) => AggregationOption;
constructor(private _innerExporter: PushMetricExporter) {
if (_innerExporter.selectAggregationTemporality) {
this.selectAggregationTemporality =
_innerExporter.selectAggregationTemporality.bind(_innerExporter);
}
if (_innerExporter.selectAggregation) {
this.selectAggregation = _innerExporter.selectAggregation.bind(_innerExporter);
}
}
export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void {
if (!taskContext.ctx) {
// No task context yet — pass through without adding context attributes
this._innerExporter.export(metrics, resultCallback);
return;
}
const ctx = taskContext.ctx;
let contextAttrs: Attributes;
if (taskContext.isRunDisabled) {
// Between runs: keep environment/project/org/machine attrs, strip run-specific ones
contextAttrs = {
[SemanticInternalAttributes.ENVIRONMENT_ID]: ctx.environment.id,
[SemanticInternalAttributes.ENVIRONMENT_TYPE]: ctx.environment.type,
[SemanticInternalAttributes.ORGANIZATION_ID]: ctx.organization.id,
[SemanticInternalAttributes.PROJECT_ID]: ctx.project.id,
[SemanticInternalAttributes.MACHINE_PRESET_NAME]: ctx.machine?.name,
};
} else {
// During a run: full context attrs
contextAttrs = {
[SemanticInternalAttributes.RUN_ID]: ctx.run.id,
[SemanticInternalAttributes.TASK_SLUG]: ctx.task.id,
[SemanticInternalAttributes.ATTEMPT_NUMBER]: ctx.attempt.number,
[SemanticInternalAttributes.ENVIRONMENT_ID]: ctx.environment.id,
[SemanticInternalAttributes.ORGANIZATION_ID]: ctx.organization.id,
[SemanticInternalAttributes.PROJECT_ID]: ctx.project.id,
[SemanticInternalAttributes.MACHINE_PRESET_NAME]: ctx.machine?.name,
[SemanticInternalAttributes.ENVIRONMENT_TYPE]: ctx.environment.type,
};
}
if (taskContext.worker) {
contextAttrs[SemanticInternalAttributes.WORKER_ID] = taskContext.worker.id;
contextAttrs[SemanticInternalAttributes.WORKER_VERSION] = taskContext.worker.version;
}
if (!taskContext.isRunDisabled && ctx.run.tags?.length) {
contextAttrs[SemanticInternalAttributes.RUN_TAGS] = ctx.run.tags;
}
const modified: ResourceMetrics = {
resource: metrics.resource,
scopeMetrics: metrics.scopeMetrics.map((scope) => ({
...scope,
metrics: scope.metrics.map(
(metric) =>
({
...metric,
dataPoints: metric.dataPoints.map((dp) => ({
...dp,
attributes: { ...dp.attributes, ...contextAttrs },
})),
}) as MetricData
),
})),
};
this._innerExporter.export(modified, resultCallback);
}
forceFlush(): Promise<void> {
return this._innerExporter.forceFlush();
}
shutdown(): Promise<void> {
return this._innerExporter.shutdown();
}
}
export class BufferingMetricExporter implements PushMetricExporter {
selectAggregationTemporality?: (instrumentType: InstrumentType) => AggregationTemporality;
selectAggregation?: (instrumentType: InstrumentType) => AggregationOption;
private _buffer: ResourceMetrics[] = [];
private _lastFlushTime = Date.now();
constructor(
private _innerExporter: PushMetricExporter,
private _flushIntervalMs: number
) {
if (_innerExporter.selectAggregationTemporality) {
this.selectAggregationTemporality =
_innerExporter.selectAggregationTemporality.bind(_innerExporter);
}
if (_innerExporter.selectAggregation) {
this.selectAggregation = _innerExporter.selectAggregation.bind(_innerExporter);
}
}
export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void {
this._buffer.push(metrics);
const now = Date.now();
if (now - this._lastFlushTime >= this._flushIntervalMs) {
this._lastFlushTime = now;
const merged = this._mergeBuffer();
this._innerExporter.export(merged, resultCallback);
} else {
resultCallback({ code: ExportResultCode.SUCCESS });
}
}
forceFlush(): Promise<void> {
if (this._buffer.length > 0) {
this._lastFlushTime = Date.now();
const merged = this._mergeBuffer();
return new Promise<void>((resolve, reject) => {
this._innerExporter.export(merged, (result) => {
if (result.code === ExportResultCode.SUCCESS) {
resolve();
} else {
reject(result.error ?? new Error("Export failed"));
}
});
}).then(() => this._innerExporter.forceFlush());
}
return this._innerExporter.forceFlush();
}
shutdown(): Promise<void> {
return this.forceFlush().then(() => this._innerExporter.shutdown());
}
private _mergeBuffer(): ResourceMetrics {
const batch = this._buffer;
this._buffer = [];
if (batch.length === 1) {
return batch[0]!;
}
const base = batch[0]!;
// Merge all scopeMetrics by scope name, then metrics by descriptor name
const scopeMap = new Map<string, { scope: ScopeMetrics["scope"]; metricsMap: Map<string, MetricData> }>();
for (const rm of batch) {
for (const sm of rm.scopeMetrics) {
const scopeKey = sm.scope.name;
let scopeEntry = scopeMap.get(scopeKey);
if (!scopeEntry) {
scopeEntry = { scope: sm.scope, metricsMap: new Map() };
scopeMap.set(scopeKey, scopeEntry);
}
for (const metric of sm.metrics) {
const metricKey = metric.descriptor.name;
const existing = scopeEntry.metricsMap.get(metricKey);
if (existing) {
// Append data points from this collection to the existing metric
scopeEntry.metricsMap.set(metricKey, {
...existing,
dataPoints: [...existing.dataPoints, ...metric.dataPoints],
} as MetricData);
} else {
scopeEntry.metricsMap.set(metricKey, {
...metric,
dataPoints: [...metric.dataPoints],
} as MetricData);
}
}
}
}
return {
resource: base.resource,
scopeMetrics: Array.from(scopeMap.values()).map(({ scope, metricsMap }) => ({
scope,
metrics: Array.from(metricsMap.values()),
})),
};
}
}
packages/core/src/v3/workers/index.ts
+1
View File
@@ -14,6 +14,7 @@ export { StandardResourceCatalog } from "../resource-catalog/standardResourceCat
export {
TaskContextSpanProcessor,
TaskContextLogProcessor,
TaskContextMetricExporter,
} from "../taskContext/otelProcessors.js";
export * from "../usage-api.js";
export { DevUsageManager } from "../usage/devUsageManager.js";
packages/python/CHANGELOG.md
+18
View File
@@ -1,5 +1,23 @@
# @trigger.dev/python
## 4.4.1
### Patch Changes
- Updated dependencies:
- `@trigger.dev/sdk@4.4.1`
- `@trigger.dev/build@4.4.1`
- `@trigger.dev/core@4.4.1`
## 4.4.0
### Patch Changes
- Updated dependencies:
- `@trigger.dev/core@4.4.0`
- `@trigger.dev/sdk@4.4.0`
- `@trigger.dev/build@4.4.0`
## 4.3.3
### Patch Changes
packages/python/package.json
+6 -6
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/python",
"version": "4.3.3",
"version": "4.4.1",
"description": "Python runtime and build extension for Trigger.dev",
"license": "MIT",
"publishConfig": {
@@ -45,7 +45,7 @@
"check-exports": "attw --pack ."
},
"dependencies": {
"@trigger.dev/core": "workspace:4.3.3",
"@trigger.dev/core": "workspace:4.4.1",
"tinyexec": "^0.3.2"
},
"devDependencies": {
@@ -56,12 +56,12 @@
"tsx": "4.17.0",
"esbuild": "^0.23.0",
"@arethetypeswrong/cli": "^0.15.4",
"@trigger.dev/build": "workspace:4.3.3",
"@trigger.dev/sdk": "workspace:4.3.3"
"@trigger.dev/build": "workspace:4.4.1",
"@trigger.dev/sdk": "workspace:4.4.1"
},
"peerDependencies": {
"@trigger.dev/sdk": "workspace:^4.3.3",
"@trigger.dev/build": "workspace:^4.3.3"
"@trigger.dev/sdk": "workspace:^4.4.1",
"@trigger.dev/build": "workspace:^4.4.1"
},
"engines": {
"node": ">=18.20.0"
packages/react-hooks/CHANGELOG.md
+15
View File
@@ -1,5 +1,20 @@
# @trigger.dev/react-hooks
## 4.4.1
### Patch Changes
- Updated dependencies:
- `@trigger.dev/core@4.4.1`
## 4.4.0
### Patch Changes
- Fix `onComplete` callback firing prematurely when the realtime stream disconnects before the run finishes. ([#2929](https://github.com/triggerdotdev/trigger.dev/pull/2929))
- Updated dependencies:
- `@trigger.dev/core@4.4.0`
## 4.3.3
### Patch Changes
packages/react-hooks/package.json
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@trigger.dev/react-hooks",
"version": "4.3.3",
"version": "4.4.1",
"description": "trigger.dev react hooks",
"license": "MIT",
"publishConfig": {
@@ -37,7 +37,7 @@
"check-exports": "attw --pack ."
},
"dependencies": {
"@trigger.dev/core": "workspace:^4.3.3",
"@trigger.dev/core": "workspace:^4.4.1",
"swr": "^2.2.5"
},
"devDependencies": {

Some files were not shown because too many files have changed in this diff Show More