Release 3.0.0-beta.44

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/beige-pears-explode.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add an e2e suite to test compiling with v3 CLI.

.changeset/hot-wasps-sin.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+v3: Usage tracking

.changeset/mighty-parrots-sin.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: Remove aggressive otel flush timeouts in dev/prod

.changeset/polite-pears-grow.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Use global setTimeout to ensure cross-runtime support

.changeset/pre.json

@@ -46,6 +46,7 @@
   "changesets": [
     "afraid-sheep-joke",
     "angry-eagles-trade",
+    "beige-pears-explode",
     "beige-pens-dance",
     "big-tomatoes-deliver",
     "blue-pumas-whisper",
@@ -71,6 +72,7 @@
     "green-bags-wink",
     "hot-buckets-behave",
     "hot-fishes-retire",
+    "hot-wasps-sin",
     "itchy-chairs-itch",
     "khaki-apricots-design",
     "khaki-poems-lay",
@@ -87,6 +89,7 @@
     "many-ligers-pump",
     "mighty-camels-joke",
     "mighty-flowers-train",
+    "mighty-parrots-sin",
     "nasty-jars-pump",
     "new-pants-beg",
     "new-rivers-tell",
@@ -96,6 +99,7 @@
     "pink-pumas-rhyme",
     "plenty-ducks-beam",
     "polite-ducks-switch",
+    "polite-pears-grow",
     "polite-rockets-matter",
     "poor-flowers-cross",
     "purple-garlics-shop",
@@ -107,31 +111,41 @@
     "rotten-dryers-exercise",
     "rude-toys-compare",
     "selfish-ducks-sort",
+    "serious-hats-rest",
     "shaggy-spoons-taste",
     "sharp-emus-compare",
     "sharp-zebras-serve",
     "shiny-coats-cry",
+    "silly-buses-obey",
     "silly-suits-switch",
+    "silver-doors-juggle",
     "six-ligers-exist",
+    "six-rats-hunt",
     "sixty-insects-watch",
     "slow-buses-own",
+    "slow-kiwis-hide",
     "slow-sloths-retire",
     "smart-needles-move",
     "smart-olives-eat",
+    "sour-pugs-teach",
     "spicy-lamps-smoke",
     "spicy-terms-bow",
     "strange-ghosts-matter",
     "strange-sheep-pull",
     "strong-lemons-add",
     "strong-owls-know",
+    "strong-phones-smoke",
     "stupid-adults-sniff",
     "stupid-bulldogs-applaud",
+    "sweet-ducks-remember",
     "sweet-lizards-press",
     "swift-dragons-peel",
     "tall-bees-wave",
     "tame-guests-know",
     "tender-moose-tell",
     "tender-oranges-rhyme",
+    "tender-turkeys-compete",
+    "thick-carrots-sneeze",
     "thin-parents-heal",
     "thirty-islands-kiss",
     "tidy-balloons-suffer",
@@ -142,7 +156,9 @@
     "tricky-bulldogs-heal",
     "tricky-keys-attack",
     "tricky-ladybugs-unite",
+    "twelve-knives-notice",
     "two-pumas-wait",
+    "violet-clocks-notice",
     "warm-olives-provide",
     "warm-planes-taste",
     "young-snails-sell"

.changeset/serious-hats-rest.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: [prod] force flush timeout should be 1s

.changeset/silly-buses-obey.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add callback to checkpoint created message

.changeset/silver-doors-juggle.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Make deduplicationKey required when creating/updating a schedule

.changeset/six-rats-hunt.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improved ESM module require error detection logic

.changeset/slow-kiwis-hide.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: vercel edge runtime support

.changeset/sour-pugs-teach.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: fix otel flushing causing CLEANUP ack timeout errors by always setting a forceFlushTimeoutMillis value

.changeset/strong-phones-smoke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+fix: allow command login to read api url from cli args

.changeset/sweet-ducks-remember.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+v3: Adding SDK functions for triggering tasks in a typesafe way, without importing task file

.changeset/tender-turkeys-compete.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added timezone support to schedules

.changeset/thick-carrots-sneeze.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+v3: Include presigned urls for downloading large payloads and outputs when using runs.retrieve

.changeset/twelve-knives-notice.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: fix missing init output in task run function when no middleware is defined

.changeset/violet-clocks-notice.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix jsonc-parser import

.github/workflows/e2e.yml

@@ -1,9 +1,53 @@
-name: "🧪 E2E Tests"
+name: "E2E"
 on:
   workflow_call:
+    inputs:
+      package:
+        description: The identifier of the job to run
+        default: webapp
+        required: false
+        type: string
 jobs:
-  e2e:
-    name: "🧪 E2E Tests"
+  cli-v3:
+    name: "🧪 CLI v3 tests"
+    if: inputs.package == 'cli-v3' || inputs.package == ''
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    strategy:
+      fail-fast: false
+      matrix:
+        package-manager: ["npm", "pnpm", "yarn"]
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile --filter trigger.dev...
+
+      - name: 🔧 Build v3 cli monorepo dependencies
+        run: pnpm run build --filter trigger.dev^...
+
+      - name: 🔧 Build worker template files
+        run: pnpm --filter trigger.dev run build:workers
+
+      - name: Run E2E Tests
+        run: |
+          PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e
+  webapp:
+    name: "🧪 Webapp tests"
+    if: inputs.package == 'webapp' || inputs.package == ''
     runs-on: buildjet-16vcpu-ubuntu-2204
     steps:
       - name: 🐳 Login to Docker Hub

.github/workflows/pr_checks.yml

@@ -29,4 +29,6 @@ jobs:
 
   # e2e:
   #   uses: ./.github/workflows/e2e.yml
+  #   with:
+  #     package: webapp
   #   secrets: inherit

.github/workflows/publish.yml

@@ -49,9 +49,11 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  # e2e:
-  #   uses: ./.github/workflows/e2e.yml
-  #   secrets: inherit
+  e2e:
+    uses: ./.github/workflows/e2e.yml
+    with:
+      package: cli-v3
+    secrets: inherit
 
   publish:
     needs: [typecheck, units]

apps/coordinator/src/index.ts

@@ -1162,13 +1162,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (!checkpoint.docker || !willSimulate) {
-            socket.emit("REQUEST_EXIT", {
-              version: "v1",
-            });
-          }
-
-          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
             version: "v1",
             attemptFriendlyId: message.attemptFriendlyId,
             docker: checkpoint.docker,
@@ -1179,6 +1173,17 @@ class TaskCoordinator {
               now: message.now,
             },
           });
+
+          if (ack?.keepRunAlive) {
+            logger.log("keeping run alive after duration checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
         });
 
         socket.on("WAIT_FOR_TASK", async (message, callback) => {
@@ -1205,13 +1210,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (!checkpoint.docker || !willSimulate) {
-            socket.emit("REQUEST_EXIT", {
-              version: "v1",
-            });
-          }
-
-          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
             version: "v1",
             attemptFriendlyId: message.attemptFriendlyId,
             docker: checkpoint.docker,
@@ -1221,6 +1220,17 @@ class TaskCoordinator {
               friendlyId: message.friendlyId,
             },
           });
+
+          if (ack?.keepRunAlive) {
+            logger.log("keeping run alive after task checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
         });
 
         socket.on("WAIT_FOR_BATCH", async (message, callback) => {
@@ -1247,13 +1257,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (!checkpoint.docker || !willSimulate) {
-            socket.emit("REQUEST_EXIT", {
-              version: "v1",
-            });
-          }
-
-          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
             version: "v1",
             attemptFriendlyId: message.attemptFriendlyId,
             docker: checkpoint.docker,
@@ -1264,6 +1268,17 @@ class TaskCoordinator {
               runFriendlyIds: message.runFriendlyIds,
             },
           });
+
+          if (ack?.keepRunAlive) {
+            logger.log("keeping run alive after batch checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
         });
 
         socket.on("INDEX_TASKS", async (message, callback) => {

apps/kubernetes-provider/src/index.ts

@@ -7,7 +7,12 @@ import {
   TaskOperationsIndexOptions,
   TaskOperationsRestoreOptions,
 } from "@trigger.dev/core-apps";
-import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
+import {
+  MachinePreset,
+  PostStartCauses,
+  PreStopCauses,
+  EnvironmentType,
+} from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
 import { TaskMonitor } from "./taskMonitor";
 import { PodCleaner } from "./podCleaner";
@@ -16,6 +21,7 @@ const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
 const OTEL_EXPORTER_OTLP_ENDPOINT =
   process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
+const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");
 
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 logger.log(`running in ${RUNTIME_ENV} mode`);
@@ -398,10 +404,10 @@ class KubernetesTaskOperations implements TaskOperations {
     };
   }
 
-  #getResourcesFromMachineConfig(config: Machine): ComputeResources {
+  #getResourcesFromMachineConfig(preset: MachinePreset): ComputeResources {
     return {
-      cpu: `${config.cpu}`,
-      memory: `${config.memory}G`,
+      cpu: `${preset.cpu}`,
+      memory: `${preset.memory}G`,
     };
   }
 
@@ -555,7 +561,7 @@ taskMonitor.start();
 const podCleaner = new PodCleaner({
   runtimeEnv: RUNTIME_ENV,
   namespace: "default",
-  intervalInSeconds: 300,
+  intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS,
 });
 
 podCleaner.start();

apps/webapp/app/components/primitives/DateTime.tsx

@@ -6,6 +6,7 @@ type DateTimeProps = {
   timeZone?: string;
   includeSeconds?: boolean;
   includeTime?: boolean;
+  showTimezone?: boolean;
 };
 
 export const DateTime = ({
@@ -13,6 +14,7 @@ export const DateTime = ({
   timeZone,
   includeSeconds = true,
   includeTime = true,
+  showTimezone = false,
 }: DateTimeProps) => {
   const locales = useLocales();
 
@@ -42,7 +44,12 @@ export const DateTime = ({
     );
   }, [locales, includeSeconds, realDate]);
 
-  return <Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>;
+  return (
+    <Fragment>
+      {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}
+      {showTimezone ? ` (${timeZone ?? "UTC"})` : null}
+    </Fragment>
+  );
 };
 
 export function formatDateTime(

apps/webapp/app/components/primitives/Select.tsx

@@ -8,6 +8,7 @@ import { ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys";
 import { cn } from "~/utils/cn";
 import { ShortcutKey } from "./ShortcutKey";
 import { ChevronDown } from "lucide-react";
+import { MatchSorterOptions, matchSorter } from "match-sorter";
 
 const sizes = {
   small: {
@@ -75,7 +76,10 @@ export interface SelectProps<TValue extends string | string[], TItem>
   showHeading?: boolean;
   items?: TItem[] | Section<TItem>[];
   empty?: React.ReactNode;
-  filter?: (item: ItemFromSection<TItem>, search: string, title?: string) => boolean;
+  filter?:
+    | boolean
+    | MatchSorterOptions<TItem>
+    | ((item: ItemFromSection<TItem>, search: string, title?: string) => boolean);
   children:
     | React.ReactNode
     | ((
@@ -129,18 +133,44 @@ export function Select<TValue extends string | string[], TItem>({
     if (!items) return [];
     if (!searchValue || !filter) return items;
 
+    if (typeof filter === "function") {
+      if (isSection(items)) {
+        return items
+          .map((section) => ({
+            ...section,
+            items: section.items.filter((item) =>
+              filter(item as ItemFromSection<TItem>, searchValue, section.title)
+            ),
+          }))
+          .filter((section) => section.items.length > 0);
+      }
+
+      return items.filter((item) => filter(item as ItemFromSection<TItem>, searchValue));
+    }
+
+    if (typeof filter === "boolean" && filter) {
+      if (isSection(items)) {
+        return items
+          .map((section) => ({
+            ...section,
+            items: matchSorter(section.items, searchValue),
+          }))
+          .filter((section) => section.items.length > 0);
+      }
+
+      return matchSorter(items, searchValue);
+    }
+
     if (isSection(items)) {
       return items
         .map((section) => ({
           ...section,
-          items: section.items.filter((item) =>
-            filter(item as ItemFromSection<TItem>, searchValue, section.title)
-          ),
+          items: matchSorter(section.items, searchValue, filter),
         }))
         .filter((section) => section.items.length > 0);
     }
 
-    return items.filter((item) => filter(item as ItemFromSection<TItem>, searchValue));
+    return matchSorter(items, searchValue, filter);
   }, [searchValue, items]);
 
   const enableItemShortcuts = allowItemShortcuts && matches.length === items?.length;

apps/webapp/app/components/primitives/TreeView/TreeView.tsx

@@ -528,7 +528,7 @@ export type Tree<TData> = {
 /** A tree but flattened so it can easily be used for DOM elements */
 export type FlatTreeItem<TData> = {
   id: string;
-  parentId: string | undefined;
+  parentId?: string | undefined;
   children: string[];
   hasChildren: boolean;
   /** The indentation level, the root is 0 */

apps/webapp/app/components/scheduled/timezones.tsx

@@ -0,0 +1,63 @@
+import { useVirtualizer } from "@tanstack/react-virtual";
+import { useRef } from "react";
+import { SelectItem } from "../primitives/Select";
+
+export function TimezoneList({ timezones }: { timezones: string[] }) {
+  const parentRef = useRef<HTMLDivElement>(null);
+
+  const rowVirtualizer = useVirtualizer({
+    count: timezones.length,
+    getScrollElement: () => parentRef.current,
+    estimateSize: () => 28,
+  });
+
+  return (
+    <div
+      ref={parentRef}
+      className="max-h-[calc(min(480px,var(--popover-available-height))-2.35rem)] overflow-y-auto overscroll-contain scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
+    >
+      <div
+        style={{
+          height: `${rowVirtualizer.getTotalSize()}px`,
+          width: "100%",
+          position: "relative",
+        }}
+      >
+        {rowVirtualizer.getVirtualItems().map((virtualItem) => (
+          <TimezoneCell
+            key={virtualItem.key}
+            size={virtualItem.size}
+            start={virtualItem.start}
+            timezone={timezones[virtualItem.index]}
+          />
+        ))}
+      </div>
+    </div>
+  );
+}
+
+function TimezoneCell({
+  timezone,
+  size,
+  start,
+}: {
+  timezone: string;
+  size: number;
+  start: number;
+}) {
+  return (
+    <SelectItem
+      value={timezone}
+      style={{
+        position: "absolute",
+        top: 0,
+        left: 0,
+        width: "100%",
+        height: `${size}px`,
+        transform: `translateY(${start}px)`,
+      }}
+    >
+      {timezone}
+    </SelectItem>
+  );
+}

apps/webapp/app/entry.server.tsx

@@ -186,3 +186,9 @@ export { apiRateLimiter } from "./services/apiRateLimit.server";
 export { socketIo } from "./v3/handleSocketIo.server";
 export { wss } from "./v3/handleWebsockets.server";
 export { registryProxy } from "./v3/registryProxy.server";
+import { eventLoopMonitor } from "./eventLoopMonitor.server";
+import { env } from "./env.server";
+
+if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
+  eventLoopMonitor.enable();
+}

apps/webapp/app/env.server.ts

@@ -1,5 +1,5 @@
-import { z } from "zod";
 import { SecretStoreOptionsSchema } from "./services/secrets/secretStoreOptionsSchema.server";
+import { z } from "zod";
 import { isValidRegex } from "./utils/regex";
 import { isValidDatabaseUrl } from "./utils/db";
 
@@ -189,6 +189,26 @@ const EnvironmentSchema = z.object({
   V2_MARQS_VERBOSE: z.string().default("0"),
   V3_MARQS_CONCURRENCY_MONITOR_ENABLED: z.string().default("0"),
   V2_MARQS_CONCURRENCY_MONITOR_ENABLED: z.string().default("0"),
+  /* Usage settings */
+  USAGE_EVENT_URL: z.string().optional(),
+  PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(),
+
+  CENTS_PER_HOUR_MICRO: z.coerce.number().default(0),
+  CENTS_PER_HOUR_SMALL_1X: z.coerce.number().default(0),
+  CENTS_PER_HOUR_SMALL_2X: z.coerce.number().default(0),
+  CENTS_PER_HOUR_MEDIUM_1X: z.coerce.number().default(0),
+  CENTS_PER_HOUR_MEDIUM_2X: z.coerce.number().default(0),
+  CENTS_PER_HOUR_LARGE_1X: z.coerce.number().default(0),
+  CENTS_PER_HOUR_LARGE_2X: z.coerce.number().default(0),
+  BASE_RUN_COST_IN_CENTS: z.coerce.number().default(0),
+
+  USAGE_OPEN_METER_API_KEY: z.string().optional(),
+  USAGE_OPEN_METER_BASE_URL: z.string().optional(),
+  EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
+  MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
+  MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
+  TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB
+  TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/eventLoopMonitor.server.ts

@@ -0,0 +1,77 @@
+import { createHook } from "node:async_hooks";
+import { singleton } from "./utils/singleton";
+import { tracer } from "./v3/tracer.server";
+
+const THRESHOLD_NS = 1e8; // 100ms
+
+const cache = new Map<number, { type: string; start?: [number, number] }>();
+
+function init(asyncId: number, type: string, triggerAsyncId: number, resource: any) {
+  cache.set(asyncId, {
+    type,
+  });
+}
+
+function destroy(asyncId: number) {
+  cache.delete(asyncId);
+}
+
+function before(asyncId: number) {
+  const cached = cache.get(asyncId);
+
+  if (!cached) {
+    return;
+  }
+
+  cache.set(asyncId, {
+    ...cached,
+    start: process.hrtime(),
+  });
+}
+
+function after(asyncId: number) {
+  const cached = cache.get(asyncId);
+
+  if (!cached) {
+    return;
+  }
+
+  cache.delete(asyncId);
+
+  if (!cached.start) {
+    return;
+  }
+
+  const diff = process.hrtime(cached.start);
+  const diffNs = diff[0] * 1e9 + diff[1];
+  if (diffNs > THRESHOLD_NS) {
+    const time = diffNs / 1e6; // in ms
+
+    const newSpan = tracer.startSpan("event-loop-blocked", {
+      startTime: new Date(new Date().getTime() - time),
+      attributes: {
+        asyncType: cached.type,
+        label: "EventLoopMonitor",
+      },
+    });
+
+    newSpan.end();
+  }
+}
+
+export const eventLoopMonitor = singleton("eventLoopMonitor", () => {
+  const hook = createHook({ init, before, after, destroy });
+
+  return {
+    enable: () => {
+      console.log("🥸  Initializing event loop monitor");
+
+      hook.enable();
+    },
+    disable: () => {
+      console.log("🥸  Disabling event loop monitor");
+
+      hook.disable();
+    },
+  };
+});

apps/webapp/app/hooks/useEventSource.tsx

@@ -26,7 +26,7 @@ export function useEventSource(
     const eventSource = new EventSource(url, init);
     eventSource.addEventListener(event ?? "message", handler);
 
-    // rest data if dependencies change
+    // reset data if dependencies change
     setData(null);
 
     function handler(event: MessageEvent) {

apps/webapp/app/models/endpoint.server.ts

@@ -27,6 +27,7 @@ export function detectResponseIsTimeout(rawBody: string, response?: Response) {
 
   return (
     isResponseVercelTimeout(response) ||
+    isResponseCloudfrontTimeout(response) ||
     isResponseDenoDeployTimeout(rawBody, response) ||
     isResponseCloudflareTimeout(rawBody, response)
   );
@@ -50,3 +51,7 @@ function isResponseVercelTimeout(response: Response) {
 function isResponseDenoDeployTimeout(rawBody: string, response: Response) {
   return response.status === 502 && rawBody.includes("TIME_LIMIT");
 }
+
+function isResponseCloudfrontTimeout(response: Response) {
+  return response.status === 504 && typeof response.headers.get("x-amz-cf-id") === "string";
+}

apps/webapp/app/models/runtimeEnvironment.server.ts

@@ -65,59 +65,55 @@ export async function findEnvironmentById(id: string) {
 }
 
 export async function createNewSession(environment: RuntimeEnvironment, ipAddress: string) {
-  return prisma.$transaction(async (tx) => {
-    const session = await tx.runtimeEnvironmentSession.create({
-      data: {
-        environmentId: environment.id,
-        ipAddress,
-      },
-    });
-
-    await tx.runtimeEnvironment.update({
-      where: {
-        id: environment.id,
-      },
-      data: {
-        currentSessionId: session.id,
-      },
-    });
-
-    return session;
+  const session = await prisma.runtimeEnvironmentSession.create({
+    data: {
+      environmentId: environment.id,
+      ipAddress,
+    },
   });
+
+  await prisma.runtimeEnvironment.update({
+    where: {
+      id: environment.id,
+    },
+    data: {
+      currentSessionId: session.id,
+    },
+  });
+
+  return session;
 }
 
 export async function disconnectSession(environmentId: string) {
-  return prisma.$transaction(async (tx) => {
-    const environment = await tx.runtimeEnvironment.findUnique({
-      where: {
-        id: environmentId,
-      },
-    });
-
-    if (!environment || !environment.currentSessionId) {
-      return null;
-    }
-
-    const session = await tx.runtimeEnvironmentSession.update({
-      where: {
-        id: environment.currentSessionId,
-      },
-      data: {
-        disconnectedAt: new Date(),
-      },
-    });
-
-    await tx.runtimeEnvironment.update({
-      where: {
-        id: environment.id,
-      },
-      data: {
-        currentSessionId: null,
-      },
-    });
-
-    return session;
+  const environment = await prisma.runtimeEnvironment.findUnique({
+    where: {
+      id: environmentId,
+    },
   });
+
+  if (!environment || !environment.currentSessionId) {
+    return null;
+  }
+
+  const session = await prisma.runtimeEnvironmentSession.update({
+    where: {
+      id: environment.currentSessionId,
+    },
+    data: {
+      disconnectedAt: new Date(),
+    },
+  });
+
+  await prisma.runtimeEnvironment.update({
+    where: {
+      id: environment.id,
+    },
+    data: {
+      currentSessionId: null,
+    },
+  });
+
+  return session;
 }
 
 type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{

apps/webapp/app/presenters/RunListPresenter.server.ts

@@ -1,11 +1,9 @@
-import { z } from "zod";
 import {
   Direction,
   FilterableEnvironment,
   FilterableStatus,
   filterableStatuses,
 } from "~/components/runs/RunStatuses";
-import { PrismaClient, prisma } from "~/db.server";
 import { getUsername } from "~/utils/username";
 import { BasePresenter } from "./v3/basePresenter.server";
 
@@ -29,8 +27,6 @@ const DEFAULT_PAGE_SIZE = 20;
 export type RunList = Awaited<ReturnType<RunListPresenter["call"]>>;
 
 export class RunListPresenter extends BasePresenter {
-  
-
   public async call({
     userId,
     eventId,

apps/webapp/app/presenters/ScheduledTriggersPresenter.server.ts

@@ -1,27 +1,52 @@
-import { User } from "@trigger.dev/database";
 import { ScheduleMetadataSchema } from "@trigger.dev/core";
-import { PrismaClient, prisma } from "~/db.server";
+import { User } from "@trigger.dev/database";
 import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
 import { calculateNextScheduledEvent } from "~/services/schedules/nextScheduledEvent.server";
+import { BasePresenter } from "./v3/basePresenter.server";
 
-export class ScheduledTriggersPresenter {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
+const DEFAULT_PAGE_SIZE = 20;
 
+export class ScheduledTriggersPresenter extends BasePresenter {
   public async call({
     userId,
     projectSlug,
     organizationSlug,
+    direction = "forward",
+    pageSize = DEFAULT_PAGE_SIZE,
+    cursor,
   }: {
     userId: User["id"];
     projectSlug: Project["slug"];
     organizationSlug: Organization["slug"];
+    direction?: "forward" | "backward";
+    pageSize?: number;
+    cursor?: string;
   }) {
-    const scheduled = await this.#prismaClient.scheduleSource.findMany({
+    const organization = await this._replica.organization.findFirstOrThrow({
+      select: {
+        id: true,
+      },
+      where: {
+        slug: organizationSlug,
+        members: { some: { userId } },
+      },
+    });
+
+    // Find the project scoped to the organization
+    const project = await this._replica.project.findFirstOrThrow({
+      select: {
+        id: true,
+      },
+      where: {
+        slug: projectSlug,
+        organizationId: organization.id,
+      },
+    });
+
+    const directionMultiplier = direction === "forward" ? 1 : -1;
+
+    const scheduled = await this._replica.scheduleSource.findMany({
       select: {
         id: true,
         key: true,
@@ -50,23 +75,50 @@ export class ScheduledTriggersPresenter {
               },
             },
           ],
-          organization: {
-            slug: organizationSlug,
-            members: {
-              some: {
-                userId,
-              },
-            },
-          },
-          project: {
-            slug: projectSlug,
-          },
+          projectId: project.id,
         },
       },
+      orderBy: [{ id: "desc" }],
+      //take an extra record to tell if there are more
+      take: directionMultiplier * (pageSize + 1),
+      //skip the cursor if there is one
+      skip: cursor ? 1 : 0,
+      cursor: cursor
+        ? {
+            id: cursor,
+          }
+        : undefined,
     });
 
+    const hasMore = scheduled.length > pageSize;
+
+    //get cursors for next and previous pages
+    let next: string | undefined;
+    let previous: string | undefined;
+    switch (direction) {
+      case "forward":
+        previous = cursor ? scheduled.at(0)?.id : undefined;
+        if (hasMore) {
+          next = scheduled[pageSize - 1]?.id;
+        }
+        break;
+      case "backward":
+        if (hasMore) {
+          previous = scheduled[1]?.id;
+          next = scheduled[pageSize]?.id;
+        } else {
+          next = scheduled[pageSize - 1]?.id;
+        }
+        break;
+    }
+
+    const scheduledToReturn =
+      direction === "backward" && hasMore
+        ? scheduled.slice(1, pageSize + 1)
+        : scheduled.slice(0, pageSize);
+
     return {
-      scheduled: scheduled.map((s) => {
+      scheduled: scheduledToReturn.map((s) => {
         const schedule = ScheduleMetadataSchema.parse(s.schedule);
         const nextEventTimestamp = s.active
           ? calculateNextScheduledEvent(schedule, s.lastEventTimestamp)
@@ -78,6 +130,10 @@ export class ScheduledTriggersPresenter {
           nextEventTimestamp,
         };
       }),
+      pagination: {
+        next,
+        previous,
+      },
     };
   }
 }

apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

@@ -12,6 +12,7 @@ import {
 import { Prisma, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { generatePresignedUrl } from "~/v3/r2.server";
 import { BasePresenter } from "./basePresenter.server";
 
 export class ApiRetrieveRunPresenter extends BasePresenter {
@@ -44,7 +45,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
       }
 
       let $payload: any;
+      let $payloadPresignedUrl: string | undefined;
       let $output: any;
+      let $outputPresignedUrl: string | undefined;
 
       if (showSecretDetails) {
         const payloadPacket = await conditionallyImportPacket({
@@ -52,7 +55,19 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
           dataType: taskRun.payloadType,
         });
 
-        $payload = await parsePacket(payloadPacket);
+        if (
+          payloadPacket.dataType === "application/store" &&
+          typeof payloadPacket.data === "string"
+        ) {
+          $payloadPresignedUrl = await generatePresignedUrl(
+            env.project.externalRef,
+            env.slug,
+            payloadPacket.data,
+            "GET"
+          );
+        } else {
+          $payload = await parsePacket(payloadPacket);
+        }
 
         if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
           const completedAttempt = taskRun.attempts.find(
@@ -65,7 +80,19 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
               dataType: completedAttempt.outputType,
             });
 
-            $output = await parsePacket(outputPacket);
+            if (
+              outputPacket.dataType === "application/store" &&
+              typeof outputPacket.data === "string"
+            ) {
+              $outputPresignedUrl = await generatePresignedUrl(
+                env.project.externalRef,
+                env.slug,
+                outputPacket.data,
+                "GET"
+              );
+            } else {
+              $output = await parsePacket(outputPacket);
+            }
           }
         }
       }
@@ -80,12 +107,14 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
         version: taskRun.lockedToVersion ? taskRun.lockedToVersion.version : undefined,
         createdAt: taskRun.createdAt ?? undefined,
         updatedAt: taskRun.updatedAt ?? undefined,
-        startedAt: taskRun.lockedAt ?? undefined,
+        startedAt: taskRun.startedAt ?? taskRun.lockedAt ?? undefined,
         finishedAt: ApiRetrieveRunPresenter.isStatusFinished(apiStatus)
           ? taskRun.updatedAt
           : undefined,
         payload: $payload,
+        payloadPresignedUrl: $payloadPresignedUrl,
         output: $output,
+        outputPresignedUrl: $outputPresignedUrl,
         isTest: taskRun.isTest,
         schedule: taskRun.schedule
           ? {

apps/webapp/app/presenters/v3/EditSchedulePresenter.server.ts

@@ -1,6 +1,7 @@
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { PrismaClient, prisma } from "~/db.server";
 import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
+import { getTimezones } from "~/utils/timezones.server";
 
 type EditScheduleOptions = {
   userId: string;
@@ -74,6 +75,7 @@ export class EditSchedulePresenter {
     return {
       possibleTasks: possibleTasks.map((task) => task.slug),
       possibleEnvironments,
+      possibleTimezones: getTimezones(),
       schedule: await this.#getExistingSchedule(friendlyId, possibleEnvironments),
     };
   }
@@ -91,6 +93,7 @@ export class EditSchedulePresenter {
         externalId: true,
         deduplicationKey: true,
         userProvidedDeduplicationKey: true,
+        timezone: true,
         taskIdentifier: true,
         instances: {
           select: {

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -4,8 +4,8 @@ import { Direction } from "~/components/runs/RunStatuses";
 import { FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { sqlDatabaseSchema } from "~/db.server";
 import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
-import { CANCELLABLE_STATUSES } from "~/v3/services/cancelTaskRun.server";
 import { BasePresenter } from "./basePresenter.server";
+import { isCancellableRunStatus } from "~/v3/taskStatus";
 
 export type RunListOptions = {
   userId?: string;
@@ -156,6 +156,7 @@ export class RunListPresenter extends BasePresenter {
         runtimeEnvironmentId: string;
         status: TaskRunStatus;
         createdAt: Date;
+        startedAt: Date | null;
         lockedAt: Date | null;
         updatedAt: Date;
         isTest: boolean;
@@ -172,6 +173,7 @@ export class RunListPresenter extends BasePresenter {
     tr."runtimeEnvironmentId" AS "runtimeEnvironmentId",
     tr.status AS status,
     tr."createdAt" AS "createdAt",
+    tr."startedAt" AS "startedAt",
     tr."lockedAt" AS "lockedAt",
     tr."updatedAt" AS "updatedAt",
     tr."isTest" AS "isTest",
@@ -272,13 +274,15 @@ export class RunListPresenter extends BasePresenter {
 
         const hasFinished = FINISHED_STATUSES.includes(run.status);
 
+        const startedAt = run.startedAt ?? run.lockedAt;
+
         return {
           id: run.id,
           friendlyId: run.runFriendlyId,
           number: Number(run.number),
           createdAt: run.createdAt.toISOString(),
           updatedAt: run.updatedAt.toISOString(),
-          startedAt: run.lockedAt ? run.lockedAt.toISOString() : undefined,
+          startedAt: startedAt ? startedAt.toISOString() : undefined,
           hasFinished,
           finishedAt: hasFinished ? run.updatedAt.toISOString() : undefined,
           isTest: run.isTest,
@@ -287,7 +291,7 @@ export class RunListPresenter extends BasePresenter {
           taskIdentifier: run.taskIdentifier,
           spanId: run.spanId,
           isReplayable: true,
-          isCancellable: CANCELLABLE_STATUSES.includes(run.status),
+          isCancellable: isCancellableRunStatus(run.status),
           environment: displayableEnvironment(environment, userId),
           idempotencyKey: run.idempotencyKey ? run.idempotencyKey : undefined,
         };

apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts

@@ -94,17 +94,24 @@ export class RunStreamPresenter {
 
         eventEmitter.removeAllListeners();
 
-        unsubscribe().catch((error) => {
-          logger.error("RunStreamPresenter.abort.unsubscribe", {
-            runFriendlyId,
-            traceId: run.traceId,
-            error: {
-              name: error.name,
-              message: error.message,
-              stack: error.stack,
-            },
+        unsubscribe()
+          .then(() => {
+            logger.info("RunStreamPresenter.abort.unsubscribe succeeded", {
+              runFriendlyId,
+              traceId: run.traceId,
+            });
+          })
+          .catch((error) => {
+            logger.error("RunStreamPresenter.abort.unsubscribe failed", {
+              runFriendlyId,
+              traceId: run.traceId,
+              error: {
+                name: error.name,
+                message: error.message,
+                stack: error.stack,
+              },
+            });
           });
-        });
       };
     });
   }

apps/webapp/app/presenters/v3/ScheduleListPresenter.server.ts

@@ -22,6 +22,7 @@ export type ScheduleListItem = {
   userProvidedDeduplicationKey: boolean;
   cron: string;
   cronDescription: string;
+  timezone: string;
   externalId: string | null;
   nextRun: Date;
   lastRun: Date | undefined;
@@ -36,7 +37,6 @@ export type ScheduleList = Awaited<ReturnType<ScheduleListPresenter["call"]>>;
 export type ScheduleListAppliedFilters = ScheduleList["filters"];
 
 export class ScheduleListPresenter extends BasePresenter {
-
   public async call({
     userId,
     projectId,
@@ -71,12 +71,23 @@ export class ScheduleListPresenter extends BasePresenter {
             },
           },
         },
+        organization: {
+          select: {
+            maximumSchedulesLimit: true,
+          },
+        },
       },
       where: {
         id: projectId,
       },
     });
 
+    const schedulesCount = await this._prisma.taskSchedule.count({
+      where: {
+        projectId,
+      },
+    });
+
     //get all possible scheduled tasks
     const possibleTasks = await this._replica.backgroundWorkerTask.findMany({
       distinct: ["slug"],
@@ -140,6 +151,7 @@ export class ScheduleListPresenter extends BasePresenter {
         userProvidedDeduplicationKey: true,
         generatorExpression: true,
         generatorDescription: true,
+        timezone: true,
         externalId: true,
         instances: {
           select: {
@@ -218,10 +230,11 @@ export class ScheduleListPresenter extends BasePresenter {
         userProvidedDeduplicationKey: schedule.userProvidedDeduplicationKey,
         cron: schedule.generatorExpression,
         cronDescription: schedule.generatorDescription,
+        timezone: schedule.timezone,
         active: schedule.active,
         externalId: schedule.externalId,
         lastRun: latestRun?.createdAt,
-        nextRun: calculateNextScheduledTimestamp(schedule.generatorExpression),
+        nextRun: calculateNextScheduledTimestamp(schedule.generatorExpression, schedule.timezone),
         environments: schedule.instances.map((instance) => {
           const environment = project.environments.find((env) => env.id === instance.environmentId);
           if (!environment) {
@@ -245,6 +258,10 @@ export class ScheduleListPresenter extends BasePresenter {
         return displayableEnvironment(environment, userId);
       }),
       hasFilters,
+      limits: {
+        used: schedulesCount,
+        limit: project.organization.maximumSchedulesLimit,
+      },
       filters: {
         tasks,
         environments,

apps/webapp/app/presenters/v3/TaskListPresenter.server.ts

@@ -311,7 +311,7 @@ export class TaskListPresenter extends BasePresenter {
     >`    
     SELECT 
       tr."taskIdentifier", 
-      AVG(EXTRACT(EPOCH FROM (tr."updatedAt" - tr."lockedAt"))) as duration
+      AVG(EXTRACT(EPOCH FROM (tr."updatedAt" - COALESCE(tr."startedAt", tr."lockedAt")))) as duration
       FROM 
       ${sqlDatabaseSchema}."TaskRun" as tr
     WHERE 

apps/webapp/app/presenters/v3/TestTaskPresenter.server.ts

@@ -6,6 +6,7 @@ import {
   TaskTriggerSource,
 } from "@trigger.dev/database";
 import { sqlDatabaseSchema, PrismaClient, prisma } from "~/db.server";
+import { getTimezones } from "~/utils/timezones.server";
 import { getUsername } from "~/utils/username";
 
 type TestTaskOptions = {
@@ -37,6 +38,7 @@ export type TestTask =
   | {
       triggerSource: "SCHEDULED";
       task: Task;
+      possibleTimezones: string[];
       runs: ScheduledRun[];
     };
 
@@ -61,6 +63,7 @@ export type ScheduledRun = Omit<RawRun, "number" | "payload"> & {
     timestamp: Date;
     lastTimestamp?: Date;
     externalId?: string;
+    timezone: string;
   };
 };
 
@@ -168,9 +171,11 @@ export class TestTaskPresenter {
           ),
         };
       case "SCHEDULED":
+        const possibleTimezones = getTimezones();
         return {
           triggerSource: "SCHEDULED",
           task: taskWithEnvironment,
+          possibleTimezones,
           runs: (
             await Promise.all(
               latestRuns.map(async (r) => {
@@ -195,6 +200,9 @@ export class TestTaskPresenter {
 
 async function getScheduleTaskRunPayload(run: RawRun) {
   const payload = await parsePacket({ data: run.payload, dataType: run.payloadType });
+  if (!payload.timezone) {
+    payload.timezone = "UTC";
+  }
   const parsed = ScheduledTaskPayload.safeParse(payload);
   return parsed;
 }

apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts

@@ -1,8 +1,8 @@
+import { ScheduleObject } from "@trigger.dev/core/v3";
 import { PrismaClient, prisma } from "~/db.server";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 import { nextScheduledTimestamps } from "~/v3/utils/calculateNextSchedule.server";
 import { RunListPresenter } from "./RunListPresenter.server";
-import { ScheduleObject } from "@trigger.dev/core/v3";
-import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 
 type ViewScheduleOptions = {
   userId?: string;
@@ -24,6 +24,7 @@ export class ViewSchedulePresenter {
         friendlyId: true,
         generatorExpression: true,
         generatorDescription: true,
+        timezone: true,
         externalId: true,
         deduplicationKey: true,
         userProvidedDeduplicationKey: true,
@@ -68,7 +69,7 @@ export class ViewSchedulePresenter {
     }
 
     const nextRuns = schedule.active
-      ? nextScheduledTimestamps(schedule.generatorExpression, new Date(), 5)
+      ? nextScheduledTimestamps(schedule.generatorExpression, schedule.timezone, new Date(), 5)
       : [];
 
     const runPresenter = new RunListPresenter(this.#prismaClient);
@@ -82,6 +83,7 @@ export class ViewSchedulePresenter {
     return {
       schedule: {
         ...schedule,
+        timezone: schedule.timezone,
         cron: schedule.generatorExpression,
         cronDescription: schedule.generatorDescription,
         nextRuns,
@@ -105,6 +107,7 @@ export class ViewSchedulePresenter {
         expression: result.schedule.cron,
         description: result.schedule.cronDescription,
       },
+      timezone: result.schedule.timezone,
       externalId: result.schedule.externalId ?? undefined,
       deduplicationKey: result.schedule.userProvidedDeduplicationKey
         ? result.schedule.deduplicationKey ?? undefined

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers.scheduled/route.tsx

@@ -2,6 +2,8 @@ import { NoSymbolIcon } from "@heroicons/react/20/solid";
 import { CheckCircleIcon, XCircleIcon } from "@heroicons/react/24/solid";
 import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { z } from "zod";
+import { ListPagination } from "~/components/ListPagination";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { DateTime } from "~/components/primitives/DateTime";
 import { LabelValueStack } from "~/components/primitives/LabelValueStack";
@@ -17,30 +19,38 @@ import {
   TableRow,
 } from "~/components/primitives/Table";
 import { TextLink } from "~/components/primitives/TextLink";
-import { useOrganization } from "~/hooks/useOrganizations";
-import { useProject } from "~/hooks/useProject";
+import { DirectionSchema } from "~/components/runs/RunStatuses";
 import { ScheduledTriggersPresenter } from "~/presenters/ScheduledTriggersPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { ProjectParamSchema, docsPath } from "~/utils/pathBuilder";
 
+const SearchSchema = z.object({
+  cursor: z.string().optional(),
+  direction: DirectionSchema.optional(),
+});
+
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
   const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
 
+  const url = new URL(request.url);
+  const s = Object.fromEntries(url.searchParams.entries());
+  const searchParams = SearchSchema.parse(s);
+
   const presenter = new ScheduledTriggersPresenter();
   const data = await presenter.call({
     userId,
     organizationSlug,
     projectSlug: projectParam,
+    direction: searchParams.direction,
+    cursor: searchParams.cursor,
   });
 
   return typedjson(data);
 };
 
-export default function Integrations() {
-  const { scheduled } = useTypedLoaderData<typeof loader>();
-  const organization = useOrganization();
-  const project = useProject();
+export default function Route() {
+  const { scheduled, pagination } = useTypedLoaderData<typeof loader>();
 
   return (
     <>
@@ -49,6 +59,10 @@ export default function Integrations() {
         expression or an interval.
       </Paragraph>
 
+      {scheduled.length > 0 && (
+        <ListPagination list={{ pagination }} className="mt-2 justify-end" />
+      )}
+
       <Table containerClassName="mt-4">
         <TableHeader>
           <TableRow>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx

@@ -1,12 +1,13 @@
 import {
+  BoltSlashIcon,
   ChevronDownIcon,
   ChevronRightIcon,
   MagnifyingGlassMinusIcon,
   MagnifyingGlassPlusIcon,
 } from "@heroicons/react/20/solid";
 import type { Location } from "@remix-run/react";
-import { useParams, useRevalidator } from "@remix-run/react";
-import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { useLoaderData, useParams, useRevalidator } from "@remix-run/react";
+import { LoaderFunctionArgs, SerializeFrom } from "@remix-run/server-runtime";
 import { Virtualizer } from "@tanstack/react-virtual";
 import {
   formatDurationMilliseconds,
@@ -17,10 +18,10 @@ import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
 import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
@@ -32,6 +33,7 @@ import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -54,7 +56,7 @@ import { useProject } from "~/hooks/useProject";
 import { useReplaceLocation } from "~/hooks/useReplaceLocation";
 import { Shortcut, useShortcutKeys } from "~/hooks/useShortcutKeys";
 import { useUser } from "~/hooks/useUser";
-import { RunEvent, RunPresenter } from "~/presenters/v3/RunPresenter.server";
+import { RunPresenter } from "~/presenters/v3/RunPresenter.server";
 import { getResizableRunSettings, setResizableRunSettings } from "~/services/resizablePanel";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
@@ -67,8 +69,10 @@ import {
   v3RunsPath,
 } from "~/utils/pathBuilder";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route";
-import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
+import { env } from "~/env.server";
+
+type TraceEvent = NonNullable<SerializeFrom<typeof loader>["trace"]>["events"][0];
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -85,10 +89,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   //resizable settings
   const resizeSettings = await getResizableRunSettings(request);
 
-  return typedjson({
-    ...result,
+  return {
+    run: result.run,
+    trace: result.trace,
+    maximumLiveReloadingSetting: env.MAXIMUM_LIVE_RELOADING_EVENTS,
     resizeSettings,
-  });
+  };
 };
 
 function getSpanId(location: Location<any>): string | undefined {
@@ -97,7 +103,8 @@ function getSpanId(location: Location<any>): string | undefined {
 }
 
 export default function Page() {
-  const { run, trace, resizeSettings } = useTypedLoaderData<typeof loader>();
+  const { run, trace, resizeSettings, maximumLiveReloadingSetting } =
+    useLoaderData<typeof loader>();
   const organization = useOrganization();
   const project = useProject();
   const user = useUser();
@@ -167,6 +174,7 @@ export default function Page() {
   }
 
   const { events, parentRunFriendlyId, duration, rootSpanStatus, rootStartedAt } = trace;
+  const shouldLiveReload = events.length <= maximumLiveReloadingSetting;
 
   const changeToSpan = useDebounce((selectedSpan: string) => {
     replaceSearchParam("span", selectedSpan);
@@ -175,6 +183,7 @@ export default function Page() {
   const revalidator = useRevalidator();
   const streamedEvents = useEventSource(v3RunStreamingPath(organization, project, run), {
     event: "message",
+    disabled: !shouldLiveReload,
   });
   useEffect(() => {
     if (streamedEvents !== null) {
@@ -252,8 +261,10 @@ export default function Page() {
                 }}
                 totalDuration={duration}
                 rootSpanStatus={rootSpanStatus}
-                rootStartedAt={rootStartedAt}
+                rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined}
                 environmentType={run.environment.type}
+                shouldLiveReload={shouldLiveReload}
+                maximumLiveReloadingSetting={maximumLiveReloadingSetting}
               />
             </ResizablePanel>
             <ResizableHandle withHandle />
@@ -274,7 +285,7 @@ export default function Page() {
 }
 
 type TasksTreeViewProps = {
-  events: RunEvent[];
+  events: TraceEvent[];
   selectedId?: string;
   parentRunFriendlyId?: string;
   onSelectedIdChanged: (selectedId: string | undefined) => void;
@@ -282,6 +293,8 @@ type TasksTreeViewProps = {
   rootSpanStatus: "executing" | "completed" | "failed";
   rootStartedAt: Date | undefined;
   environmentType: RuntimeEnvironmentType;
+  shouldLiveReload: boolean;
+  maximumLiveReloadingSetting: number;
 };
 
 function TasksTreeView({
@@ -293,6 +306,8 @@ function TasksTreeView({
   rootSpanStatus,
   rootStartedAt,
   environmentType,
+  shouldLiveReload,
+  maximumLiveReloadingSetting,
 }: TasksTreeViewProps) {
   const [filterText, setFilterText] = useState("");
   const [errorsOnly, setErrorsOnly] = useState(false);
@@ -367,7 +382,11 @@ function TasksTreeView({
                   This is the root task
                 </Paragraph>
               )}
-              <LiveReloadingStatus rootSpanCompleted={rootSpanStatus !== "executing"} />
+              <LiveReloadingStatus
+                rootSpanCompleted={rootSpanStatus !== "executing"}
+                isLiveReloading={shouldLiveReload}
+                settingValue={maximumLiveReloadingSetting}
+              />
             </div>
             <TreeView
               parentRef={parentRef}
@@ -750,7 +769,7 @@ function TimelineView({
   );
 }
 
-function NodeText({ node }: { node: RunEvent }) {
+function NodeText({ node }: { node: TraceEvent }) {
   const className = "truncate";
   return (
     <Paragraph variant="small" className={cn(className)}>
@@ -759,7 +778,7 @@ function NodeText({ node }: { node: RunEvent }) {
   );
 }
 
-function NodeStatusIcon({ node }: { node: RunEvent }) {
+function NodeStatusIcon({ node }: { node: TraceEvent }) {
   if (node.data.level !== "TRACE") return null;
   if (node.data.style.variant !== "primary") return null;
 
@@ -834,16 +853,40 @@ function ShowParentLink({ runFriendlyId }: { runFriendlyId: string }) {
   );
 }
 
-function LiveReloadingStatus({ rootSpanCompleted }: { rootSpanCompleted: boolean }) {
+function LiveReloadingStatus({
+  rootSpanCompleted,
+  isLiveReloading,
+  settingValue,
+}: {
+  rootSpanCompleted: boolean;
+  isLiveReloading: boolean;
+  settingValue: number;
+}) {
   if (rootSpanCompleted) return null;
 
   return (
-    <div className="flex items-center gap-1">
-      <PulsingDot />
-      <Paragraph variant="extra-small" className="whitespace-nowrap text-blue-500">
-        Live reloading
-      </Paragraph>
-    </div>
+    <>
+      {isLiveReloading ? (
+        <div className="flex items-center gap-1">
+          <PulsingDot />
+          <Paragraph variant="extra-small" className="whitespace-nowrap text-blue-500">
+            Live reloading
+          </Paragraph>
+        </div>
+      ) : (
+        <SimpleTooltip
+          content={`Live reloading is disabled because you've exceeded ${settingValue} logs.`}
+          button={
+            <div className="flex items-center gap-1">
+              <BoltSlashIcon className="size-3.5 text-text-dimmed" />
+              <Paragraph variant="extra-small" className="whitespace-nowrap text-text-dimmed">
+                Live reloading disabled
+              </Paragraph>
+            </div>
+          }
+        ></SimpleTooltip>
+      )}
+    </>
   );
 }
 
@@ -862,7 +905,7 @@ function SpanWithDuration({
   showDuration,
   node,
   ...props
-}: Timeline.SpanProps & { node: RunEvent; showDuration: boolean }) {
+}: Timeline.SpanProps & { node: TraceEvent; showDuration: boolean }) {
   return (
     <Timeline.Span {...props}>
       <motion.div

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules.$scheduleParam/route.tsx

@@ -185,7 +185,8 @@ export default function Page() {
   const location = useLocation();
   const organization = useOrganization();
   const project = useProject();
-  const user = useUser();
+
+  const isUtc = schedule.timezone === "UTC";
 
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_1fr_3.25rem] overflow-hidden bg-background-bright">
@@ -210,6 +211,7 @@ export default function Page() {
                   <Paragraph variant="small">{schedule.cronDescription}</Paragraph>
                 </div>
               </Property>
+              <Property label="Timezone">{schedule.timezone}</Property>
               <Property label="Environments">
                 <EnvironmentLabels size="small" environments={schedule.environments} />
               </Property>
@@ -245,19 +247,21 @@ export default function Page() {
               <Table>
                 <TableHeader>
                   <TableRow>
+                    {!isUtc && <TableHeaderCell>{schedule.timezone}</TableHeaderCell>}
                     <TableHeaderCell>UTC</TableHeaderCell>
-                    <TableHeaderCell>Local time</TableHeaderCell>
                   </TableRow>
                 </TableHeader>
                 <TableBody>
                   {schedule.nextRuns.map((run, index) => (
                     <TableRow key={index}>
+                      {!isUtc && (
+                        <TableCell>
+                          <DateTime date={run} timeZone={schedule.timezone} />
+                        </TableCell>
+                      )}
                       <TableCell>
                         <DateTime date={run} timeZone="UTC" />
                       </TableCell>
-                      <TableCell>
-                        <DateTime date={run} />
-                      </TableCell>
                     </TableRow>
                   ))}
                 </TableBody>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules.edit.$scheduleParam/route.tsx

@@ -21,7 +21,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 };
 
 export default function Page() {
-  const { schedule, possibleTasks, possibleEnvironments, showGenerateField } =
+  const { schedule, possibleTasks, possibleEnvironments, possibleTimezones, showGenerateField } =
     useTypedLoaderData<typeof loader>();
 
   return (
@@ -29,6 +29,7 @@ export default function Page() {
       schedule={schedule}
       possibleTasks={possibleTasks}
       possibleEnvironments={possibleEnvironments}
+      possibleTimezones={possibleTimezones}
       showGenerateField={showGenerateField}
     />
   );

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules.new/route.tsx

@@ -20,7 +20,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 };
 
 export default function Page() {
-  const { schedule, possibleTasks, possibleEnvironments, showGenerateField } =
+  const { schedule, possibleTasks, possibleEnvironments, possibleTimezones, showGenerateField } =
     useTypedLoaderData<typeof loader>();
 
   return (
@@ -29,6 +29,7 @@ export default function Page() {
       possibleTasks={possibleTasks}
       possibleEnvironments={possibleEnvironments}
       showGenerateField={showGenerateField}
+      possibleTimezones={possibleTimezones}
     />
   );
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules/route.tsx

@@ -4,12 +4,21 @@ import { Outlet, useLocation, useParams } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
+import { Feedback } from "~/components/Feedback";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
-import { EnvironmentLabel, EnvironmentLabels } from "~/components/environments/EnvironmentLabel";
+import { EnvironmentLabels } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
-import { LinkButton } from "~/components/primitives/Buttons";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { DateTime } from "~/components/primitives/DateTime";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTrigger,
+} from "~/components/primitives/Dialog";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -78,6 +87,7 @@ export default function Page() {
     possibleEnvironments,
     hasFilters,
     filters,
+    limits,
     currentPage,
     totalPages,
   } = useTypedLoaderData<typeof loader>();
@@ -107,15 +117,43 @@ export default function Page() {
             </PropertyTable>
           </AdminDebugTooltip>
 
-          <LinkButton
-            LeadingIcon={PlusIcon}
-            to={`${v3NewSchedulePath(organization, project)}${location.search}`}
-            variant="primary/small"
-            shortcut={{ key: "n" }}
-            disabled={possibleTasks.length === 0 || isShowingNewPane}
-          >
-            New schedule
-          </LinkButton>
+          {limits.used >= limits.limit ? (
+            <Dialog>
+              <DialogTrigger asChild>
+                <Button
+                  LeadingIcon={PlusIcon}
+                  variant="primary/small"
+                  shortcut={{ key: "n" }}
+                  disabled={possibleTasks.length === 0 || isShowingNewPane}
+                >
+                  New schedule
+                </Button>
+              </DialogTrigger>
+              <DialogContent>
+                <DialogHeader>You've exceeded your limit</DialogHeader>
+                <DialogDescription>
+                  You've used {limits.used}/{limits.limit} of your schedules. You can request more
+                  schedules.
+                </DialogDescription>
+                <DialogFooter>
+                  <Feedback
+                    button={<Button variant="primary/medium">Request more</Button>}
+                    defaultValue="help"
+                  />
+                </DialogFooter>
+              </DialogContent>
+            </Dialog>
+          ) : (
+            <LinkButton
+              LeadingIcon={PlusIcon}
+              to={`${v3NewSchedulePath(organization, project)}${location.search}`}
+              variant="primary/small"
+              shortcut={{ key: "n" }}
+              disabled={possibleTasks.length === 0 || isShowingNewPane}
+            >
+              New schedule
+            </LinkButton>
+          )}
         </PageAccessories>
       </NavBar>
       <PageBody scrollable={false}>
@@ -142,7 +180,21 @@ export default function Page() {
                 </div>
 
                 <SchedulesTable schedules={schedules} hasFilters={hasFilters} />
-                <div className="mt-2 justify-end">
+                <div className="mt-2 justify-between">
+                  <Paragraph variant="extra-small" className="mt-3">
+                    <span className={limits.used >= limits.limit ? "text-warning" : ""}>
+                      You've used {limits.used}/{limits.limit} of your schedules.
+                    </span>{" "}
+                    <Feedback
+                      button={
+                        <button className=" text-secondary transition hover:text-indigo-400">
+                          Request more
+                        </button>
+                      }
+                      defaultValue="help"
+                    />
+                    .
+                  </Paragraph>
                   <PaginationControls currentPage={currentPage} totalPages={totalPages} />
                 </div>
               </div>
@@ -236,12 +288,13 @@ function SchedulesTable({
         <TableRow>
           <TableHeaderCell>ID</TableHeaderCell>
           <TableHeaderCell>Task ID</TableHeaderCell>
+          <TableHeaderCell>External ID</TableHeaderCell>
           <TableHeaderCell>CRON</TableHeaderCell>
           <TableHeaderCell hiddenLabel>CRON description</TableHeaderCell>
-          <TableHeaderCell>External ID</TableHeaderCell>
+          <TableHeaderCell>Timezone</TableHeaderCell>
+          <TableHeaderCell>Next run</TableHeaderCell>
+          <TableHeaderCell>Last run</TableHeaderCell>
           <TableHeaderCell>Deduplication key</TableHeaderCell>
-          <TableHeaderCell>Next run (UTC)</TableHeaderCell>
-          <TableHeaderCell>Last run (UTC)</TableHeaderCell>
           <TableHeaderCell>Environments</TableHeaderCell>
           <TableHeaderCell>Enabled</TableHeaderCell>
         </TableRow>
@@ -262,6 +315,9 @@ function SchedulesTable({
                 <TableCell to={path} className={cellClass}>
                   {schedule.taskIdentifier}
                 </TableCell>
+                <TableCell to={path} className={cellClass}>
+                  {schedule.externalId ? schedule.externalId : "–"}
+                </TableCell>
                 <TableCell to={path} className={cellClass}>
                   {schedule.cron}
                 </TableCell>
@@ -269,17 +325,21 @@ function SchedulesTable({
                   {schedule.cronDescription}
                 </TableCell>
                 <TableCell to={path} className={cellClass}>
-                  {schedule.externalId ? schedule.externalId : "–"}
+                  {schedule.timezone}
+                </TableCell>
+                <TableCell to={path} className={cellClass}>
+                  <DateTime date={schedule.nextRun} timeZone={schedule.timezone} />
+                </TableCell>
+                <TableCell to={path} className={cellClass}>
+                  {schedule.lastRun ? (
+                    <DateTime date={schedule.lastRun} timeZone={schedule.timezone} />
+                  ) : (
+                    "–"
+                  )}
                 </TableCell>
                 <TableCell to={path} className={cellClass}>
                   {schedule.userProvidedDeduplicationKey ? schedule.deduplicationKey : "–"}
                 </TableCell>
-                <TableCell to={path} className={cellClass}>
-                  <DateTime date={schedule.nextRun} timeZone="utc" />
-                </TableCell>
-                <TableCell to={path} className={cellClass}>
-                  {schedule.lastRun ? <DateTime date={schedule.lastRun} timeZone="utc" /> : "–"}
-                </TableCell>
                 <TableCell to={path} className={cellClass}>
                   <EnvironmentLabels environments={schedule.environments} size="small" />
                 </TableCell>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.test.tasks.$taskParam/route.tsx

@@ -26,8 +26,10 @@ import {
   ResizablePanel,
   ResizablePanelGroup,
 } from "~/components/primitives/Resizable";
+import { Select } from "~/components/primitives/Select";
 import { TextLink } from "~/components/primitives/TextLink";
 import { TaskRunStatusCombo } from "~/components/runs/v3/TaskRunStatus";
+import { TimezoneList } from "~/components/scheduled/timezones";
 import { redirectBackWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
 import {
   ScheduledRun,
@@ -95,7 +97,13 @@ export default function Page() {
       return <StandardTaskForm task={result.task} runs={result.runs} />;
     }
     case "SCHEDULED": {
-      return <ScheduledTaskForm task={result.task} runs={result.runs} />;
+      return (
+        <ScheduledTaskForm
+          task={result.task}
+          runs={result.runs}
+          possibleTimezones={result.possibleTimezones}
+        />
+      );
     }
   }
 }
@@ -215,12 +223,21 @@ function StandardTaskForm({ task, runs }: { task: TestTask["task"]; runs: Standa
   );
 }
 
-function ScheduledTaskForm({ task, runs }: { task: TestTask["task"]; runs: ScheduledRun[] }) {
+function ScheduledTaskForm({
+  task,
+  runs,
+  possibleTimezones,
+}: {
+  task: TestTask["task"];
+  runs: ScheduledRun[];
+  possibleTimezones: string[];
+}) {
   const lastSubmission = useActionData();
   const [selectedCodeSampleId, setSelectedCodeSampleId] = useState(runs.at(0)?.id);
   const [timestampValue, setTimestampValue] = useState<Date | undefined>();
   const [lastTimestampValue, setLastTimestampValue] = useState<Date | undefined>();
   const [externalIdValue, setExternalIdValue] = useState<string | undefined>();
+  const [timezoneValue, setTimezoneValue] = useState<string>("UTC");
 
   //set initial values
   useEffect(() => {
@@ -233,11 +250,20 @@ function ScheduledTaskForm({ task, runs }: { task: TestTask["task"]; runs: Sched
     setTimestampValue(initialRun.payload.timestamp);
     setLastTimestampValue(initialRun.payload.lastTimestamp);
     setExternalIdValue(initialRun.payload.externalId);
+    setTimezoneValue(initialRun.payload.timezone);
   }, [selectedCodeSampleId]);
 
   const [
     form,
-    { timestamp, lastTimestamp, externalId, triggerSource, taskIdentifier, environmentId },
+    {
+      timestamp,
+      lastTimestamp,
+      externalId,
+      triggerSource,
+      taskIdentifier,
+      environmentId,
+      timezone,
+    },
   ] = useForm({
     id: "test-task-scheduled",
     // TODO: type this
@@ -314,6 +340,30 @@ function ScheduledTaskForm({ task, runs }: { task: TestTask["task"]; runs: Sched
                 </Hint>
                 <FormError id={lastTimestamp.errorId}>{lastTimestamp.error}</FormError>
               </InputGroup>
+              <InputGroup>
+                <Label htmlFor={timezone.id}>Timezone</Label>
+                <Select
+                  {...conform.select(timezone)}
+                  placeholder="Select a timezone"
+                  defaultValue={timezoneValue}
+                  value={timezoneValue}
+                  setValue={(e) => {
+                    if (Array.isArray(e)) return;
+                    setTimezoneValue(e);
+                  }}
+                  items={possibleTimezones}
+                  filter={{ keys: [(item) => item.replace(/\//g, " ").replace(/_/g, " ")] }}
+                  dropdownIcon
+                  variant="tertiary/medium"
+                >
+                  {(matches) => <TimezoneList timezones={matches} />}
+                </Select>
+                <Hint>
+                  The Timestamp and Last timestamp are in UTC so this just changes the timezone
+                  string that comes through in the payload.
+                </Hint>
+                <FormError id={timezone.errorId}>{timezone.error}</FormError>
+              </InputGroup>
               <InputGroup>
                 <Label required={false} htmlFor={externalId.id}>
                   External ID

apps/webapp/app/routes/_app.timezones/route.tsx

@@ -0,0 +1,35 @@
+import { Link } from "@remix-run/react";
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { LogoIcon } from "~/components/LogoIcon";
+import { Header1 } from "~/components/primitives/Headers";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { getTimezones } from "~/utils/timezones.server";
+
+export const loader = async ({ request }: LoaderFunctionArgs) => {
+  return typedjson({
+    timezones: getTimezones(),
+  });
+};
+
+export default function Page() {
+  const { timezones } = useTypedLoaderData<typeof loader>();
+  return (
+    <div className="grid grid-rows-[2.5rem,1fr]">
+      <div className="flex items-center border-b border-b-grid-dimmed px-3">
+        <Link to="/">
+          <LogoIcon className="relative -top-px mr-2 h-4 w-4 min-w-[1rem]" />
+        </Link>
+      </div>
+      <div className="overflow-y-auto p-8 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
+        <Header1 spacing>Supported timezones</Header1>
+        <Paragraph spacing>We support these timezones when creating a schedule.</Paragraph>
+        <ul className="">
+          {timezones.map((timezone) => (
+            <li key={timezone}>{timezone}</li>
+          ))}
+        </ul>
+      </div>
+    </div>
+  );
+}

apps/webapp/app/routes/api.v1.deployments.ts

@@ -1,5 +1,8 @@
 import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
-import { InitializeDeploymentRequestBody, InitializeDeploymentResponseBody } from "@trigger.dev/core/v3";
+import {
+  InitializeDeploymentRequestBody,
+  InitializeDeploymentResponseBody,
+} from "@trigger.dev/core/v3";
 import { env } from "~/env.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
@@ -37,13 +40,11 @@ export async function action({ request, params }: ActionFunctionArgs) {
     contentHash: deployment.contentHash,
     shortCode: deployment.shortCode,
     version: deployment.version,
-    externalBuildData: deployment.externalBuildData as InitializeDeploymentResponseBody["externalBuildData"],
+    externalBuildData:
+      deployment.externalBuildData as InitializeDeploymentResponseBody["externalBuildData"],
     imageTag,
-    registryHost: env.DEPLOY_REGISTRY_HOST
-  }
+    registryHost: env.DEPLOY_REGISTRY_HOST,
+  };
 
-  return json(
-    responseBody,
-    { status: 200 }
-  );
+  return json(responseBody, { status: 200 });
 }

apps/webapp/app/routes/api.v1.packets.$.ts

@@ -1,10 +1,8 @@
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { z } from "zod";
-import { env } from "~/env.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { logger } from "~/services/logger.server";
-import { r2 } from "~/v3/r2.server";
+import { generatePresignedUrl } from "~/v3/r2.server";
 
 const ParamsSchema = z.object({
   "*": z.string(),
@@ -26,34 +24,19 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const parsedParams = ParamsSchema.parse(params);
   const filename = parsedParams["*"];
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    return json({ error: "Object store base URL is not set" }, { status: 500 });
-  }
-
-  if (!r2) {
-    return json({ error: "Object store credentials are not set" }, { status: 500 });
-  }
-
-  const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${authenticationResult.environment.project.externalRef}/${authenticationResult.environment.slug}/${filename}`;
-  url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
-
-  const signed = await r2.sign(
-    new Request(url, {
-      method: "PUT",
-    }),
-    {
-      aws: { signQuery: true },
-    }
+  const presignedUrl = await generatePresignedUrl(
+    authenticationResult.environment.project.externalRef,
+    authenticationResult.environment.slug,
+    filename,
+    "PUT"
   );
 
-  logger.debug("Generated presigned URL", {
-    url: signed.url,
-    headers: Object.fromEntries(signed.headers),
-  });
+  if (!presignedUrl) {
+    return json({ error: "Failed to generate presigned URL" }, { status: 500 });
+  }
 
   // Caller can now use this URL to upload to that object.
-  return json({ presignedUrl: signed.url });
+  return json({ presignedUrl });
 }
 
 export async function loader({ request, params }: ActionFunctionArgs) {
@@ -67,35 +50,17 @@ export async function loader({ request, params }: ActionFunctionArgs) {
   const parsedParams = ParamsSchema.parse(params);
   const filename = parsedParams["*"];
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    return json({ error: "Object store base URL is not set" }, { status: 500 });
-  }
-
-  if (!r2) {
-    return json({ error: "Object store credentials are not set" }, { status: 500 });
-  }
-
-  const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${authenticationResult.environment.project.externalRef}/${authenticationResult.environment.slug}/${filename}`;
-  url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
-
-  const signed = await r2.sign(
-    new Request(url, {
-      method: request.method,
-    }),
-    {
-      aws: { signQuery: true },
-    }
+  const presignedUrl = await generatePresignedUrl(
+    authenticationResult.environment.project.externalRef,
+    authenticationResult.environment.slug,
+    filename,
+    "GET"
   );
 
-  logger.debug("Generated presigned URL", {
-    url: signed.url,
-    headers: Object.fromEntries(signed.headers),
-  });
+  if (!presignedUrl) {
+    return json({ error: "Failed to generate presigned URL" }, { status: 500 });
+  }
 
-  const getUrl = new URL(url.href);
-  getUrl.searchParams.delete("X-Amz-Expires");
-
-  // Caller can now use this URL to upload to that object.
-  return json({ presignedUrl: signed.url });
+  // Caller can now use this URL to fetch that object.
+  return json({ presignedUrl });
 }

apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts

@@ -123,7 +123,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
 
   const repository = new EnvironmentVariablesRepository();
 
-  const variables = await repository.getEnvironment(environment.project.id, environment.id, true);
+  const variables = await repository.getEnvironment(environment.project.id, environment.id);
 
   const environmentVariable = variables.find((v) => v.key === parsedParams.data.name);
 

apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts

@@ -80,7 +80,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
 
   const repository = new EnvironmentVariablesRepository();
 
-  const variables = await repository.getEnvironment(environment.project.id, environment.id, true);
+  const variables = await repository.getEnvironment(environment.project.id, environment.id);
 
   return json(variables.map((variable) => ({ name: variable.key, value: variable.value })));
 }

apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.ts

@@ -2,7 +2,7 @@ import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
+import { resolveVariablesForEnvironment } from "~/v3/environmentVariables/environmentVariablesRepository.server";
 
 const ParamsSchema = z.object({
   projectRef: z.string(),
@@ -41,9 +41,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     return json({ error: "Project not found" }, { status: 404 });
   }
 
-  const repository = new EnvironmentVariablesRepository();
-
-  const variables = await repository.getEnvironmentVariables(project.id, authenticatedEnv.id);
+  const variables = await resolveVariablesForEnvironment(authenticatedEnv);
 
   return json({
     variables: variables.reduce((acc: Record<string, string>, variable) => {

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete/CompleteRunTaskService.server.ts

@@ -3,6 +3,7 @@ import { PrismaClientOrTransaction, prisma } from "~/db.server";
 import { taskWithAttemptsToServerTask } from "~/models/task.server";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { startActiveSpan } from "~/v3/tracer.server";
 
 export class CompleteRunTaskService {
   #prismaClient: PrismaClientOrTransaction;
@@ -17,76 +18,81 @@ export class CompleteRunTaskService {
     id: string,
     taskBody: CompleteTaskBodyOutput
   ): Promise<ServerTask | undefined> {
-    const existingTask = await this.#prismaClient.task.findUnique({
-      where: {
-        id,
-      },
-      include: {
-        run: true,
-        attempts: {
-          where: {
-            status: "PENDING",
-          },
-          orderBy: {
-            number: "desc",
-          },
-          take: 1,
+    return startActiveSpan("CompleteRunTaskService.call", async (span) => {
+      span.setAttribute("runId", runId);
+      span.setAttribute("taskId", id);
+
+      const existingTask = await this.#prismaClient.task.findUnique({
+        where: {
+          id,
+        },
+        include: {
+          run: true,
+          attempts: {
+            where: {
+              status: "PENDING",
+            },
+            orderBy: {
+              number: "desc",
+            },
+            take: 1,
+          },
         },
-      },
-    });
-
-    if (!existingTask) {
-      return;
-    }
-
-    if (existingTask.runId !== runId) {
-      return;
-    }
-
-    if (existingTask.run.environmentId !== environment.id) {
-      return;
-    }
-
-    if (
-      existingTask.status === "COMPLETED" ||
-      existingTask.status === "ERRORED" ||
-      existingTask.status === "CANCELED"
-    ) {
-      logger.debug("Task already completed", {
-        existingTask,
       });
 
-      return taskWithAttemptsToServerTask(existingTask);
-    }
+      if (!existingTask) {
+        return;
+      }
 
-    if (existingTask.attempts.length === 1) {
-      await this.#prismaClient.taskAttempt.update({
+      if (existingTask.runId !== runId) {
+        return;
+      }
+
+      if (existingTask.run.environmentId !== environment.id) {
+        return;
+      }
+
+      if (
+        existingTask.status === "COMPLETED" ||
+        existingTask.status === "ERRORED" ||
+        existingTask.status === "CANCELED"
+      ) {
+        logger.debug("Task already completed", {
+          taskId: id,
+        });
+
+        return taskWithAttemptsToServerTask(existingTask);
+      }
+
+      if (existingTask.attempts.length === 1) {
+        await this.#prismaClient.taskAttempt.update({
+          where: {
+            id: existingTask.attempts[0].id,
+          },
+          data: {
+            status: "COMPLETED",
+          },
+        });
+      }
+
+      const updatedTask = await this.#prismaClient.task.update({
         where: {
-          id: existingTask.attempts[0].id,
+          id,
         },
         data: {
           status: "COMPLETED",
+          output: taskBody.output as any,
+          outputIsUndefined: typeof taskBody.output === "undefined",
+          completedAt: new Date(),
+          outputProperties: taskBody.properties,
+        },
+        include: {
+          attempts: true,
+          run: true,
         },
       });
-    }
 
-    const updatedTask = await this.#prismaClient.task.update({
-      where: {
-        id,
-      },
-      data: {
-        status: "COMPLETED",
-        output: taskBody.output as any,
-        outputIsUndefined: typeof taskBody.output === "undefined",
-        completedAt: new Date(),
-        outputProperties: taskBody.properties,
-      },
-      include: {
-        attempts: true,
-        run: true,
-      },
+      return taskWithAttemptsToServerTask(updatedTask);
     });
-
-    return taskWithAttemptsToServerTask(updatedTask);
   }
 }

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete/route.ts

@@ -9,8 +9,10 @@ import {
 import { z } from "zod";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { logger } from "~/services/logger.server";
 import { CompleteRunTaskService } from "./CompleteRunTaskService.server";
+import { startActiveSpan } from "~/v3/tracer.server";
+import { parseRequestJsonAsync } from "~/utils/parseRequestJson.server";
+import { FailRunTaskService } from "../api.v1.runs.$runId.tasks.$id.fail/FailRunTaskService.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -44,46 +46,51 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Invalid headers" }, { status: 400 });
   }
 
+  // Check the content size of the request and make sure it's not too large
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > 3 * 1024 * 1024) {
+    const service = new FailRunTaskService();
+
+    await service.call(authenticatedEnv, runId, id, {
+      error: {
+        message: "Task output is too large. The limit is 3MB",
+      },
+    });
+
+    return json({ error: "Task output is too large. The limit is 3MB" }, { status: 413 });
+  }
+
   const { "trigger-version": triggerVersion } = headers.data;
 
   // Now parse the request body
-  const anyBody = await request.json();
-
-  logger.debug("CompleteRunTaskService.call() request body", {
-    body: anyBody,
-    runId,
-    id,
-  });
+  const anyBody = await parseRequestJsonAsync(request, { runId });
 
   if (triggerVersion === API_VERSIONS.SERIALIZED_TASK_OUTPUT) {
-    const body = CompleteTaskBodyV2InputSchema.safeParse(anyBody);
+    const body = await startActiveSpan("CompleteTaskBodyV2InputSchema.safeParse()", async () => {
+      return CompleteTaskBodyV2InputSchema.safeParse(anyBody);
+    });
 
     if (!body.success) {
       return json({ error: "Invalid request body" }, { status: 400 });
     }
 
-    // Make sure the length of the output is less than 3MB
-    if (body.data.output && body.data.output.length > 3 * 1024 * 1024) {
-      return json({ error: "Output must be less than 3MB" }, { status: 400 });
-    }
-
     return await completeRunTask(authenticatedEnv, runId, id, {
       ...body.data,
       output: body.data.output ? (JSON.parse(body.data.output) as any) : undefined,
     });
   } else {
-    const body = CompleteTaskBodyInputSchema.safeParse(anyBody);
+    const body = await startActiveSpan("CompleteTaskBodyInputSchema.safeParse()", async () => {
+      return CompleteTaskBodyInputSchema.omit({ output: true }).safeParse(anyBody);
+    });
 
     if (!body.success) {
       return json({ error: "Invalid request body" }, { status: 400 });
     }
 
-    // Make sure the length of the output is less than 3MB
-    if (JSON.stringify(body.data.output).length > 3 * 1024 * 1024) {
-      return json({ error: "Output must be less than 3MB" }, { status: 400 });
-    }
+    const output = (anyBody as any).output;
 
-    return await completeRunTask(authenticatedEnv, runId, id, body.data);
+    return await completeRunTask(authenticatedEnv, runId, id, { ...body.data, output });
   }
 }
 
@@ -98,12 +105,6 @@ async function completeRunTask(
   try {
     const task = await service.call(environment, runId, id, taskBody);
 
-    logger.debug("CompleteRunTaskService.call() response body", {
-      runId,
-      id,
-      task,
-    });
-
     if (!task) {
       return json({ message: "Task not found" }, { status: 404 });
     }

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.fail/FailRunTaskService.server.ts

@@ -57,10 +57,6 @@ export class FailRunTaskService {
         existingTask.status === "ERRORED" ||
         existingTask.status === "CANCELED"
       ) {
-        logger.debug("Task already completed", {
-          existingTask,
-        });
-
         return existingTask;
       }
 

apps/webapp/app/routes/api.v1.runs.$runId.tasks/route.ts

@@ -6,6 +6,8 @@ import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { RunTaskService } from "~/services/tasks/runTask.server";
 import { ChangeRequestLazyLoadedCachedTasks } from "./ChangeRequestLazyLoadedCachedTasks.server";
+import { startActiveSpan } from "~/v3/tracer.server";
+import { parseRequestJsonAsync } from "~/utils/parseRequestJson.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -17,6 +19,8 @@ const HeadersSchema = z.object({
   "x-cached-tasks-cursor": z.string().optional().nullable(),
 });
 
+const BodySchema = RunTaskBodyOutputSchema.omit({ params: true });
+
 export async function action({ request, params }: ActionFunctionArgs) {
   // Ensure this is a POST request
   if (request.method.toUpperCase() !== "POST") {
@@ -44,18 +48,26 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const { runId } = ParamsSchema.parse(params);
 
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > 3 * 1024 * 1024) {
+    return json({ error: "Request body too large" }, { status: 413 });
+  }
+
   // Now parse the request body
-  const anyBody = await request.json();
+  const anyBody = await parseRequestJsonAsync(request, { runId });
 
-  logger.debug("RunTaskService.call() request body", {
-    body: anyBody,
-    runId,
-    idempotencyKey,
-    triggerVersion,
-    cachedTasksCursor,
-  });
-
-  const body = RunTaskBodyOutputSchema.safeParse(anyBody);
+  const body = await startActiveSpan(
+    "BodySchema.safeParse",
+    async () => {
+      return BodySchema.safeParse(anyBody);
+    },
+    {
+      attributes: {
+        runId,
+      },
+    }
+  );
 
   if (!body.success) {
     return json({ error: "Invalid request body" }, { status: 400 });
@@ -64,12 +76,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const service = new RunTaskService();
 
   try {
-    const task = await service.call(runId, idempotencyKey, body.data);
-
-    logger.debug("RunTaskService.call() response body", {
-      runId,
-      idempotencyKey,
-      task,
+    const task = await service.call(runId, idempotencyKey, {
+      ...body.data,
+      params: (anyBody as any).params,
     });
 
     if (!task) {
@@ -84,7 +93,6 @@ export async function action({ request, params }: ActionFunctionArgs) {
       logger.debug(
         "RunTaskService.call() response migrating with ChangeRequestLazyLoadedCachedTasks",
         {
-          responseBody,
           cachedTasksCursor,
         }
       );

apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts

@@ -79,9 +79,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
           friendlyId: parsedParams.data.scheduleId,
           taskIdentifier: body.data.task,
           cron: body.data.cron,
+          timezone: body.data.timezone,
           environments: [authenticationResult.environment.id],
           externalId: body.data.externalId,
-          deduplicationKey: body.data.deduplicationKey,
         };
 
         const schedule = await service.call(authenticationResult.environment.projectId, options);
@@ -95,6 +95,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
             expression: schedule.cron,
             description: schedule.cronDescription,
           },
+          timezone: schedule.timezone,
           externalId: schedule.externalId ?? undefined,
           deduplicationKey: schedule.deduplicationKey,
           environments: schedule.environments,

apps/webapp/app/routes/api.v1.schedules.ts

@@ -43,6 +43,7 @@ export async function action({ request }: ActionFunctionArgs) {
       environments: [authenticationResult.environment.id],
       externalId: body.data.externalId,
       deduplicationKey: body.data.deduplicationKey,
+      timezone: body.data.timezone,
     };
 
     const schedule = await service.call(authenticationResult.environment.projectId, options);
@@ -56,6 +57,7 @@ export async function action({ request }: ActionFunctionArgs) {
         expression: schedule.cron,
         description: schedule.cronDescription,
       },
+      timezone: schedule.timezone,
       externalId: schedule.externalId ?? undefined,
       deduplicationKey: schedule.deduplicationKey,
       environments: schedule.environments,
@@ -111,6 +113,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
         expression: schedule.cron,
         description: schedule.cronDescription,
       },
+      timezone: schedule.timezone,
       deduplicationKey: schedule.userProvidedDeduplicationKey
         ? schedule.deduplicationKey
         : undefined,

apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts

@@ -7,6 +7,7 @@ import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { BatchTriggerTaskService } from "~/v3/services/batchTriggerTask.server";
 import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger";
+import { env } from "~/env.server";
 
 const ParamsSchema = z.object({
   taskId: z.string(),
@@ -43,6 +44,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const { taskId } = ParamsSchema.parse(params);
 
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > env.TASK_PAYLOAD_MAXIMUM_SIZE) {
+    return json({ error: "Request body too large" }, { status: 413 });
+  }
+
   // Now parse the request body
   const anyBody = await request.json();
 

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -2,9 +2,12 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { TriggerTaskRequestBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
+import { env } from "~/env.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { parseRequestJsonAsync } from "~/utils/parseRequestJson.server";
 import { TriggerTaskService } from "~/v3/services/triggerTask.server";
+import { startActiveSpan } from "~/v3/tracer.server";
 
 const ParamsSchema = z.object({
   taskId: z.string(),
@@ -32,6 +35,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Invalid or Missing API key" }, { status: 401 });
   }
 
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > env.TASK_PAYLOAD_MAXIMUM_SIZE) {
+    return json({ error: "Request body too large" }, { status: 413 });
+  }
+
   const rawHeaders = Object.fromEntries(request.headers);
 
   const headers = HeadersSchema.safeParse(rawHeaders);
@@ -52,9 +61,11 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const { taskId } = ParamsSchema.parse(params);
 
   // Now parse the request body
-  const anyBody = await request.json();
+  const anyBody = await parseRequestJsonAsync(request, { taskId });
 
-  const body = TriggerTaskRequestBody.safeParse(anyBody);
+  const body = await startActiveSpan("TriggerTaskRequestBody.safeParse()", async (span) => {
+    return TriggerTaskRequestBody.safeParse(anyBody);
+  });
 
   if (!body.success) {
     return json({ error: "Invalid request body" }, { status: 400 });
@@ -76,17 +87,23 @@ export async function action({ request, params }: ActionFunctionArgs) {
       idempotencyKey,
       triggerVersion,
       headers: Object.fromEntries(request.headers),
-      body: body.data,
+      options: body.data.options,
       isFromWorker,
       traceContext,
     });
 
-    const run = await service.call(taskId, authenticationResult.environment, body.data, {
-      idempotencyKey: idempotencyKey ?? undefined,
-      triggerVersion: triggerVersion ?? undefined,
-      traceContext,
-      spanParentAsLink: spanParentAsLink === 1,
-    });
+    const run = await service.call(
+      taskId,
+      authenticationResult.environment,
+      { ...body.data },
+      // { ...body.data, payload: (anyBody as any).payload },
+      {
+        idempotencyKey: idempotencyKey ?? undefined,
+        triggerVersion: triggerVersion ?? undefined,
+        traceContext,
+        spanParentAsLink: spanParentAsLink === 1,
+      }
+    );
 
     if (!run) {
       return json({ error: "Task not found" }, { status: 404 });

apps/webapp/app/routes/api.v1.timezones.ts

@@ -0,0 +1,28 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { apiCors } from "~/utils/apiCors";
+import { getTimezones } from "~/utils/timezones.server";
+
+const SearchParamsSchema = z.object({
+  excludeUtc: z.preprocess((value) => value === "true", z.boolean()).default(false),
+});
+
+export async function loader({ request }: LoaderFunctionArgs) {
+  if (request.method.toUpperCase() === "OPTIONS") {
+    return apiCors(request, json({}));
+  }
+
+  const rawSearchParams = new URL(request.url).searchParams;
+  const params = SearchParamsSchema.safeParse(Object.fromEntries(rawSearchParams.entries()));
+
+  if (!params.success) {
+    return apiCors(
+      request,
+      json({ error: "Invalid request parameters", issues: params.error.issues }, { status: 400 })
+    );
+  }
+
+  const timezones = getTimezones(!params.data.excludeUtc);
+  return apiCors(request, json({ timezones }));
+}

apps/webapp/app/routes/api.v1.usage.ingest.ts

@@ -0,0 +1,97 @@
+import { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { MachinePresetName } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { validateJWTTokenAndRenew } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { workerQueue } from "~/services/worker.server";
+import { machinePresetFromName } from "~/v3/machinePresets.server";
+import { reportUsageEvent } from "~/v3/openMeter.server";
+
+const JWTPayloadSchema = z.object({
+  environment_id: z.string(),
+  org_id: z.string(),
+  project_id: z.string(),
+  run_id: z.string(),
+  machine_preset: z.string(),
+});
+
+const BodySchema = z.object({
+  durationMs: z.number(),
+});
+
+export async function action({ request }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const jwtResult = await validateJWTTokenAndRenew(request, JWTPayloadSchema);
+
+  if (!jwtResult) {
+    return { status: 401, body: "Unauthorized" };
+  }
+
+  const rawJson = await request.json();
+
+  const json = BodySchema.safeParse(rawJson);
+
+  if (!json.success) {
+    logger.error("Failed to parse request body", { rawJson });
+
+    return { status: 400, body: "Bad Request" };
+  }
+
+  const preset = machinePresetFromName(jwtResult.payload.machine_preset as MachinePresetName);
+
+  logger.debug("[/api/v1/usage/ingest] Reporting usage", { jwtResult, json: json.data, preset });
+
+  if (json.data.durationMs > 0) {
+    const costInCents = json.data.durationMs * preset.centsPerMs;
+
+    await prisma.taskRun.update({
+      where: {
+        id: jwtResult.payload.run_id,
+      },
+      data: {
+        usageDurationMs: {
+          increment: json.data.durationMs,
+        },
+        costInCents: {
+          increment: json.data.durationMs * preset.centsPerMs,
+        },
+      },
+    });
+
+    try {
+      await reportUsageEvent({
+        source: "webapp",
+        type: "usage",
+        subject: jwtResult.payload.org_id,
+        data: {
+          durationMs: json.data.durationMs,
+          costInCents: String(costInCents),
+        },
+      });
+    } catch (e) {
+      logger.error("Failed to report usage event, enqueing v3.reportUsage", { error: e });
+
+      await workerQueue.enqueue("v3.reportUsage", {
+        orgId: jwtResult.payload.org_id,
+        data: {
+          costInCents: String(costInCents),
+        },
+        additionalData: {
+          durationMs: json.data.durationMs,
+        },
+      });
+    }
+  }
+
+  return new Response(null, {
+    status: 200,
+    headers: {
+      "x-trigger-jwt": jwtResult.jwt,
+    },
+  });
+}

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.schedules.new/route.tsx

@@ -3,9 +3,10 @@ import { parse } from "@conform-to/zod";
 import { CheckIcon, XMarkIcon } from "@heroicons/react/20/solid";
 import { Form, useActionData, useLocation, useNavigation } from "@remix-run/react";
 import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { useVirtualizer } from "@tanstack/react-virtual";
 import { parseExpression } from "cron-parser";
 import cronstrue from "cronstrue";
-import { useState } from "react";
+import { useRef, useState } from "react";
 import {
   environmentTextClassName,
   environmentTitle,
@@ -42,6 +43,7 @@ import { ProjectParamSchema, docsPath, v3SchedulesPath } from "~/utils/pathBuild
 import { CronPattern, UpsertSchedule } from "~/v3/schedules";
 import { UpsertTaskScheduleService } from "~/v3/services/upsertTaskSchedule.server";
 import { AIGeneratedCronField } from "../resources.orgs.$organizationSlug.projects.$projectParam.schedules.new.natural-language";
+import { TimezoneList } from "~/components/scheduled/timezones";
 
 const cronFormat = `*    *    *    *    *
 ┬    ┬    ┬    ┬    ┬
@@ -117,9 +119,12 @@ export function UpsertScheduleForm({
   schedule,
   possibleTasks,
   possibleEnvironments,
+  possibleTimezones,
   showGenerateField,
 }: EditableScheduleElements & { showGenerateField: boolean }) {
   const lastSubmission = useActionData();
+  const [selectedTimezone, setSelectedTimezone] = useState<string>(schedule?.timezone ?? "UTC");
+  const isUtc = selectedTimezone === "UTC";
   const [cronPattern, setCronPattern] = useState<string>(schedule?.cron ?? "");
   const navigation = useNavigation();
   const isLoading = navigation.state !== "idle";
@@ -127,18 +132,20 @@ export function UpsertScheduleForm({
   const project = useProject();
   const location = useLocation();
 
-  const [form, { taskIdentifier, cron, externalId, environments, deduplicationKey }] = useForm({
-    id: "create-schedule",
-    // TODO: type this
-    lastSubmission: lastSubmission as any,
-    shouldRevalidate: "onSubmit",
-    onValidate({ formData }) {
-      return parse(formData, { schema: UpsertSchedule });
-    },
-  });
+  const [form, { taskIdentifier, cron, timezone, externalId, environments, deduplicationKey }] =
+    useForm({
+      id: "create-schedule",
+      // TODO: type this
+      lastSubmission: lastSubmission as any,
+      shouldRevalidate: "onSubmit",
+      onValidate({ formData }) {
+        return parse(formData, { schema: UpsertSchedule });
+      },
+    });
 
   let cronPatternResult: CronPatternResult | undefined = undefined;
   let nextRuns: Date[] | undefined = undefined;
+
   if (cronPattern !== "") {
     const result = CronPattern.safeParse(cronPattern);
 
@@ -149,7 +156,10 @@ export function UpsertScheduleForm({
       };
     } else {
       try {
-        const expression = parseExpression(cronPattern, { utc: true });
+        const expression = parseExpression(
+          cronPattern,
+          isUtc ? { utc: true } : { tz: selectedTimezone }
+        );
         cronPatternResult = {
           isValid: true,
           description: cronstrue.toString(cronPattern),
@@ -195,6 +205,7 @@ export function UpsertScheduleForm({
                 items={possibleTasks}
                 filter={(task, search) => task.toLowerCase().includes(search.toLowerCase())}
                 dropdownIcon
+                variant="tertiary/medium"
               >
                 {(matches) => (
                   <>
@@ -241,25 +252,52 @@ export function UpsertScheduleForm({
                 <ValidCronMessage isValid={false} message={cronPatternResult.error} />
               )}
             </InputGroup>
+            <InputGroup>
+              <Label htmlFor={timezone.id}>Timezone</Label>
+              <Select
+                {...conform.select(timezone)}
+                placeholder="Select a timezone"
+                defaultValue={selectedTimezone}
+                value={selectedTimezone}
+                setValue={(e) => {
+                  if (Array.isArray(e)) return;
+                  setSelectedTimezone(e);
+                }}
+                items={possibleTimezones}
+                filter={{ keys: [(item) => item.replace(/\//g, " ").replace(/_/g, " ")] }}
+                dropdownIcon
+                variant="tertiary/medium"
+              >
+                {(matches) => <TimezoneList timezones={matches} />}
+              </Select>
+              <Hint>
+                {isUtc
+                  ? "UTC will not change with daylight savings time."
+                  : "This will automatically adjust for daylight savings time."}
+              </Hint>
+              <FormError id={timezone.errorId}>{timezone.error}</FormError>
+            </InputGroup>
             {nextRuns !== undefined && (
               <div className="flex flex-col gap-1">
                 <Header3>Next 5 runs</Header3>
                 <Table>
                   <TableHeader>
                     <TableRow>
+                      {!isUtc && <TableHeaderCell>{selectedTimezone}</TableHeaderCell>}
                       <TableHeaderCell>UTC</TableHeaderCell>
-                      <TableHeaderCell>Local time</TableHeaderCell>
                     </TableRow>
                   </TableHeader>
                   <TableBody>
                     {nextRuns.map((run, index) => (
                       <TableRow key={index}>
+                        {!isUtc && (
+                          <TableCell>
+                            <DateTime date={run} timeZone={selectedTimezone} />
+                          </TableCell>
+                        )}
                         <TableCell>
                           <DateTime date={run} timeZone="UTC" />
                         </TableCell>
-                        <TableCell>
-                          <DateTime date={run} />
-                        </TableCell>
                       </TableRow>
                     ))}
                   </TableBody>

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -33,7 +33,13 @@ import { redirectWithErrorMessage } from "~/models/message.server";
 import { Span, SpanPresenter } from "~/presenters/v3/SpanPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
-import { v3RunPath, v3RunSpanPath, v3SpanParamsSchema, v3TraceSpanPath } from "~/utils/pathBuilder";
+import {
+  v3RunDownloadLogsPath,
+  v3RunPath,
+  v3RunSpanPath,
+  v3SpanParamsSchema,
+  v3TraceSpanPath,
+} from "~/utils/pathBuilder";
 import { SpanLink } from "~/v3/eventRepository.server";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
@@ -256,6 +262,15 @@ function RunActionButtons({ span }: { span: Span }) {
   if (span.isPartial) {
     return (
       <Dialog>
+        <LinkButton
+          to={v3RunDownloadLogsPath({ friendlyId: runParam })}
+          LeadingIcon={CloudArrowDownIcon}
+          variant="tertiary/medium"
+          target="_blank"
+          download
+        >
+          Download logs
+        </LinkButton>
         <DialogTrigger asChild>
           <Button variant="danger/medium" LeadingIcon={StopCircleIcon}>
             Cancel run
@@ -276,6 +291,15 @@ function RunActionButtons({ span }: { span: Span }) {
 
   return (
     <Dialog>
+      <LinkButton
+        to={v3RunDownloadLogsPath({ friendlyId: runParam })}
+        LeadingIcon={CloudArrowDownIcon}
+        variant="tertiary/medium"
+        target="_blank"
+        download
+      >
+        Download logs
+      </LinkButton>
       <DialogTrigger asChild>
         <Button variant="tertiary/medium" LeadingIcon={ArrowPathIcon}>
           Replay run

apps/webapp/app/routes/resources.packets.$environmentId.$.ts

@@ -2,9 +2,8 @@ import { LoaderFunctionArgs } from "@remix-run/node";
 import { basename } from "node:path";
 import { z } from "zod";
 import { prisma } from "~/db.server";
-import { env } from "~/env.server";
 import { requireUserId } from "~/services/session.server";
-import { r2 } from "~/v3/r2.server";
+import { generatePresignedRequest } from "~/v3/r2.server";
 
 const ParamSchema = z.object({
   environmentId: z.string(),
@@ -35,27 +34,17 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     return new Response("Not found", { status: 404 });
   }
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    return new Response("Object store base URL is not set", { status: 500 });
-  }
-
-  if (!r2) {
-    return new Response("Object store credentials are not set", { status: 500 });
-  }
-
-  const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${environment.project.externalRef}/${environment.slug}/${filename}`;
-  url.searchParams.set("X-Amz-Expires", "30"); // 30 seconds
-
-  const signed = await r2.sign(
-    new Request(url, {
-      method: "GET",
-    }),
-    {
-      aws: { signQuery: true },
-    }
+  const signed = await generatePresignedRequest(
+    environment.project.externalRef,
+    environment.slug,
+    filename,
+    "GET"
   );
 
+  if (!signed) {
+    return new Response("Failed to generate presigned URL", { status: 500 });
+  }
+
   const response = await fetch(signed.url, {
     headers: signed.headers,
   });
@@ -64,7 +53,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     status: 200,
     headers: {
       "Content-Type": "application/octet-stream",
-      "Content-Disposition": `attachment; filename="${basename(url.pathname)}"`,
+      "Content-Disposition": `attachment; filename="${basename(filename)}"`,
     },
   });
 }

apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts

@@ -0,0 +1,112 @@
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { requireUserId } from "~/services/session.server";
+import { v3RunParamsSchema } from "~/utils/pathBuilder";
+import {
+  PreparedEvent,
+  RunPreparedEvent,
+  eventRepository,
+  getDateFromNanoseconds,
+} from "~/v3/eventRepository.server";
+import { createGzip } from "zlib";
+import { Readable } from "stream";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3/utils/durations";
+
+export async function loader({ params, request }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+  const parsedParams = v3RunParamsSchema.pick({ runParam: true }).parse(params);
+
+  const run = await prisma.taskRun.findFirst({
+    where: {
+      friendlyId: parsedParams.runParam,
+      project: {
+        organization: {
+          members: {
+            some: {
+              userId,
+            },
+          },
+        },
+      },
+    },
+  });
+
+  if (!run) {
+    return new Response("Not found", { status: 404 });
+  }
+
+  const runEvents = await eventRepository.getRunEvents(run.friendlyId);
+
+  // Create a Readable stream from the runEvents array
+  const readable = new Readable({
+    read() {
+      runEvents.forEach((event) => {
+        try {
+          this.push(formatRunEvent(event) + "\n");
+        } catch {}
+      });
+      this.push(null); // End of stream
+    },
+  });
+
+  // Create a gzip transform stream
+  const gzip = createGzip();
+
+  // Pipe the readable stream into the gzip stream
+  const compressedStream = readable.pipe(gzip);
+
+  // Return the response with the compressed stream
+  return new Response(compressedStream as any, {
+    status: 200,
+    headers: {
+      "Content-Type": "application/octet-stream",
+      "Content-Disposition": `attachment; filename="${parsedParams.runParam}.log"`,
+      "Content-Encoding": "gzip",
+    },
+  });
+}
+
+function formatRunEvent(event: RunPreparedEvent): string {
+  const entries = [];
+  const parts: string[] = [];
+
+  parts.push(getDateFromNanoseconds(event.startTime).toISOString());
+
+  if (event.taskSlug) {
+    parts.push(event.taskSlug);
+  }
+
+  parts.push(event.level);
+  parts.push(event.message);
+
+  if (event.level === "TRACE") {
+    parts.push(`(${formatDurationMilliseconds(event.duration / 1_000_000)})`);
+  }
+
+  entries.push(parts.join(" "));
+
+  if (event.events) {
+    for (const subEvent of event.events) {
+      if (subEvent.name === "exception") {
+        const subEventParts: string[] = [];
+
+        subEventParts.push(subEvent.time as unknown as string);
+
+        if (event.taskSlug) {
+          subEventParts.push(event.taskSlug);
+        }
+
+        subEventParts.push(subEvent.name);
+        subEventParts.push((subEvent.properties as any).exception.message);
+
+        if ((subEvent.properties as any).exception.stack) {
+          subEventParts.push((subEvent.properties as any).exception.stack);
+        }
+
+        entries.push(subEventParts.join(" "));
+      }
+    }
+  }
+
+  return entries.join("\n");
+}

apps/webapp/app/services/apiAuth.server.ts

@@ -1,6 +1,7 @@
 import { Prettify } from "@trigger.dev/core";
 import { z } from "zod";
 import {
+  RuntimeEnvironment,
   findEnvironmentByApiKey,
   findEnvironmentByPublicApiKey,
 } from "~/models/runtimeEnvironment.server";
@@ -12,6 +13,9 @@ import {
 import { prisma } from "~/db.server";
 import { json } from "@remix-run/server-runtime";
 import { findProjectByRef } from "~/models/project.server";
+import { SignJWT, jwtVerify, errors } from "jose";
+import { env } from "~/env.server";
+import { logger } from "./logger.server";
 
 type Optional<T, K extends keyof T> = Prettify<Omit<T, K> & Partial<Pick<T, K>>>;
 
@@ -209,3 +213,127 @@ export async function authenticatedEnvironmentForAuthentication(
     }
   }
 }
+
+const JWT_SECRET = new TextEncoder().encode(env.SESSION_SECRET);
+const JWT_ALGORITHM = "HS256";
+const DEFAULT_JWT_EXPIRATION_IN_MS = 1000 * 60 * 60; // 1 hour
+
+export async function generateJWTTokenForEnvironment(
+  environment: RuntimeEnvironment,
+  payload: Record<string, string>
+) {
+  const jwt = await new SignJWT({
+    environment_id: environment.id,
+    org_id: environment.organizationId,
+    project_id: environment.projectId,
+    ...payload,
+  })
+    .setProtectedHeader({ alg: JWT_ALGORITHM })
+    .setIssuedAt()
+    .setIssuer("https://id.trigger.dev")
+    .setAudience("https://api.trigger.dev")
+    .setExpirationTime(calculateJWTExpiration())
+    .sign(JWT_SECRET);
+
+  return jwt;
+}
+
+export async function validateJWTTokenAndRenew<T extends z.ZodTypeAny>(
+  request: Request,
+  payloadSchema: T
+): Promise<{ payload: z.infer<T>; jwt: string } | undefined> {
+  try {
+    const jwt = request.headers.get("x-trigger-jwt");
+
+    if (!jwt) {
+      logger.debug("Missing JWT token in request", {
+        headers: Object.fromEntries(request.headers),
+      });
+
+      return;
+    }
+
+    const { payload: rawPayload } = await jwtVerify(jwt, JWT_SECRET, {
+      issuer: "https://id.trigger.dev",
+      audience: "https://api.trigger.dev",
+    });
+
+    const payload = payloadSchema.safeParse(rawPayload);
+
+    if (!payload.success) {
+      logger.error("Failed to validate JWT", { payload: rawPayload, issues: payload.error.issues });
+
+      return;
+    }
+
+    const renewedJwt = await renewJWTToken(payload.data);
+
+    return {
+      payload: payload.data,
+      jwt: renewedJwt,
+    };
+  } catch (error) {
+    if (error instanceof errors.JWTExpired) {
+      // Now we need to try and renew the token using the API key auth
+      const authenticatedEnv = await authenticateApiRequest(request);
+
+      if (!authenticatedEnv) {
+        logger.error("Failed to renew JWT token, missing or invalid Authorization header", {
+          error: error.message,
+        });
+
+        return;
+      }
+
+      const payload = payloadSchema.safeParse(error.payload);
+
+      if (!payload.success) {
+        logger.error("Failed to parse jwt payload after expired", {
+          payload: error.payload,
+          issues: payload.error.issues,
+        });
+
+        return;
+      }
+
+      const renewedJwt = await generateJWTTokenForEnvironment(authenticatedEnv.environment, {
+        ...payload.data,
+      });
+
+      logger.debug("Renewed JWT token from Authorization header API Key", {
+        environment: authenticatedEnv.environment,
+        payload: payload.data,
+      });
+
+      return {
+        payload: payload.data,
+        jwt: renewedJwt,
+      };
+    }
+
+    logger.error("Failed to validate JWT token", { error });
+  }
+}
+
+async function renewJWTToken(payload: Record<string, string>) {
+  const jwt = await new SignJWT(payload)
+    .setProtectedHeader({ alg: JWT_ALGORITHM })
+    .setIssuedAt()
+    .setIssuer("https://id.trigger.dev")
+    .setAudience("https://api.trigger.dev")
+    .setExpirationTime(calculateJWTExpiration())
+    .sign(JWT_SECRET);
+
+  return jwt;
+}
+
+function calculateJWTExpiration() {
+  if (env.PROD_USAGE_HEARTBEAT_INTERVAL_MS) {
+    return (
+      (Date.now() + Math.max(DEFAULT_JWT_EXPIRATION_IN_MS, env.PROD_USAGE_HEARTBEAT_INTERVAL_MS)) /
+      1000
+    );
+  }
+
+  return (Date.now() + DEFAULT_JWT_EXPIRATION_IN_MS) / 1000;
+}

apps/webapp/app/services/apiRateLimit.server.ts

@@ -145,11 +145,13 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
     "/api/internal/stripe_webhooks",
     "/api/v1/authorization-code",
     "/api/v1/token",
+    "/api/v1/usage/ingest",
     /^\/api\/v1\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/tasks/$id/callback/$secret
     /^\/api\/v1\/runs\/[^\/]+\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/runs/$runId/tasks/$id/callback/$secret
     /^\/api\/v1\/http-endpoints\/[^\/]+\/env\/[^\/]+\/[^\/]+$/, // /api/v1/http-endpoints/$httpEndpointId/env/$envType/$shortcode
     /^\/api\/v1\/sources\/http\/[^\/]+$/, // /api/v1/sources/http/$id
     /^\/api\/v1\/endpoints\/[^\/]+\/[^\/]+\/index\/[^\/]+$/, // /api/v1/endpoints/$environmentId/$endpointSlug/index/$indexHookIdentifier
+    "/api/v1/timezones",
   ],
   log: {
     rejections: env.API_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",

apps/webapp/app/services/endpointApi.server.ts

@@ -136,7 +136,7 @@ export class EndpointApi {
     };
   }
 
-  async executeJobRequest(options: RunJobBody) {
+  async executeJobRequest(options: RunJobBody, timeoutInMs?: number) {
     const startTimeInMs = performance.now();
 
     const response = await safeFetch(this.url, {
@@ -147,8 +147,18 @@ export class EndpointApi {
         "x-trigger-action": "EXECUTE_JOB",
       },
       body: JSON.stringify(options),
+      signal: timeoutInMs ? AbortSignal.timeout(timeoutInMs) : undefined,
     });
 
+    if (response) {
+      logger.debug("executeJobRequest() response from endpoint", {
+        status: response.status,
+        headers: Object.fromEntries(response.headers.entries()),
+      });
+    } else {
+      logger.debug("executeJobRequest() no response from endpoint");
+    }
+
     return {
       response,
       parser: RunJobResponseSchema,
@@ -434,7 +444,10 @@ async function safeFetch(url: string, options: RequestInit) {
   } catch (error) {
     logger.debug("Error while trying to connect to endpoint", {
       url,
-      error,
+      error:
+        error instanceof Error
+          ? { name: error.name, message: error.message, stack: error.stack }
+          : String(error),
     });
   }
 }

apps/webapp/app/services/runs/performRunExecutionV3.server.ts

@@ -269,7 +269,10 @@ export class PerformRunExecutionV3Service {
 
       // TODO: add the ability to abort the execution from any server using Redis pub/sub
       const { response, parser, errorParser, headersParser, durationInMs } =
-        await client.executeJobRequest(executionBody);
+        await client.executeJobRequest(
+          executionBody,
+          run.environment.type === "DEVELOPMENT" ? 60_000 * 5 : undefined
+        );
 
       await createExecutionEvent({
         eventType: "finish",
@@ -929,6 +932,25 @@ export class PerformRunExecutionV3Service {
     executionCount: number = 1
   ) {
     await $transaction(this.#prismaClient, async (tx) => {
+      const service = new CompleteRunTaskService(tx);
+
+      const task = await service.call(run.environment, run.id, data.id, {
+        properties: data.properties,
+        output: data.output ? (JSON.parse(data.output) as any) : undefined,
+      });
+
+      if (!task || task.status === "ERRORED") {
+        return await this.#failRunExecution(
+          tx,
+          run,
+          {
+            message: task ? `Task '${task.name}' failed to complete` : "Task failed to complete",
+          },
+          "FAILURE",
+          durationInMs
+        );
+      }
+
       await tx.jobRun.update({
         where: {
           id: run.id,
@@ -958,13 +980,6 @@ export class PerformRunExecutionV3Service {
         },
       });
 
-      const service = new CompleteRunTaskService(tx);
-
-      await service.call(run.environment, run.id, data.id, {
-        properties: data.properties,
-        output: data.output ? (JSON.parse(data.output) as any) : undefined,
-      });
-
       await ResumeRunService.enqueue(run, tx);
     });
   }

apps/webapp/app/services/tasks/runTask.server.ts

@@ -6,6 +6,7 @@ import { taskWithAttemptsToServerTask } from "~/models/task.server";
 import { generateSecret } from "~/services/sources/utils.server";
 import { ulid } from "~/services/ulid.server";
 import { taskOperationWorker, workerQueue } from "~/services/worker.server";
+import { startActiveSpan } from "~/v3/tracer.server";
 
 export class RunTaskService {
   #prismaClient: PrismaClient;
@@ -19,142 +20,154 @@ export class RunTaskService {
     idempotencyKey: string,
     taskBody: RunTaskBodyOutput
   ): Promise<ServerTask | undefined> {
-    const delayUntilInFuture = taskBody.delayUntil
-      ? taskBody.delayUntil.getTime() > Date.now()
-      : false;
-    const callbackEnabled = taskBody.callback?.enabled ?? false;
+    return startActiveSpan("RunTaskService.call", async (span) => {
+      span.setAttribute("runId", runId);
 
-    // First
-    const existingTask = await this.#handleExistingTask(
-      runId,
-      idempotencyKey,
-      taskBody,
-      delayUntilInFuture,
-      callbackEnabled
-    );
+      const delayUntilInFuture = taskBody.delayUntil
+        ? taskBody.delayUntil.getTime() > Date.now()
+        : false;
+      const callbackEnabled = taskBody.callback?.enabled ?? false;
 
-    if (existingTask) {
-      return taskWithAttemptsToServerTask(existingTask);
-    }
+      // First
+      const existingTask = await this.#handleExistingTask(
+        runId,
+        idempotencyKey,
+        taskBody,
+        delayUntilInFuture,
+        callbackEnabled
+      );
 
-    const run = await this.#prismaClient.jobRun.findUnique({
-      where: {
-        id: runId,
-      },
-      select: {
-        status: true,
-        forceYieldImmediately: true,
-      },
-    });
+      if (existingTask) {
+        span.setAttribute("taskId", existingTask.id);
 
-    if (!run) throw new Error("Run not found");
-
-    const runConnection = taskBody.connectionKey
-      ? await this.#prismaClient.runConnection.findUnique({
-          where: {
-            runId_key: {
-              runId,
-              key: taskBody.connectionKey,
-            },
-          },
-          select: {
-            id: true,
-          },
-        })
-      : undefined;
-
-    const results = await $transaction(this.#prismaClient, async (tx) => {
-      // If task.delayUntil is set and is in the future, we'll set the task's status to "WAITING", else set it to RUNNING
-      let status: TaskStatus;
-
-      if (run.status === "CANCELED") {
-        status = "CANCELED";
-      } else {
-        status =
-          delayUntilInFuture || callbackEnabled
-            ? "WAITING"
-            : taskBody.noop
-            ? "COMPLETED"
-            : "RUNNING";
+        return taskWithAttemptsToServerTask(existingTask);
       }
 
-      const taskId = ulid();
-      const callbackUrl = callbackEnabled
-        ? `${env.APP_ORIGIN}/api/v1/tasks/${taskId}/callback/${generateSecret(12)}`
+      const run = await this.#prismaClient.jobRun.findUnique({
+        where: {
+          id: runId,
+        },
+        select: {
+          status: true,
+          forceYieldImmediately: true,
+        },
+      });
+
+      if (!run) throw new Error("Run not found");
+
+      const runConnection = taskBody.connectionKey
+        ? await this.#prismaClient.runConnection.findUnique({
+            where: {
+              runId_key: {
+                runId,
+                key: taskBody.connectionKey,
+              },
+            },
+            select: {
+              id: true,
+            },
+          })
         : undefined;
 
-      const task = await tx.task.create({
-        data: {
-          id: taskId,
-          idempotencyKey,
-          displayKey: taskBody.displayKey,
-          runConnectionId: runConnection ? runConnection.id : undefined,
-          icon: taskBody.icon,
-          runId,
-          parentId: taskBody.parentId,
-          name: taskBody.name ?? "Task",
-          description: taskBody.description,
-          status,
-          startedAt: new Date(),
-          completedAt: status === "COMPLETED" || status === "CANCELED" ? new Date() : undefined,
-          noop: taskBody.noop,
-          delayUntil: taskBody.delayUntil,
-          params: taskBody.params ?? undefined,
-          properties: this.#filterProperties(taskBody.properties) ?? undefined,
-          redact: taskBody.redact ?? undefined,
-          operation: taskBody.operation,
-          callbackUrl,
-          style: taskBody.style ?? { style: "normal" },
-          childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
-        },
-      });
+      const results = await $transaction(
+        this.#prismaClient,
+        async (tx) => {
+          // If task.delayUntil is set and is in the future, we'll set the task's status to "WAITING", else set it to RUNNING
+          let status: TaskStatus;
 
-      const taskAttempt = await tx.taskAttempt.create({
-        data: {
-          number: 1,
-          taskId: task.id,
-          status: "PENDING",
-        },
-      });
+          if (run.status === "CANCELED") {
+            status = "CANCELED";
+          } else {
+            status =
+              delayUntilInFuture || callbackEnabled
+                ? "WAITING"
+                : taskBody.noop
+                ? "COMPLETED"
+                : "RUNNING";
+          }
 
-      if (task.status === "RUNNING" && typeof taskBody.operation === "string") {
-        // We need to schedule the operation
-        await taskOperationWorker.enqueue(
-          "performTaskOperation",
-          {
-            id: task.id,
-          },
-          { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
-        );
-      } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
-        if (taskBody.callback.timeoutInSeconds > 0) {
-          // We need to schedule the callback timeout
-          await workerQueue.enqueue(
-            "processCallbackTimeout",
-            {
-              id: task.id,
+          const taskId = ulid();
+          const callbackUrl = callbackEnabled
+            ? `${env.APP_ORIGIN}/api/v1/tasks/${taskId}/callback/${generateSecret(12)}`
+            : undefined;
+
+          const task = await tx.task.create({
+            data: {
+              id: taskId,
+              idempotencyKey,
+              displayKey: taskBody.displayKey,
+              runConnectionId: runConnection ? runConnection.id : undefined,
+              icon: taskBody.icon,
+              runId,
+              parentId: taskBody.parentId,
+              name: taskBody.name ?? "Task",
+              description: taskBody.description,
+              status,
+              startedAt: new Date(),
+              completedAt: status === "COMPLETED" || status === "CANCELED" ? new Date() : undefined,
+              noop: taskBody.noop,
+              delayUntil: taskBody.delayUntil,
+              params: taskBody.params ?? undefined,
+              properties: this.#filterProperties(taskBody.properties) ?? undefined,
+              redact: taskBody.redact ?? undefined,
+              operation: taskBody.operation,
+              callbackUrl,
+              style: taskBody.style ?? { style: "normal" },
+              childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
             },
-            {
-              tx,
-              runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
-              jobKey: `process-callback:${task.id}`,
+          });
+
+          span.setAttribute("taskId", task.id);
+
+          const taskAttempt = await tx.taskAttempt.create({
+            data: {
+              number: 1,
+              taskId: task.id,
+              status: "PENDING",
+            },
+          });
+
+          if (task.status === "RUNNING" && typeof taskBody.operation === "string") {
+            // We need to schedule the operation
+            await taskOperationWorker.enqueue(
+              "performTaskOperation",
+              {
+                id: task.id,
+              },
+              { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
+            );
+          } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
+            if (taskBody.callback.timeoutInSeconds > 0) {
+              // We need to schedule the callback timeout
+              await workerQueue.enqueue(
+                "processCallbackTimeout",
+                {
+                  id: task.id,
+                },
+                {
+                  tx,
+                  runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
+                  jobKey: `process-callback:${task.id}`,
+                }
+              );
             }
-          );
-        }
+          }
+
+          return { task, taskAttempt };
+        },
+        { timeout: 10000 }
+      );
+
+      if (!results) {
+        return;
       }
 
-      return { task, taskAttempt };
+      const { task, taskAttempt } = results;
+
+      return task
+        ? taskWithAttemptsToServerTask({ ...task, attempts: [taskAttempt], run })
+        : undefined;
     });
-
-    if (!results) {
-      return;
-    }
-
-    const { task, taskAttempt } = results;
-
-    return task
-      ? taskWithAttemptsToServerTask({ ...task, attempts: [taskAttempt], run })
-      : undefined;
   }
 
   async #handleExistingTask(

apps/webapp/app/services/worker.server.ts

@@ -46,6 +46,7 @@ import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout.se
 import { ResumeTaskService } from "./tasks/resumeTask.server";
 import { RequeueV2Message } from "~/v3/marqs/requeueV2Message.server";
 import { MarqsConcurrencyMonitor } from "~/v3/marqs/concurrencyMonitor.server";
+import { reportUsageEvent } from "~/v3/openMeter.server";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -169,6 +170,13 @@ const workerCatalog = {
   "v2.requeueMessage": z.object({
     runId: z.string(),
   }),
+  "v3.reportUsage": z.object({
+    orgId: z.string(),
+    data: z.object({
+      costInCents: z.string(),
+    }),
+    additionalData: z.record(z.any()).optional(),
+  }),
 };
 
 const executionWorkerCatalog = {
@@ -649,6 +657,21 @@ function getWorkerQueue() {
           await service.call(payload.runId);
         },
       },
+      "v3.reportUsage": {
+        priority: 0,
+        maxAttempts: 8,
+        handler: async (payload, job) => {
+          await reportUsageEvent({
+            source: "webapp",
+            type: "usage",
+            subject: payload.orgId,
+            data: {
+              costInCents: payload.data.costInCents,
+              ...payload.additionalData,
+            },
+          });
+        },
+      },
     },
   });
 }

apps/webapp/app/utils/parseRequestJson.server.ts

@@ -0,0 +1,29 @@
+import { Attributes } from "@opentelemetry/api";
+import { startActiveSpan } from "~/v3/tracer.server";
+
+export async function parseRequestJsonAsync(
+  request: Request,
+  attributes?: Attributes
+): Promise<unknown> {
+  return await startActiveSpan(
+    "parseRequestJsonAsync()",
+    async (span) => {
+      span.setAttribute("content-length", parseInt(request.headers.get("content-length") ?? "0"));
+      span.setAttribute("content-type", request.headers.get("content-type") ?? "application/json");
+      span.setAttribute("experiment.async", false);
+
+      const rawText = await startActiveSpan("request.text()", async () => {
+        return await request.text();
+      });
+
+      if (rawText.length === 0) {
+        return;
+      }
+
+      return JSON.parse(rawText);
+    },
+    {
+      attributes,
+    }
+  );
+}

apps/webapp/app/utils/pathBuilder.ts

@@ -375,6 +375,10 @@ export function v3RunPath(organization: OrgForPath, project: ProjectForPath, run
   return `${v3RunsPath(organization, project)}/${run.friendlyId}`;
 }
 
+export function v3RunDownloadLogsPath(run: v3RunForPath) {
+  return `/resources/runs/${run.friendlyId}/logs/download`;
+}
+
 export function v3RunSpanPath(
   organization: OrgForPath,
   project: ProjectForPath,

apps/webapp/app/utils/timezones.server.ts

@@ -0,0 +1,7 @@
+export function getTimezones(includeUtc = true) {
+  const possibleTimezones = Intl.supportedValuesOf("timeZone").sort();
+  if (includeUtc) {
+    possibleTimezones.unshift("UTC");
+  }
+  return possibleTimezones;
+}

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -1,4 +1,9 @@
-import { Prisma, PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database";
+import {
+  Prisma,
+  PrismaClient,
+  RuntimeEnvironment,
+  RuntimeEnvironmentType,
+} from "@trigger.dev/database";
 import { z } from "zod";
 import { environmentTitle } from "~/components/environments/EnvironmentLabel";
 import { $transaction, prisma } from "~/db.server";
@@ -427,11 +432,7 @@ export class EnvironmentVariablesRepository implements Repository {
     return results;
   }
 
-  async getEnvironment(
-    projectId: string,
-    environmentId: string,
-    excludeInternalVariables?: boolean
-  ): Promise<EnvironmentVariable[]> {
+  async getEnvironment(projectId: string, environmentId: string): Promise<EnvironmentVariable[]> {
     const project = await this.prismaClient.project.findUnique({
       where: {
         id: projectId,
@@ -453,124 +454,7 @@ export class EnvironmentVariablesRepository implements Repository {
       return [];
     }
 
-    return this.getEnvironmentVariables(projectId, environmentId, excludeInternalVariables);
-  }
-
-  async #getTriggerEnvironmentVariables(environmentId: string): Promise<EnvironmentVariable[]> {
-    const environment = await this.prismaClient.runtimeEnvironment.findFirst({
-      where: {
-        id: environmentId,
-      },
-    });
-
-    if (!environment) {
-      return [];
-    }
-
-    if (environment.type === "DEVELOPMENT") {
-      return [
-        {
-          key: "OTEL_EXPORTER_OTLP_ENDPOINT",
-          value: env.DEV_OTEL_EXPORTER_OTLP_ENDPOINT ?? env.APP_ORIGIN,
-        },
-      ].concat(
-        env.DEV_OTEL_BATCH_PROCESSING_ENABLED === "1"
-          ? [
-              {
-                key: "OTEL_BATCH_PROCESSING_ENABLED",
-                value: "1",
-              },
-              {
-                key: "OTEL_SPAN_MAX_EXPORT_BATCH_SIZE",
-                value: env.DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE,
-              },
-              {
-                key: "OTEL_SPAN_SCHEDULED_DELAY_MILLIS",
-                value: env.DEV_OTEL_SPAN_SCHEDULED_DELAY_MILLIS,
-              },
-              {
-                key: "OTEL_SPAN_EXPORT_TIMEOUT_MILLIS",
-                value: env.DEV_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS,
-              },
-              {
-                key: "OTEL_SPAN_MAX_QUEUE_SIZE",
-                value: env.DEV_OTEL_SPAN_MAX_QUEUE_SIZE,
-              },
-              {
-                key: "OTEL_LOG_MAX_EXPORT_BATCH_SIZE",
-                value: env.DEV_OTEL_LOG_MAX_EXPORT_BATCH_SIZE,
-              },
-              {
-                key: "OTEL_LOG_SCHEDULED_DELAY_MILLIS",
-                value: env.DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS,
-              },
-              {
-                key: "OTEL_LOG_EXPORT_TIMEOUT_MILLIS",
-                value: env.DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS,
-              },
-              {
-                key: "OTEL_LOG_MAX_QUEUE_SIZE",
-                value: env.DEV_OTEL_LOG_MAX_QUEUE_SIZE,
-              },
-            ]
-          : []
-      );
-    }
-
-    return [
-      {
-        key: "TRIGGER_SECRET_KEY",
-        value: environment.apiKey,
-      },
-      {
-        key: "TRIGGER_API_URL",
-        value: env.APP_ORIGIN,
-      },
-      {
-        key: "TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS",
-        value: String(env.RUNTIME_WAIT_THRESHOLD_IN_MS),
-      },
-      ...(env.PROD_OTEL_BATCH_PROCESSING_ENABLED === "1"
-        ? [
-            {
-              key: "OTEL_BATCH_PROCESSING_ENABLED",
-              value: "1",
-            },
-            {
-              key: "OTEL_SPAN_MAX_EXPORT_BATCH_SIZE",
-              value: env.PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE,
-            },
-            {
-              key: "OTEL_SPAN_SCHEDULED_DELAY_MILLIS",
-              value: env.PROD_OTEL_SPAN_SCHEDULED_DELAY_MILLIS,
-            },
-            {
-              key: "OTEL_SPAN_EXPORT_TIMEOUT_MILLIS",
-              value: env.PROD_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS,
-            },
-            {
-              key: "OTEL_SPAN_MAX_QUEUE_SIZE",
-              value: env.PROD_OTEL_SPAN_MAX_QUEUE_SIZE,
-            },
-            {
-              key: "OTEL_LOG_MAX_EXPORT_BATCH_SIZE",
-              value: env.PROD_OTEL_LOG_MAX_EXPORT_BATCH_SIZE,
-            },
-            {
-              key: "OTEL_LOG_SCHEDULED_DELAY_MILLIS",
-              value: env.PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS,
-            },
-            {
-              key: "OTEL_LOG_EXPORT_TIMEOUT_MILLIS",
-              value: env.PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS,
-            },
-            {
-              key: "OTEL_LOG_MAX_QUEUE_SIZE",
-              value: env.PROD_OTEL_LOG_MAX_QUEUE_SIZE,
-            },
-          ]
-        : []),
-    ];
+    return this.getEnvironmentVariables(projectId, environmentId);
   }
 
   async #getSecretEnvironmentVariables(
@@ -597,18 +481,9 @@ export class EnvironmentVariablesRepository implements Repository {
 
   async getEnvironmentVariables(
     projectId: string,
-    environmentId: string,
-    excludeInternalVariables?: boolean
+    environmentId: string
   ): Promise<EnvironmentVariable[]> {
-    const secretEnvVars = await this.#getSecretEnvironmentVariables(projectId, environmentId);
-
-    if (excludeInternalVariables) {
-      return secretEnvVars;
-    }
-
-    const triggerEnvVars = await this.#getTriggerEnvironmentVariables(environmentId);
-
-    return [...secretEnvVars, ...triggerEnvVars];
+    return this.#getSecretEnvironmentVariables(projectId, environmentId);
   }
 
   async delete(projectId: string, options: DeleteEnvironmentVariable): Promise<Result> {
@@ -782,3 +657,158 @@ export class EnvironmentVariablesRepository implements Repository {
     }
   }
 }
+
+export const environmentVariablesRepository = new EnvironmentVariablesRepository();
+
+export async function resolveVariablesForEnvironment(runtimeEnvironment: RuntimeEnvironment) {
+  const projectSecrets = await environmentVariablesRepository.getEnvironmentVariables(
+    runtimeEnvironment.projectId,
+    runtimeEnvironment.id
+  );
+
+  const builtInVariables =
+    runtimeEnvironment.type === "DEVELOPMENT"
+      ? await resolveBuiltInDevVariables(runtimeEnvironment)
+      : await resolveBuiltInProdVariables(runtimeEnvironment);
+
+  return [...projectSecrets, ...builtInVariables];
+}
+
+async function resolveBuiltInDevVariables(runtimeEnvironment: RuntimeEnvironment) {
+  let result: Array<EnvironmentVariable> = [
+    {
+      key: "OTEL_EXPORTER_OTLP_ENDPOINT",
+      value: env.DEV_OTEL_EXPORTER_OTLP_ENDPOINT ?? env.APP_ORIGIN,
+    },
+  ];
+
+  if (env.DEV_OTEL_BATCH_PROCESSING_ENABLED === "1") {
+    result = result.concat([
+      {
+        key: "OTEL_BATCH_PROCESSING_ENABLED",
+        value: "1",
+      },
+      {
+        key: "OTEL_SPAN_MAX_EXPORT_BATCH_SIZE",
+        value: env.DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "OTEL_SPAN_SCHEDULED_DELAY_MILLIS",
+        value: env.DEV_OTEL_SPAN_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "OTEL_SPAN_EXPORT_TIMEOUT_MILLIS",
+        value: env.DEV_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "OTEL_SPAN_MAX_QUEUE_SIZE",
+        value: env.DEV_OTEL_SPAN_MAX_QUEUE_SIZE,
+      },
+      {
+        key: "OTEL_LOG_MAX_EXPORT_BATCH_SIZE",
+        value: env.DEV_OTEL_LOG_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "OTEL_LOG_SCHEDULED_DELAY_MILLIS",
+        value: env.DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "OTEL_LOG_EXPORT_TIMEOUT_MILLIS",
+        value: env.DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "OTEL_LOG_MAX_QUEUE_SIZE",
+        value: env.DEV_OTEL_LOG_MAX_QUEUE_SIZE,
+      },
+    ]);
+  }
+
+  const commonVariables = await resolveCommonBuiltInVariables(runtimeEnvironment);
+
+  return [...result, ...commonVariables];
+}
+
+async function resolveBuiltInProdVariables(runtimeEnvironment: RuntimeEnvironment) {
+  let result: Array<EnvironmentVariable> = [
+    {
+      key: "TRIGGER_SECRET_KEY",
+      value: runtimeEnvironment.apiKey,
+    },
+    {
+      key: "TRIGGER_API_URL",
+      value: env.APP_ORIGIN,
+    },
+    {
+      key: "TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS",
+      value: String(env.RUNTIME_WAIT_THRESHOLD_IN_MS),
+    },
+    {
+      key: "TRIGGER_ORG_ID",
+      value: runtimeEnvironment.organizationId,
+    },
+  ];
+
+  if (env.PROD_OTEL_BATCH_PROCESSING_ENABLED === "1") {
+    result = result.concat([
+      {
+        key: "OTEL_BATCH_PROCESSING_ENABLED",
+        value: "1",
+      },
+      {
+        key: "OTEL_SPAN_MAX_EXPORT_BATCH_SIZE",
+        value: env.PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "OTEL_SPAN_SCHEDULED_DELAY_MILLIS",
+        value: env.PROD_OTEL_SPAN_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "OTEL_SPAN_EXPORT_TIMEOUT_MILLIS",
+        value: env.PROD_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "OTEL_SPAN_MAX_QUEUE_SIZE",
+        value: env.PROD_OTEL_SPAN_MAX_QUEUE_SIZE,
+      },
+      {
+        key: "OTEL_LOG_MAX_EXPORT_BATCH_SIZE",
+        value: env.PROD_OTEL_LOG_MAX_EXPORT_BATCH_SIZE,
+      },
+      {
+        key: "OTEL_LOG_SCHEDULED_DELAY_MILLIS",
+        value: env.PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS,
+      },
+      {
+        key: "OTEL_LOG_EXPORT_TIMEOUT_MILLIS",
+        value: env.PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS,
+      },
+      {
+        key: "OTEL_LOG_MAX_QUEUE_SIZE",
+        value: env.PROD_OTEL_LOG_MAX_QUEUE_SIZE,
+      },
+    ]);
+  }
+
+  if (env.PROD_USAGE_HEARTBEAT_INTERVAL_MS && env.USAGE_EVENT_URL) {
+    result = result.concat([
+      {
+        key: "USAGE_HEARTBEAT_INTERVAL_MS",
+        value: String(env.PROD_USAGE_HEARTBEAT_INTERVAL_MS),
+      },
+      {
+        key: "USAGE_EVENT_URL",
+        value: env.USAGE_EVENT_URL,
+      },
+    ]);
+  }
+
+  const commonVariables = await resolveCommonBuiltInVariables(runtimeEnvironment);
+
+  return [...result, ...commonVariables];
+}
+
+async function resolveCommonBuiltInVariables(
+  runtimeEnvironment: RuntimeEnvironment
+): Promise<Array<EnvironmentVariable>> {
+  return [];
+}

apps/webapp/app/v3/environmentVariables/repository.ts

@@ -76,16 +76,8 @@ export interface Repository {
   create(projectId: string, options: CreateEnvironmentVariables): Promise<CreateResult>;
   edit(projectId: string, options: EditEnvironmentVariable): Promise<Result>;
   getProject(projectId: string): Promise<ProjectEnvironmentVariable[]>;
-  getEnvironment(
-    projectId: string,
-    environmentId: string,
-    excludeInternalVariables?: boolean
-  ): Promise<EnvironmentVariable[]>;
-  getEnvironmentVariables(
-    projectId: string,
-    environmentId: string,
-    excludeInternalVariables?: boolean
-  ): Promise<EnvironmentVariable[]>;
+  getEnvironment(projectId: string, environmentId: string): Promise<EnvironmentVariable[]>;
+  getEnvironmentVariables(projectId: string, environmentId: string): Promise<EnvironmentVariable[]>;
   delete(projectId: string, options: DeleteEnvironmentVariable): Promise<Result>;
   deleteValue(projectId: string, options: DeleteEnvironmentVariableValue): Promise<Result>;
 }

apps/webapp/app/v3/eventRepository.server.ts

@@ -4,6 +4,7 @@ import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions"
 import {
   ExceptionEventProperties,
   ExceptionSpanEvent,
+  NULL_SENTINEL,
   PRIMARY_VARIANT,
   SemanticInternalAttributes,
   SpanEvent,
@@ -14,7 +15,6 @@ import {
   correctErrorStackTrace,
   createPacketAttributesAsJson,
   flattenAttributes,
-  NULL_SENTINEL,
   isExceptionSpanEvent,
   omit,
   unflattenAttributes,
@@ -23,14 +23,15 @@ import { Prisma, TaskEvent, TaskEventStatus, type TaskEventKind } from "@trigger
 import Redis, { RedisOptions } from "ioredis";
 import { createHash } from "node:crypto";
 import { EventEmitter } from "node:stream";
+import { Gauge } from "prom-client";
 import { $replica, PrismaClient, PrismaReplicaClient, prisma } from "~/db.server";
 import { env } from "~/env.server";
+import { metricsRegister } from "~/metrics.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
 import { singleton } from "~/utils/singleton";
-import { Gauge } from "prom-client";
-import { metricsRegister } from "~/metrics.server";
+import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
+import { startActiveSpan } from "./tracer.server";
 
 export type CreatableEvent = Omit<
   Prisma.TaskEventCreateInput,
@@ -128,6 +129,10 @@ export type PreparedEvent = Omit<QueriedEvent, "events" | "style" | "duration">
   style: TaskEventStyle;
 };
 
+export type RunPreparedEvent = PreparedEvent & {
+  taskSlug?: string;
+};
+
 export type SpanLink =
   | {
       type: "run";
@@ -374,43 +379,261 @@ export class EventRepository {
   }
 
   public async getTraceSummary(traceId: string): Promise<TraceSummary | undefined> {
-    const events = await this.readReplica.taskEvent.findMany({
-      select: {
-        id: true,
-        spanId: true,
-        parentId: true,
-        runId: true,
-        idempotencyKey: true,
-        message: true,
-        style: true,
-        startTime: true,
-        duration: true,
-        isError: true,
-        isPartial: true,
-        isCancelled: true,
-        level: true,
-        events: true,
-        environmentType: true,
-      },
-      where: {
-        traceId,
-      },
-      orderBy: {
-        startTime: "asc",
-      },
+    return await startActiveSpan("getTraceSummary", async (span) => {
+      const events = await this.readReplica.taskEvent.findMany({
+        select: {
+          id: true,
+          spanId: true,
+          parentId: true,
+          runId: true,
+          idempotencyKey: true,
+          message: true,
+          style: true,
+          startTime: true,
+          duration: true,
+          isError: true,
+          isPartial: true,
+          isCancelled: true,
+          level: true,
+          events: true,
+          environmentType: true,
+        },
+        where: {
+          traceId,
+        },
+        orderBy: {
+          startTime: "asc",
+        },
+        take: env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT,
+      });
+
+      let preparedEvents: Array<PreparedEvent> = [];
+      let rootSpanId: string | undefined;
+      const eventsBySpanId = new Map<string, PreparedEvent>();
+
+      for (const event of events) {
+        preparedEvents.push(prepareEvent(event));
+
+        if (!rootSpanId && !event.parentId) {
+          rootSpanId = event.spanId;
+        }
+      }
+
+      for (const event of preparedEvents) {
+        const existingEvent = eventsBySpanId.get(event.spanId);
+
+        if (!existingEvent) {
+          eventsBySpanId.set(event.spanId, event);
+          continue;
+        }
+
+        if (event.isCancelled || !event.isPartial) {
+          eventsBySpanId.set(event.spanId, event);
+        }
+      }
+
+      preparedEvents = Array.from(eventsBySpanId.values());
+
+      const spansBySpanId = new Map<string, SpanSummary>();
+
+      const spans = preparedEvents.map((event) => {
+        const ancestorCancelled = isAncestorCancelled(eventsBySpanId, event.spanId);
+        const duration = calculateDurationIfAncestorIsCancelled(
+          eventsBySpanId,
+          event.spanId,
+          event.duration
+        );
+
+        const span = {
+          recordId: event.id,
+          id: event.spanId,
+          parentId: event.parentId ?? undefined,
+          runId: event.runId,
+          idempotencyKey: event.idempotencyKey,
+          data: {
+            message: event.message,
+            style: event.style,
+            duration,
+            isError: event.isError,
+            isPartial: ancestorCancelled ? false : event.isPartial,
+            isCancelled: event.isCancelled === true ? true : event.isPartial && ancestorCancelled,
+            startTime: getDateFromNanoseconds(event.startTime),
+            level: event.level,
+            events: event.events,
+            environmentType: event.environmentType,
+          },
+        };
+
+        spansBySpanId.set(event.spanId, span);
+
+        return span;
+      });
+
+      if (!rootSpanId) {
+        return;
+      }
+
+      const rootSpan = spansBySpanId.get(rootSpanId);
+
+      if (!rootSpan) {
+        return;
+      }
+
+      return {
+        rootSpan,
+        spans,
+      };
     });
+  }
 
-    const preparedEvents = removeDuplicateEvents(events.map(prepareEvent));
+  public async getRunEvents(runId: string): Promise<RunPreparedEvent[]> {
+    return await startActiveSpan("getRunEvents", async (span) => {
+      const events = await this.readReplica.taskEvent.findMany({
+        select: {
+          id: true,
+          spanId: true,
+          parentId: true,
+          runId: true,
+          idempotencyKey: true,
+          message: true,
+          style: true,
+          startTime: true,
+          duration: true,
+          isError: true,
+          isPartial: true,
+          isCancelled: true,
+          level: true,
+          events: true,
+          environmentType: true,
+          taskSlug: true,
+        },
+        where: {
+          runId,
+          isPartial: false,
+        },
+        orderBy: {
+          startTime: "asc",
+        },
+      });
 
-    const spans = preparedEvents.map((event) => {
-      const ancestorCancelled = isAncestorCancelled(preparedEvents, event.spanId);
-      const duration = calculateDurationIfAncestorIsCancelled(
-        preparedEvents,
-        event.spanId,
-        event.duration
+      let preparedEvents: Array<PreparedEvent> = [];
+
+      for (const event of events) {
+        preparedEvents.push(prepareEvent(event));
+      }
+
+      return preparedEvents;
+    });
+  }
+
+  // A Span can be cancelled if it is partial and has a parent that is cancelled
+  // And a span's duration, if it is partial and has a cancelled parent, is the time between the start of the span and the time of the cancellation event of the parent
+  public async getSpan(spanId: string, traceId: string) {
+    return await startActiveSpan("getSpan", async (s) => {
+      const spanEvent = await this.#getSpanEvent(spanId);
+
+      if (!spanEvent) {
+        return;
+      }
+
+      const preparedEvent = prepareEvent(spanEvent);
+
+      const span = await this.#createSpanFromEvent(preparedEvent);
+
+      const output = rehydrateJson(spanEvent.output);
+      const payload = rehydrateJson(spanEvent.payload);
+
+      const show = rehydrateShow(spanEvent.properties);
+
+      const properties = sanitizedAttributes(spanEvent.properties);
+
+      const messagingEvent = SpanMessagingEvent.optional().safeParse(
+        (properties as any)?.messaging
+      );
+
+      const links: SpanLink[] = [];
+
+      if (messagingEvent.success && messagingEvent.data) {
+        if (messagingEvent.data.message && "id" in messagingEvent.data.message) {
+          if (messagingEvent.data.message.id.startsWith("run_")) {
+            links.push({
+              type: "run",
+              icon: "runs",
+              title: `Run ${messagingEvent.data.message.id}`,
+              runId: messagingEvent.data.message.id,
+            });
+          }
+        }
+      }
+
+      const backLinks = spanEvent.links as any as Link[] | undefined;
+
+      if (backLinks && backLinks.length > 0) {
+        backLinks.forEach((l) => {
+          const title = String(
+            l.attributes?.[SemanticInternalAttributes.LINK_TITLE] ?? "Triggered by"
+          );
+
+          links.push({
+            type: "span",
+            icon: "trigger",
+            title,
+            traceId: l.context.traceId,
+            spanId: l.context.spanId,
+          });
+        });
+      }
+
+      const spanEvents = transformEvents(
+        preparedEvent.events,
+        spanEvent.metadata as Attributes,
+        spanEvent.environmentType === "DEVELOPMENT"
       );
 
       return {
+        ...spanEvent,
+        ...span.data,
+        payload,
+        output,
+        properties,
+        events: spanEvents,
+        show,
+        links,
+      };
+    });
+  }
+
+  async #createSpanFromEvent(event: PreparedEvent) {
+    return await startActiveSpan("createSpanFromEvent", async (s) => {
+      let ancestorCancelled = false;
+      let duration = event.duration;
+
+      if (!event.isCancelled && event.isPartial) {
+        await this.#walkSpanAncestors(event, (ancestorEvent, level) => {
+          if (level >= 8) {
+            return { stop: true };
+          }
+
+          if (ancestorEvent.isCancelled) {
+            ancestorCancelled = true;
+
+            // We need to get the cancellation time from the cancellation span event
+            const cancellationEvent = ancestorEvent.events.find(
+              (event) => event.name === "cancellation"
+            );
+
+            if (cancellationEvent) {
+              duration = calculateDurationFromStart(event.startTime, cancellationEvent.time);
+            }
+
+            return { stop: true };
+          }
+
+          return { stop: false };
+        });
+      }
+
+      const span = {
         recordId: event.id,
         id: event.spanId,
         parentId: event.parentId ?? undefined,
@@ -429,104 +652,93 @@ export class EventRepository {
           environmentType: event.environmentType,
         },
       };
+
+      return span;
     });
-
-    const rootSpanId = events.find((event) => !event.parentId);
-    if (!rootSpanId) {
-      return;
-    }
-
-    const rootSpan = spans.find((span) => span.id === rootSpanId.spanId);
-
-    if (!rootSpan) {
-      return;
-    }
-
-    return {
-      rootSpan,
-      spans,
-    };
   }
 
-  // A Span can be cancelled if it is partial and has a parent that is cancelled
-  // And a span's duration, if it is partial and has a cancelled parent, is the time between the start of the span and the time of the cancellation event of the parent
-  public async getSpan(spanId: string, traceId: string) {
-    const traceSummary = await this.getTraceSummary(traceId);
-
-    const span = traceSummary?.spans.find((span) => span.id === spanId);
-
-    if (!span) {
+  async #walkSpanAncestors(
+    event: PreparedEvent,
+    callback: (event: PreparedEvent, level: number) => { stop: boolean }
+  ) {
+    const parentId = event.parentId;
+    if (!parentId) {
       return;
     }
 
-    const fullEvent = await this.readReplica.taskEvent.findUnique({
-      where: {
-        id: span.recordId,
-      },
-    });
+    await startActiveSpan("walkSpanAncestors", async (s) => {
+      let parentEvent = await this.#getSpanEvent(parentId);
+      let level = 1;
 
-    if (!fullEvent) {
-      return;
-    }
+      while (parentEvent) {
+        const preparedParentEvent = prepareEvent(parentEvent);
 
-    const output = rehydrateJson(fullEvent.output);
-    const payload = rehydrateJson(fullEvent.payload);
+        const result = callback(preparedParentEvent, level);
 
-    const show = rehydrateShow(fullEvent.properties);
-
-    const properties = sanitizedAttributes(fullEvent.properties);
-
-    const messagingEvent = SpanMessagingEvent.optional().safeParse((properties as any)?.messaging);
-
-    const links: SpanLink[] = [];
-
-    if (messagingEvent.success && messagingEvent.data) {
-      if (messagingEvent.data.message && "id" in messagingEvent.data.message) {
-        if (messagingEvent.data.message.id.startsWith("run_")) {
-          links.push({
-            type: "run",
-            icon: "runs",
-            title: `Run ${messagingEvent.data.message.id}`,
-            runId: messagingEvent.data.message.id,
-          });
+        if (result.stop) {
+          return;
         }
+
+        if (!preparedParentEvent.parentId) {
+          return;
+        }
+
+        parentEvent = await this.#getSpanEvent(preparedParentEvent.parentId);
+
+        level++;
       }
+    });
+  }
+
+  async #getSpanAncestors(event: PreparedEvent, levels = 1): Promise<Array<PreparedEvent>> {
+    if (levels >= 8) {
+      return [];
     }
 
-    const backLinks = fullEvent.links as any as Link[] | undefined;
+    if (!event.parentId) {
+      return [];
+    }
 
-    if (backLinks && backLinks.length > 0) {
-      backLinks.forEach((l) => {
-        const title = String(
-          l.attributes?.[SemanticInternalAttributes.LINK_TITLE] ?? "Triggered by"
-        );
+    const parentEvent = await this.#getSpanEvent(event.parentId);
 
-        links.push({
-          type: "span",
-          icon: "trigger",
-          title,
-          traceId: l.context.traceId,
-          spanId: l.context.spanId,
-        });
+    if (!parentEvent) {
+      return [];
+    }
+
+    const preparedParentEvent = prepareEvent(parentEvent);
+
+    if (!preparedParentEvent.parentId) {
+      return [preparedParentEvent];
+    }
+
+    const moreAncestors = await this.#getSpanAncestors(preparedParentEvent, levels + 1);
+
+    return [preparedParentEvent, ...moreAncestors];
+  }
+
+  async #getSpanEvent(spanId: string) {
+    return await startActiveSpan("getSpanEvent", async (s) => {
+      const events = await this.readReplica.taskEvent.findMany({
+        where: {
+          spanId,
+        },
+        orderBy: {
+          startTime: "asc",
+        },
       });
-    }
 
-    const events = transformEvents(
-      span.data.events,
-      fullEvent.metadata as Attributes,
-      traceSummary?.rootSpan.data.environmentType === "DEVELOPMENT"
-    );
+      let finalEvent: TaskEvent | undefined;
 
-    return {
-      ...fullEvent,
-      ...span.data,
-      payload,
-      output,
-      properties,
-      events,
-      show,
-      links,
-    };
+      for (const event of events) {
+        if (event.isPartial && finalEvent) {
+          continue;
+        }
+
+        finalEvent = event;
+      }
+
+      return finalEvent;
+    });
   }
 
   public async recordEvent(message: string, options: TraceEventOptions) {
@@ -973,34 +1185,38 @@ function prepareEvent(event: QueriedEvent): PreparedEvent {
 }
 
 function parseEventsField(events: Prisma.JsonValue): SpanEvents {
-  const eventsUnflattened = events
+  const unsafe = events
     ? (events as any[]).map((e) => ({
         ...e,
         properties: unflattenAttributes(e.properties as Attributes),
       }))
     : undefined;
 
-  const spanEvents = SpanEvents.safeParse(eventsUnflattened);
-
-  if (spanEvents.success) {
-    return spanEvents.data;
-  }
-
-  return [];
+  return unsafe as SpanEvents;
 }
 
 function parseStyleField(style: Prisma.JsonValue): TaskEventStyle {
-  const parsedStyle = TaskEventStyle.safeParse(unflattenAttributes(style as Attributes));
+  const unsafe = unflattenAttributes(style as Attributes);
 
-  if (parsedStyle.success) {
-    return parsedStyle.data;
+  if (!unsafe) {
+    return {};
+  }
+
+  if (typeof unsafe === "object") {
+    return Object.assign(
+      {
+        icon: undefined,
+        variant: undefined,
+      },
+      unsafe
+    ) as TaskEventStyle;
   }
 
   return {};
 }
 
-function isAncestorCancelled(events: PreparedEvent[], spanId: string) {
-  const event = events.find((event) => event.spanId === spanId);
+function isAncestorCancelled(events: Map<string, PreparedEvent>, spanId: string) {
+  const event = events.get(spanId);
 
   if (!event) {
     return false;
@@ -1018,11 +1234,11 @@ function isAncestorCancelled(events: PreparedEvent[], spanId: string) {
 }
 
 function calculateDurationIfAncestorIsCancelled(
-  events: PreparedEvent[],
+  events: Map<string, PreparedEvent>,
   spanId: string,
   defaultDuration: number
 ) {
-  const event = events.find((event) => event.spanId === spanId);
+  const event = events.get(spanId);
 
   if (!event) {
     return defaultDuration;
@@ -1054,8 +1270,9 @@ function calculateDurationIfAncestorIsCancelled(
   return defaultDuration;
 }
 
-function findFirstCancelledAncestor(events: PreparedEvent[], spanId: string) {
-  const event = events.find((event) => event.spanId === spanId);
+function findFirstCancelledAncestor(events: Map<string, PreparedEvent>, spanId: string) {
+  const event = events.get(spanId);
+
   if (!event) {
     return;
   }
@@ -1196,7 +1413,7 @@ function getNowInNanoseconds(): bigint {
   return BigInt(new Date().getTime() * 1_000_000);
 }
 
-function getDateFromNanoseconds(nanoseconds: bigint) {
+export function getDateFromNanoseconds(nanoseconds: bigint) {
   return new Date(Number(nanoseconds) / 1_000_000);
 }
 

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -138,8 +138,19 @@ function createCoordinatorNamespace(io: Server) {
         await sharedQueueTasks.taskRunHeartbeat(message.runId);
       },
       CHECKPOINT_CREATED: async (message) => {
-        const createCheckpoint = new CreateCheckpointService();
-        await createCheckpoint.call(message);
+        try {
+          const createCheckpoint = new CreateCheckpointService();
+          const result = await createCheckpoint.call(message);
+
+          return { keepRunAlive: result?.keepRunAlive ?? false };
+        } catch (error) {
+          logger.error("Error while creating checkpoint", {
+            rawMessage: message,
+            error: error instanceof Error ? error.message : error,
+          });
+
+          return { keepRunAlive: false };
+        }
       },
       CREATE_WORKER: async (message) => {
         try {

apps/webapp/app/v3/machinePresets.server.ts

@@ -0,0 +1,81 @@
+import { MachineConfig, MachinePreset, MachinePresetName } from "@trigger.dev/core/v3";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+
+export const presets = {
+  micro: {
+    cpu: 0.25,
+    memory: 0.25,
+    centsPerMs: env.CENTS_PER_HOUR_MICRO / 3_600_000,
+  },
+  "small-1x": {
+    cpu: 0.5,
+    memory: 0.5,
+    centsPerMs: env.CENTS_PER_HOUR_SMALL_1X / 3_600_000,
+  },
+  "small-2x": {
+    cpu: 1,
+    memory: 1,
+    centsPerMs: env.CENTS_PER_HOUR_SMALL_2X / 3_600_000,
+  },
+  "medium-1x": {
+    cpu: 1,
+    memory: 2,
+    centsPerMs: env.CENTS_PER_HOUR_MEDIUM_1X / 3_600_000,
+  },
+  "medium-2x": {
+    cpu: 2,
+    memory: 4,
+    centsPerMs: env.CENTS_PER_HOUR_MEDIUM_2X / 3_600_000,
+  },
+  "large-1x": {
+    cpu: 4,
+    memory: 8,
+    centsPerMs: env.CENTS_PER_HOUR_LARGE_1X / 3_600_000,
+  },
+  "large-2x": {
+    cpu: 8,
+    memory: 16,
+    centsPerMs: env.CENTS_PER_HOUR_LARGE_2X / 3_600_000,
+  },
+};
+
+export function machinePresetFromConfig(config: unknown): MachinePreset {
+  const parsedConfig = MachineConfig.safeParse(config);
+
+  if (!parsedConfig.success) {
+    logger.error("Failed to parse machine config", { config });
+
+    return machinePresetFromName("small-1x");
+  }
+
+  if (parsedConfig.data.preset) {
+    return machinePresetFromName(parsedConfig.data.preset);
+  }
+
+  if (parsedConfig.data.cpu && parsedConfig.data.memory) {
+    const name = derivePresetNameFromValues(parsedConfig.data.cpu, parsedConfig.data.memory);
+
+    return machinePresetFromName(name);
+  }
+
+  return machinePresetFromName("small-1x");
+}
+
+export function machinePresetFromName(name: MachinePresetName): MachinePreset {
+  return {
+    name,
+    ...presets[name],
+  };
+}
+
+// Finds the smallest machine preset name that satisfies the given CPU and memory requirements
+function derivePresetNameFromValues(cpu: number, memory: number): MachinePresetName {
+  for (const [name, preset] of Object.entries(presets)) {
+    if (preset.cpu >= cpu && preset.memory >= memory) {
+      return name as MachinePresetName;
+    }
+  }
+
+  return "small-1x";
+}

apps/webapp/app/v3/marqs/devQueueConsumer.server.ts

@@ -15,7 +15,8 @@ import { createNewSession, disconnectSession } from "~/models/runtimeEnvironment
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
-import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
+import { resolveVariablesForEnvironment } from "../environmentVariables/environmentVariablesRepository.server";
+import { FailedTaskRunService } from "../failedTaskRun.server";
 import { CancelTaskRunService } from "../services/cancelTaskRun.server";
 import { CompleteAttemptService } from "../services/completeAttempt.server";
 import { CreateTaskRunAttemptService } from "../services/createTaskRunAttempt.server";
@@ -25,7 +26,6 @@ import {
   tracer,
 } from "../tracer.server";
 import { DevSubscriber, devPubSub } from "./devPubSub.server";
-import { FailedTaskRunService } from "../failedTaskRun.server";
 
 const MessageBody = z.discriminatedUnion("type", [
   z.object({
@@ -415,6 +415,7 @@ export class DevQueueConsumer {
         lockedById: backgroundTask.id,
         status: "EXECUTING",
         lockedToVersionId: backgroundWorker.id,
+        startedAt: existingTaskRun.startedAt ?? new Date(),
       },
       include: {
         attempts: {
@@ -473,11 +474,7 @@ export class DevQueueConsumer {
       return;
     }
 
-    const environmentRepository = new EnvironmentVariablesRepository();
-    const variables = await environmentRepository.getEnvironmentVariables(
-      this.env.project.id,
-      this.env.id
-    );
+    const variables = await resolveVariablesForEnvironment(this.env);
 
     if (backgroundWorker.supportsLazyAttempts) {
       const payload: TaskRunExecutionLazyAttemptPayload = {
@@ -524,6 +521,7 @@ export class DevQueueConsumer {
               lockedAt: null,
               lockedById: null,
               status: "PENDING",
+              startedAt: existingTaskRun.startedAt,
             },
           }),
         ]);
@@ -581,6 +579,7 @@ export class DevQueueConsumer {
               lockedAt: null,
               lockedById: null,
               status: "PENDING",
+              startedAt: existingTaskRun.startedAt,
             },
           }),
         ]);

apps/webapp/app/v3/marqs/index.server.ts

@@ -1625,7 +1625,7 @@ function getMarQSClient() {
         defaultEnvConcurrency: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT,
         defaultOrgConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT,
         visibilityTimeoutInMs: 120 * 1000, // 2 minutes,
-        enableRebalancing: !env.MARQS_DISABLE_REBALANCING,
+        enableRebalancing: false,
       });
     } else {
       console.warn(

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -1,6 +1,6 @@
 import { Context, ROOT_CONTEXT, Span, SpanKind, context, trace } from "@opentelemetry/api";
 import {
-  Machine,
+  MachinePreset,
   ProdTaskRunExecution,
   ProdTaskRunExecutionPayload,
   TaskRunError,
@@ -15,15 +15,18 @@ import { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
 import {
   BackgroundWorker,
   BackgroundWorkerTask,
-  TaskRunAttemptStatus,
+  RuntimeEnvironment,
+  TaskRun,
   TaskRunStatus,
 } from "@trigger.dev/database";
 import { z } from "zod";
 import { prisma } from "~/db.server";
+import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
 import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
-import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
+import { resolveVariablesForEnvironment } from "../environmentVariables/environmentVariablesRepository.server";
+import { FailedTaskRunService } from "../failedTaskRun.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { socketIo } from "../handleSocketIo.server";
 import {
@@ -31,12 +34,15 @@ import {
   getWorkerDeploymentFromWorker,
   getWorkerDeploymentFromWorkerTask,
 } from "../models/workerDeployment.server";
-import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
-import { SEMINTATTRS_FORCE_RECORDING, tracer } from "../tracer.server";
 import { CrashTaskRunService } from "../services/crashTaskRun.server";
-import { FailedTaskRunService } from "../failedTaskRun.server";
 import { CreateTaskRunAttemptService } from "../services/createTaskRunAttempt.server";
-import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
+import { tracer } from "../tracer.server";
+import { generateJWTTokenForEnvironment } from "~/services/apiAuth.server";
+import { EnvironmentVariable } from "../environmentVariables/repository";
+import { machinePresetFromConfig } from "../machinePresets.server";
+import { env } from "~/env.server";
+import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
 
 const WithTraceContext = z.object({
   traceparent: z.string().optional(),
@@ -405,6 +411,9 @@ export class SharedQueueConsumer {
             lockedAt: new Date(),
             lockedById: backgroundTask.id,
             lockedToVersionId: deployment.worker.id,
+            startedAt: existingTaskRun.startedAt ?? new Date(),
+            baseCostInCents: env.BASE_RUN_COST_IN_CENTS,
+            machinePreset: machinePresetFromConfig(backgroundTask.machineConfig ?? {}).name,
           },
           include: {
             runtimeEnvironment: true,
@@ -505,18 +514,7 @@ export class SharedQueueConsumer {
             });
           } else {
             const machineConfig = lockedTaskRun.lockedBy?.machineConfig;
-            const machine = Machine.safeParse(machineConfig ?? {});
-
-            if (!machine.success) {
-              logger.error("Failed to parse machine config", {
-                queueMessage: message.data,
-                messageId: message.messageId,
-                machineConfig,
-              });
-
-              await this.#ackAndDoMoreWork(message.messageId);
-              return;
-            }
+            const machine = machinePresetFromConfig(machineConfig ?? {});
 
             await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
               backgroundWorkerId: deployment.worker.friendlyId,
@@ -524,7 +522,7 @@ export class SharedQueueConsumer {
                 type: "SCHEDULE_ATTEMPT",
                 image: deployment.imageReference,
                 version: deployment.version,
-                machine: machine.data,
+                machine,
                 // identifiers
                 id: "placeholder", // TODO: Remove this completely in a future release
                 envId: lockedTaskRun.runtimeEnvironment.id,
@@ -554,6 +552,7 @@ export class SharedQueueConsumer {
                 lockedAt: null,
                 lockedById: null,
                 status: lockedTaskRun.status,
+                startedAt: existingTaskRun.startedAt,
               },
             }),
           ]);
@@ -963,19 +962,7 @@ class SharedQueueTasks {
     }
 
     if (setToExecuting) {
-      const FINAL_RUN_STATUSES: TaskRunStatus[] = [
-        "CANCELED",
-        "COMPLETED_SUCCESSFULLY",
-        "COMPLETED_WITH_ERRORS",
-        "INTERRUPTED",
-        "SYSTEM_FAILURE",
-      ];
-      const FINAL_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["CANCELED", "COMPLETED", "FAILED"];
-
-      if (
-        FINAL_ATTEMPT_STATUSES.includes(attempt.status) ||
-        FINAL_RUN_STATUSES.includes(attempt.taskRun.status)
-      ) {
+      if (isFinalAttemptStatus(attempt.status) || isFinalRunStatus(attempt.taskRun.status)) {
         logger.error("Status already in final state", {
           attempt: {
             id: attempt.id,
@@ -1008,6 +995,8 @@ class SharedQueueTasks {
 
     const { backgroundWorkerTask, taskRun, queue } = attempt;
 
+    const machinePreset = machinePresetFromConfig(backgroundWorkerTask.machineConfig ?? {});
+
     const execution: ProdTaskRunExecution = {
       task: {
         id: backgroundWorkerTask.slug,
@@ -1028,9 +1017,13 @@ class SharedQueueTasks {
         payloadType: taskRun.payloadType,
         context: taskRun.context,
         createdAt: taskRun.createdAt,
+        startedAt: taskRun.startedAt ?? taskRun.createdAt,
         tags: taskRun.tags.map((tag) => tag.name),
         isTest: taskRun.isTest,
         idempotencyKey: taskRun.idempotencyKey ?? undefined,
+        durationMs: taskRun.usageDurationMs,
+        costInCents: taskRun.costInCents,
+        baseCostInCents: taskRun.baseCostInCents,
       },
       queue: {
         id: queue.friendlyId,
@@ -1061,12 +1054,13 @@ class SharedQueueTasks {
         contentHash: attempt.backgroundWorker.contentHash,
         version: attempt.backgroundWorker.version,
       },
+      machine: machinePreset,
     };
 
-    const environmentRepository = new EnvironmentVariablesRepository();
-    const variables = await environmentRepository.getEnvironmentVariables(
-      attempt.runtimeEnvironment.projectId,
-      attempt.runtimeEnvironmentId
+    const variables = await this.#buildEnvironmentVariables(
+      attempt.runtimeEnvironment,
+      taskRun,
+      machinePreset
     );
 
     const payload: ProdTaskRunExecutionPayload = {
@@ -1126,6 +1120,9 @@ class SharedQueueTasks {
         id: runId,
         runtimeEnvironmentId: environment.id,
       },
+      include: {
+        lockedBy: true,
+      },
     });
 
     if (!run) {
@@ -1133,11 +1130,9 @@ class SharedQueueTasks {
       return;
     }
 
-    const environmentRepository = new EnvironmentVariablesRepository();
-    const variables = await environmentRepository.getEnvironmentVariables(
-      environment.projectId,
-      environment.id
-    );
+    const machinePreset = machinePresetFromConfig(run.lockedBy?.machineConfig ?? {});
+
+    const variables = await this.#buildEnvironmentVariables(environment, run, machinePreset);
 
     return {
       traceContext: run.traceContext as Record<string, unknown>,
@@ -1178,6 +1173,31 @@ class SharedQueueTasks {
 
     await service.call(completion.id, completion);
   }
+
+  async #buildEnvironmentVariables(
+    environment: RuntimeEnvironment,
+    run: TaskRun,
+    machinePreset: MachinePreset
+  ): Promise<Array<EnvironmentVariable>> {
+    const variables = await resolveVariablesForEnvironment(environment);
+
+    const jwt = await generateJWTTokenForEnvironment(environment, {
+      run_id: run.id,
+      machine_preset: machinePreset.name,
+    });
+
+    return [
+      ...variables,
+      ...[
+        { key: "TRIGGER_JWT", value: jwt },
+        { key: "TRIGGER_RUN_ID", value: run.id },
+        {
+          key: "TRIGGER_MACHINE_PRESET",
+          value: machinePreset.name,
+        },
+      ],
+    ];
+  }
 }
 
 export const sharedQueueTasks = singleton("sharedQueueTasks", () => new SharedQueueTasks());

apps/webapp/app/v3/marqs/v2.server.ts

@@ -82,7 +82,7 @@ function getMarQSClient() {
     defaultEnvConcurrency: env.V2_MARQS_DEFAULT_ENV_CONCURRENCY, // this is so we aren't limited by the environment concurrency
     defaultOrgConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT,
     visibilityTimeoutInMs: env.V2_MARQS_VISIBILITY_TIMEOUT_MS, // 15 minutes
-    enableRebalancing: env.V2_MARQS_CONSUMER_POOL_ENABLED === "1",
+    enableRebalancing: false,
   });
 }
 

apps/webapp/app/v3/openMeter.server.ts

@@ -0,0 +1,47 @@
+import { randomUUID } from "node:crypto";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+
+export type UsageEvent = {
+  source: string;
+  subject: string;
+  type: string;
+  id?: string;
+  time?: Date;
+  data?: Record<string, unknown>;
+};
+
+export async function reportUsageEvent(event: UsageEvent) {
+  if (!env.USAGE_OPEN_METER_BASE_URL || !env.USAGE_OPEN_METER_API_KEY) {
+    return;
+  }
+
+  const body = {
+    specversion: "1.0",
+    id: event.id ?? randomUUID(),
+    source: event.source,
+    type: event.type,
+    time: (event.time ?? new Date()).toISOString(),
+    subject: event.subject,
+    datacontenttype: "application/json",
+    data: event.data,
+  };
+
+  const url = `${env.USAGE_OPEN_METER_BASE_URL}/api/v1/events`;
+
+  logger.debug("Reporting usage event to OpenMeter", { url, body });
+
+  const response = await fetch(url, {
+    method: "POST",
+    body: JSON.stringify(body),
+    headers: {
+      "Content-Type": "application/cloudevents+json",
+      Authorization: `Bearer ${env.USAGE_OPEN_METER_API_KEY}`,
+      Accept: "application/json",
+    },
+  });
+
+  if (!response.ok) {
+    logger.error(`Failed to report usage event: ${response.status} ${response.statusText}`);
+  }
+}

apps/webapp/app/v3/otlpExporter.server.ts

@@ -307,6 +307,19 @@ function convertSpansToCreateableEvents(resourceSpan: ResourceSpans): Array<Crea
                 "."
               )
             ) ?? resourceProperties.attemptNumber,
+          usageDurationMs:
+            extractDoubleAttribute(
+              span.attributes ?? [],
+              SemanticInternalAttributes.USAGE_DURATION_MS
+            ) ??
+            extractNumberAttribute(
+              span.attributes ?? [],
+              SemanticInternalAttributes.USAGE_DURATION_MS
+            ),
+          usageCostInCents: extractDoubleAttribute(
+            span.attributes ?? [],
+            SemanticInternalAttributes.USAGE_COST_IN_CENTS
+          ),
         };
       })
       .filter(Boolean);
@@ -360,6 +373,20 @@ function extractResourceProperties(attributes: KeyValue[]) {
     queueName: extractStringAttribute(attributes, SemanticInternalAttributes.QUEUE_NAME),
     batchId: extractStringAttribute(attributes, SemanticInternalAttributes.BATCH_ID),
     idempotencyKey: extractStringAttribute(attributes, SemanticInternalAttributes.IDEMPOTENCY_KEY),
+    machinePreset: extractStringAttribute(
+      attributes,
+      SemanticInternalAttributes.MACHINE_PRESET_NAME
+    ),
+    machinePresetCpu:
+      extractDoubleAttribute(attributes, SemanticInternalAttributes.MACHINE_PRESET_CPU) ??
+      extractNumberAttribute(attributes, SemanticInternalAttributes.MACHINE_PRESET_CPU),
+    machinePresetMemory:
+      extractDoubleAttribute(attributes, SemanticInternalAttributes.MACHINE_PRESET_MEMORY) ??
+      extractNumberAttribute(attributes, SemanticInternalAttributes.MACHINE_PRESET_MEMORY),
+    machinePresetCentsPerMs: extractDoubleAttribute(
+      attributes,
+      SemanticInternalAttributes.MACHINE_PRESET_CENTS_PER_MS
+    ),
   };
 }
 
@@ -611,6 +638,20 @@ function extractNumberAttribute(
   return isIntValue(attribute?.value) ? Number(attribute.value.intValue) : fallback;
 }
 
+function extractDoubleAttribute(attributes: KeyValue[], name: string): number | undefined;
+function extractDoubleAttribute(attributes: KeyValue[], name: string, fallback: number): number;
+function extractDoubleAttribute(
+  attributes: KeyValue[],
+  name: string,
+  fallback?: number
+): number | undefined {
+  const attribute = attributes.find((attribute) => attribute.key === name);
+
+  if (!attribute) return fallback;
+
+  return isDoubleValue(attribute?.value) ? Number(attribute.value.doubleValue) : fallback;
+}
+
 function extractBooleanAttribute(attributes: KeyValue[], name: string): boolean | undefined;
 function extractBooleanAttribute(attributes: KeyValue[], name: string, fallback: boolean): boolean;
 function extractBooleanAttribute(

apps/webapp/app/v3/r2.server.ts

@@ -3,6 +3,7 @@ import { env } from "~/env.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
+import { startActiveSpan } from "./tracer.server";
 
 export const r2 = singleton("r2", initializeR2);
 
@@ -23,30 +24,87 @@ export async function uploadToObjectStore(
   contentType: string,
   environment: AuthenticatedEnvironment
 ): Promise<string> {
-  if (!r2) {
-    throw new Error("Object store credentials are not set");
+  return await startActiveSpan("uploadToObjectStore()", async (span) => {
+    if (!r2) {
+      throw new Error("Object store credentials are not set");
+    }
+
+    if (!env.OBJECT_STORE_BASE_URL) {
+      throw new Error("Object store base URL is not set");
+    }
+
+    span.setAttributes({
+      projectRef: environment.project.externalRef,
+      environmentSlug: environment.slug,
+      filename: filename,
+    });
+
+    const url = new URL(env.OBJECT_STORE_BASE_URL);
+    url.pathname = `/packets/${environment.project.externalRef}/${environment.slug}/${filename}`;
+
+    logger.debug("Uploading to object store", { url: url.href });
+
+    const response = await r2.fetch(url.toString(), {
+      method: "PUT",
+      headers: {
+        "Content-Type": contentType,
+      },
+      body: data,
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to upload output to ${url}: ${response.statusText}`);
+    }
+
+    return url.href;
+  });
+}
+
+export async function generatePresignedRequest(
+  projectRef: string,
+  envSlug: string,
+  filename: string,
+  method: "PUT" | "GET" = "PUT"
+) {
+  if (!env.OBJECT_STORE_BASE_URL) {
+    return;
   }
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    throw new Error("Object store base URL is not set");
+  if (!r2) {
+    return;
   }
 
   const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${environment.project.externalRef}/${environment.slug}/${filename}`;
+  url.pathname = `/packets/${projectRef}/${envSlug}/${filename}`;
+  url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
 
-  logger.debug("Uploading to object store", { url: url.href });
+  const signed = await r2.sign(
+    new Request(url, {
+      method,
+    }),
+    {
+      aws: { signQuery: true },
+    }
+  );
 
-  const response = await r2.fetch(url.toString(), {
-    method: "PUT",
-    headers: {
-      "Content-Type": contentType,
-    },
-    body: data,
+  logger.debug("Generated presigned URL", {
+    url: signed.url,
+    headers: Object.fromEntries(signed.headers),
+    projectRef,
+    envSlug,
+    filename,
   });
 
-  if (!response.ok) {
-    throw new Error(`Failed to upload output to ${url}: ${response.statusText}`);
-  }
-
-  return url.href;
+  return signed;
+}
+
+export async function generatePresignedUrl(
+  projectRef: string,
+  envSlug: string,
+  filename: string,
+  method: "PUT" | "GET" = "PUT"
+) {
+  const signed = await generatePresignedRequest(projectRef, envSlug, filename, method);
+
+  return signed?.url;
 }

apps/webapp/app/v3/schedules.ts

@@ -55,6 +55,7 @@ export const UpsertSchedule = z.object({
   ),
   externalId: z.string().optional(),
   deduplicationKey: z.string().optional(),
+  timezone: z.string().optional(),
 });
 
 export type UpsertSchedule = z.infer<typeof UpsertSchedule>;

apps/webapp/app/v3/services/cancelAttempt.server.ts

@@ -6,7 +6,7 @@ import { logger } from "~/services/logger.server";
 
 import { PrismaClientOrTransaction, prisma } from "~/db.server";
 import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
-import { CANCELLABLE_STATUSES } from "./cancelTaskRun.server";
+import { isCancellableRunStatus } from "../taskStatus";
 
 export class CancelAttemptService extends BaseService {
   public async call(
@@ -55,7 +55,7 @@ export class CancelAttemptService extends BaseService {
           taskRun: {
             update: {
               data: {
-                status: CANCELLABLE_STATUSES.includes(taskRunAttempt.taskRun.status)
+                status: isCancellableRunStatus(taskRunAttempt.taskRun.status)
                   ? "INTERRUPTED"
                   : undefined,
               },

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -1,4 +1,4 @@
-import { Prisma, TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
+import { Prisma, TaskRun } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { logger } from "~/services/logger.server";
 import { marqs } from "~/v3/marqs/index.server";
@@ -7,22 +7,7 @@ import { socketIo } from "../handleSocketIo.server";
 import { devPubSub } from "../marqs/devPubSub.server";
 import { BaseService } from "./baseService.server";
 import { CancelAttemptService } from "./cancelAttempt.server";
-
-export const CANCELLABLE_STATUSES: Array<TaskRunStatus> = [
-  "PENDING",
-  "WAITING_FOR_DEPLOY",
-  "EXECUTING",
-  "PAUSED",
-  "WAITING_TO_RESUME",
-  "PAUSED",
-  "RETRYING_AFTER_FAILURE",
-];
-
-const CANCELLABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = [
-  "EXECUTING",
-  "PAUSED",
-  "PENDING",
-];
+import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskStatus";
 
 type ExtendedTaskRun = Prisma.TaskRunGetPayload<{
   include: {
@@ -53,7 +38,11 @@ export class CancelTaskRunService extends BaseService {
     };
 
     // Make sure the task run is in a cancellable state
-    if (!CANCELLABLE_STATUSES.includes(taskRun.status)) {
+    if (!isCancellableRunStatus(taskRun.status)) {
+      logger.error("Task run is not in a cancellable state", {
+        runId: taskRun.id,
+        status: taskRun.status,
+      });
       return;
     }
 

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -88,6 +88,7 @@ export class CompleteAttemptService extends BaseService {
         completedAt: new Date(),
         output: completion.output,
         outputType: completion.outputType,
+        usageDurationMs: completion.usage?.durationMs,
         taskRun: {
           update: {
             data: {
@@ -138,6 +139,7 @@ export class CompleteAttemptService extends BaseService {
       // We need to cancel the task run instead of fail it
       const cancelService = new CancelAttemptService();
 
+      // TODO: handle usages
       await cancelService.call(
         taskRunAttempt.friendlyId,
         taskRunAttempt.taskRunId,
@@ -157,6 +159,7 @@ export class CompleteAttemptService extends BaseService {
         status: "FAILED",
         completedAt: new Date(),
         error: completion.error,
+        usageDurationMs: completion.usage?.durationMs,
       },
     });
 

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -1,27 +1,11 @@
-import {
-  TaskRun,
-  TaskRunAttempt,
-  TaskRunAttemptStatus,
-  TaskRunStatus,
-} from "@trigger.dev/database";
+import { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
 import { eventRepository } from "../eventRepository.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
-
-export const CRASHABLE_RUN_STATUSES: Array<TaskRunStatus> = [
-  "PENDING",
-  "WAITING_FOR_DEPLOY",
-  "EXECUTING",
-  "PAUSED",
-  "WAITING_TO_RESUME",
-  "PAUSED",
-  "RETRYING_AFTER_FAILURE",
-];
-
-const CRASHABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = ["EXECUTING", "PAUSED", "PENDING"];
+import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
 
 export type CrashTaskRunServiceOptions = {
   reason?: string;
@@ -52,7 +36,8 @@ export class CrashTaskRunService extends BaseService {
     }
 
     // Make sure the task run is in a crashable state
-    if (!CRASHABLE_RUN_STATUSES.includes(taskRun.status)) {
+    if (!isCrashableRunStatus(taskRun.status)) {
+      logger.error("Task run is not in a crashable state", { runId, status: taskRun.status });
       return;
     }
 

apps/webapp/app/v3/services/createCheckpoint.server.ts

@@ -1,19 +1,13 @@
 import { CoordinatorToPlatformMessages } from "@trigger.dev/core/v3";
 import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
-import type {
-  CheckpointRestoreEvent,
-  TaskRunAttemptStatus,
-  TaskRunStatus,
-} from "@trigger.dev/database";
+import type { Checkpoint, CheckpointRestoreEvent } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "~/v3/marqs/index.server";
 import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
 import { BaseService } from "./baseService.server";
 import { CrashTaskRunService } from "./crashTaskRun.server";
-
-const FREEZABLE_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "RETRYING_AFTER_FAILURE"];
-const FREEZABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["EXECUTING", "FAILED"];
+import { isFinalRunStatus, isFreezableAttemptStatus, isFreezableRunStatus } from "../taskStatus";
 
 export class CreateCheckpointService extends BaseService {
   public async call(
@@ -21,7 +15,14 @@ export class CreateCheckpointService extends BaseService {
       InferSocketMessageSchema<typeof CoordinatorToPlatformMessages, "CHECKPOINT_CREATED">,
       "version"
     >
-  ) {
+  ): Promise<
+    | {
+        checkpoint: Checkpoint;
+        event: CheckpointRestoreEvent;
+        keepRunAlive: boolean;
+      }
+    | undefined
+  > {
     logger.debug(`Creating checkpoint`, params);
 
     const attempt = await this._prisma.taskRunAttempt.findUnique({
@@ -49,8 +50,8 @@ export class CreateCheckpointService extends BaseService {
     }
 
     if (
-      !FREEZABLE_ATTEMPT_STATUSES.includes(attempt.status) ||
-      !FREEZABLE_RUN_STATUSES.includes(attempt.taskRun.status)
+      !isFreezableAttemptStatus(attempt.status) ||
+      !isFreezableRunStatus(attempt.taskRun.status)
     ) {
       logger.error("Unfreezable state", {
         attempt: {
@@ -115,7 +116,9 @@ export class CreateCheckpointService extends BaseService {
     });
 
     const { reason } = params;
+
     let checkpointEvent: CheckpointRestoreEvent | undefined;
+    let keepRunAlive = false;
 
     switch (reason.type) {
       case "WAIT_FOR_DURATION": {
@@ -131,7 +134,12 @@ export class CreateCheckpointService extends BaseService {
           dependencyFriendlyRunId: reason.friendlyId,
         });
 
-        await marqs?.acknowledgeMessage(attempt.taskRunId);
+        keepRunAlive = await this.#isRunCompleted(reason.friendlyId);
+
+        if (!keepRunAlive) {
+          await marqs?.acknowledgeMessage(attempt.taskRunId);
+        }
+
         break;
       }
       case "WAIT_FOR_BATCH": {
@@ -140,7 +148,12 @@ export class CreateCheckpointService extends BaseService {
           batchDependencyFriendlyId: reason.batchFriendlyId,
         });
 
-        await marqs?.acknowledgeMessage(attempt.taskRunId);
+        keepRunAlive = await this.#isBatchCompleted(reason.batchFriendlyId);
+
+        if (!keepRunAlive) {
+          await marqs?.acknowledgeMessage(attempt.taskRunId);
+        }
+
         break;
       }
       case "RETRYING_AFTER_FAILURE": {
@@ -180,6 +193,37 @@ export class CreateCheckpointService extends BaseService {
     return {
       checkpoint,
       event: checkpointEvent,
+      keepRunAlive,
     };
   }
+
+  async #isBatchCompleted(friendlyId: string): Promise<boolean> {
+    const batch = await this._prisma.batchTaskRun.findUnique({
+      where: {
+        friendlyId,
+      },
+    });
+
+    if (!batch) {
+      logger.error("Batch not found", { friendlyId });
+      return false;
+    }
+
+    return batch.status === "COMPLETED";
+  }
+
+  async #isRunCompleted(friendlyId: string): Promise<boolean> {
+    const run = await this._prisma.taskRun.findUnique({
+      where: {
+        friendlyId,
+      },
+    });
+
+    if (!run) {
+      logger.error("Run not found", { friendlyId });
+      return false;
+    }
+
+    return isFinalRunStatus(run.status);
+  }
 }

apps/webapp/app/v3/services/createTaskRunAttempt.server.ts

@@ -5,18 +5,21 @@ import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { BaseService, ServiceValidationError } from "./baseService.server";
 import { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
+import { machinePresetFromConfig } from "../machinePresets.server";
+import { workerQueue } from "~/services/worker.server";
 
 export class CreateTaskRunAttemptService extends BaseService {
   public async call(
     runId: string,
-    env?: AuthenticatedEnvironment,
+    authenticatedEnv?: AuthenticatedEnvironment,
     setToExecuting = true
   ): Promise<{
     execution: TaskRunExecution;
     run: TaskRun;
     attempt: TaskRunAttempt;
   }> {
-    const environment = env ?? (await getAuthenticatedEnvironmentFromRun(runId, this._prisma));
+    const environment =
+      authenticatedEnv ?? (await getAuthenticatedEnvironmentFromRun(runId, this._prisma));
 
     if (!environment) {
       throw new ServiceValidationError("Environment not found", 404);
@@ -128,6 +131,20 @@ export class CreateTaskRunAttemptService extends BaseService {
         throw new ServiceValidationError("Failed to create task run attempt", 500);
       }
 
+      if (taskRunAttempt.number === 1 && taskRun.baseCostInCents > 0) {
+        await workerQueue.enqueue("v3.reportUsage", {
+          orgId: environment.organizationId,
+          data: {
+            costInCents: String(taskRun.baseCostInCents),
+          },
+          additionalData: {
+            runId: taskRun.id,
+          },
+        });
+      }
+
+      const machinePreset = machinePresetFromConfig(taskRun.lockedBy.machineConfig ?? {});
+
       const execution: TaskRunExecution = {
         task: {
           id: taskRun.lockedBy.slug,
@@ -151,6 +168,10 @@ export class CreateTaskRunAttemptService extends BaseService {
           tags: taskRun.tags.map((tag) => tag.name),
           isTest: taskRun.isTest,
           idempotencyKey: taskRun.idempotencyKey ?? undefined,
+          startedAt: taskRun.startedAt ?? taskRun.createdAt,
+          durationMs: taskRun.usageDurationMs,
+          costInCents: taskRun.costInCents,
+          baseCostInCents: taskRun.baseCostInCents,
         },
         queue: {
           id: queue.friendlyId,
@@ -176,6 +197,7 @@ export class CreateTaskRunAttemptService extends BaseService {
           taskRun.batchItems[0] && taskRun.batchItems[0].batchTaskRun
             ? { id: taskRun.batchItems[0].batchTaskRun.friendlyId }
             : undefined,
+        machine: machinePreset,
       };
 
       return {

apps/webapp/app/v3/services/registerNextTaskScheduleInstance.server.ts

@@ -20,6 +20,7 @@ export class RegisterNextTaskScheduleInstanceService extends BaseService {
 
     const nextScheduledTimestamp = calculateNextScheduledTimestamp(
       instance.taskSchedule.generatorExpression,
+      instance.taskSchedule.timezone,
       instance.lastScheduledTimestamp ?? new Date()
     );
 

apps/webapp/app/v3/services/restoreCheckpoint.server.ts

@@ -1,12 +1,10 @@
-import { TaskRunStatus, type Checkpoint, TaskRunAttemptStatus } from "@trigger.dev/database";
+import { type Checkpoint } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
 import { socketIo } from "../handleSocketIo.server";
-import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
+import { machinePresetFromConfig } from "../machinePresets.server";
 import { BaseService } from "./baseService.server";
-import { Machine } from "@trigger.dev/core/v3";
-
-const RESTORABLE_RUN_STATUSES: TaskRunStatus[] = ["WAITING_TO_RESUME"];
-const RESTORABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["PAUSED"];
+import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
+import { isRestorableAttemptStatus, isRestorableRunStatus } from "../taskStatus";
 
 export class RestoreCheckpointService extends BaseService {
   public async call(params: {
@@ -45,41 +43,34 @@ export class RestoreCheckpointService extends BaseService {
     });
 
     if (!checkpointEvent) {
-      logger.error("Checkpoint event not found", params);
+      logger.error("Checkpoint event not found", { eventId: params.eventId });
       return;
     }
 
     const checkpoint = checkpointEvent.checkpoint;
 
-    const runIsRestorable = RESTORABLE_RUN_STATUSES.includes(checkpoint.run.status);
-    const attemptIsRestorable = RESTORABLE_ATTEMPT_STATUSES.includes(checkpoint.attempt.status);
-
-    if (!runIsRestorable) {
+    if (!isRestorableRunStatus(checkpoint.run.status)) {
       logger.error("Run is unrestorable", {
-        id: checkpoint.runId,
-        status: checkpoint.run.status,
+        eventId: params.eventId,
+        runId: checkpoint.runId,
+        runStatus: checkpoint.run.status,
+        attemptId: checkpoint.attemptId,
       });
       return;
     }
 
-    if (!attemptIsRestorable && !params.isRetry) {
+    if (!isRestorableAttemptStatus(checkpoint.attempt.status) && !params.isRetry) {
       logger.error("Attempt is unrestorable", {
-        id: checkpoint.attemptId,
-        status: checkpoint.attempt.status,
+        eventId: params.eventId,
+        runId: checkpoint.runId,
+        attemptId: checkpoint.attemptId,
+        attemptStatus: checkpoint.attempt.status,
       });
       return;
     }
 
     const { machineConfig } = checkpoint.attempt.backgroundWorkerTask;
-    const machine = Machine.safeParse(machineConfig ?? {});
-
-    if (!machine.success) {
-      logger.error("Failed to parse machine config", {
-        attemptId: checkpoint.attemptId,
-        machineConfig: checkpoint.attempt.backgroundWorkerTask.machineConfig,
-      });
-      return;
-    }
+    const machine = machinePresetFromConfig(machineConfig ?? {});
 
     const restoreEvent = await this._prisma.checkpointRestoreEvent.findFirst({
       where: {
@@ -90,6 +81,8 @@ export class RestoreCheckpointService extends BaseService {
 
     if (restoreEvent) {
       logger.error("Restore event already exists", {
+        runId: checkpoint.runId,
+        attemptId: checkpoint.attemptId,
         checkpointId: checkpoint.id,
         restoreEventId: restoreEvent.id,
       });
@@ -106,7 +99,7 @@ export class RestoreCheckpointService extends BaseService {
       location: checkpoint.location,
       reason: checkpoint.reason ?? undefined,
       imageRef: checkpoint.imageRef,
-      machine: machine.data,
+      machine,
       // identifiers
       checkpointId: checkpoint.id,
       envId: checkpoint.runtimeEnvironment.id,

apps/webapp/app/v3/services/testTask.server.ts

@@ -27,6 +27,7 @@ export class TestTaskService extends BaseService {
           scheduleId: "sched_1234",
           timestamp: data.timestamp,
           lastTimestamp: data.lastTimestamp,
+          timezone: data.timezone,
           externalId: data.externalId,
           upcoming: [],
         };