v3: self-hosting (#1147)

* add amin email regex env var

* fix displayed init command for self-hosted setups

* shared env var to disable telemetry in cli and webapp

* pin sdk version during init

* if specified, add api url to dev command shown after init

* improve checkpoint support detection

* control forced checkpoint simulation via env var

* add public init to providers

* better checkpoint support check for coordinator

* add docker to coordinator image

* update docker provider containerfile

* bump remaining containers to node 20

* add infra image build to default publish workflow

* lockfile

* remove concurrency group from infra workflow

* add docker provider to build matrix

* fix var subst

* checkpoint test is docker specific

* enable v3 projects by default on self-hosted instances

* fix v3 setup command again

* add default posthog key

* self-hosting docs

* add latest tags to versioned infra and webapp builds

* some checkpoint errors should skip retrying

* add changeset

* shorten paragraph

* some docs updates

* update tunnelling section

* add registry setup section

* use correct cli push flag

* add checkout to v3 branch

* update the worker machine setup steps

* fix infra build

* small docs update

* remove unused feature function

* Revert "remove unused feature function"

This reverts commit cfe07887a12b6893dca8ce499964481a9b3dc9db.

* fix self-hosted v3 feature gate

* add note about missing arm support

* simplify helper script syntax

.changeset/spicy-terms-bow.md

@@ -0,0 +1,9 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+---
+
+- Fix init command SDK pinning
+- Show --api-url / -a flag where needed
+- CLI now also respects `TRIGGER_TELEMETRY_DISABLED`
+- Dedicated docker checkpoint test function

.env.example

@@ -25,6 +25,8 @@ DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
 # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=

.github/workflows/publish-docker.yml

@@ -39,11 +39,25 @@ jobs:
             exit 1
           fi
           echo "::set-output name=version::${IMAGE_TAG}"
+
       - name: 🔢 Get the commit hash
         id: get_commit
         run: |
           echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
 
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }}
+
+          # if it's a versioned tag, also tag it as latest
+          if [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            image_tags=$image_tags,$ref_without_tag:latest
+          fi
+          
+          echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT"
+
       - name: 🐙 Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -56,6 +70,5 @@ jobs:
         with:
           file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
-          tags: |
-            ghcr.io/triggerdotdev/trigger.dev:${{ steps.get_version.outputs.version }}
+          tags: ${{ steps.set_tags.outputs.IMAGE_TAGS }}
           push: true

.github/workflows/publish-infra.yml

@@ -1,6 +1,7 @@
 name: "🚢 Publish Infra Images"
 
 on:
+  workflow_call:
   push:
     tags:
       - "infra-dev-*"
@@ -29,9 +30,6 @@ permissions:
   packages: write
   contents: read
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-
 env:
   AWS_REGION: us-east-1
 
@@ -39,7 +37,7 @@ jobs:
   build:
     strategy:
       matrix:
-        package: [coordinator, kubernetes-provider]
+        package: [coordinator, docker-provider, kubernetes-provider]
     runs-on: buildjet-16vcpu-ubuntu-2204
     env:
       DOCKER_BUILDKIT: "1"
@@ -48,20 +46,40 @@ jobs:
 
       - name: Generate image reference
         id: prep
-        # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-*
         run: |
-          env=$(echo ${{ github.ref_name }} | cut -d- -f2)
-          sha=${GITHUB_SHA::7}
-          ts=$(date +%s)
+          # set image repo
           if [[ "${{ matrix.package }}" == *-provider ]]; then
-            provider_type=$(echo ${{ matrix.package }} | cut -d- -f1)
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
             repository=provider/${provider_type}
           else
-            repository=${{ matrix.package }}
+            repository="${{ matrix.package }}"
           fi
-          echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
           echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
 
+          # set image tag
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+              env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+              sha=$(echo ${{ github.sha }} | head -c7)
+              ts=$(date +%s)
+              image_tag=${env}-${sha}-${ts}
+            elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+              version="${GITHUB_REF_NAME#v.docker.}"
+              image_tag="v${version}"
+            elif [[ "${{ github.ref_name }}" == build-* ]]; then
+              image_tag="${GITHUB_REF_NAME#build-}"
+            else
+              echo "Invalid tag: ${{ github.ref_name }}"
+              exit 1
+            fi
+          elif [[ "${{ github.ref_name }}" == "main" ]]; then
+            image_tag="main"
+          else
+            echo "Invalid reference: ${{ github.ref }}"
+            exit 1
+          fi
+          echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_OUTPUT"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -92,3 +110,12 @@ jobs:
           REGISTRY: ghcr.io/triggerdotdev
           REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
           IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}
+
+      - name: 🐙 Push 'latest' to GitHub Container Registry
+        if: startsWith(github.ref_name, 'v.docker.')
+        run: |
+          docker tag infra_image $REGISTRY/$REPOSITORY:latest
+          docker push $REGISTRY/$REPOSITORY:latest
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}

.github/workflows/publish.yml

@@ -57,3 +57,8 @@ jobs:
     needs: [typecheck, units]
     uses: ./.github/workflows/publish-docker.yml
     secrets: inherit
+
+  publish-infra:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-infra.yml
+    secrets: inherit

apps/coordinator/Containerfile

@@ -1,19 +1,19 @@
 # syntax=docker/dockerfile:labs
 
-FROM node:18-bullseye-slim@sha256:a4edd54dcfdcacc8a4100fee71498e8671d99556a1acf5614539214a70092426 AS node-18
+FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20
 
 WORKDIR /app
 
-FROM node-18 AS pruner
+FROM node-20 AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18 AS base
+FROM node-20 AS base
 
 RUN apt-get update \
-  && apt-get install -y buildah ca-certificates dumb-init \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io \
   && rm -rf /var/lib/apt/lists/*
 
 COPY --chown=node:node .gitignore .gitignore

apps/coordinator/src/index.ts

@@ -12,7 +12,7 @@ import {
 } from "@trigger.dev/core/v3";
 import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace";
 import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
-import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps";
+import { HttpReply, getTextBody, SimpleLogger, testDockerCheckpoint } from "@trigger.dev/core-apps";
 import { ExponentialBackoff } from "./backoff";
 
 import { collectDefaultMetrics, register, Gauge } from "prom-client";
@@ -72,7 +72,10 @@ type CheckpointAndPushOptions = {
 
 type CheckpointAndPushResult =
   | { success: true; checkpoint: CheckpointData }
-  | { success: false; reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" };
+  | {
+      success: false;
+      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+    };
 
 type CheckpointData = {
   location: string;
@@ -125,65 +128,53 @@ class Checkpointer {
 
   constructor(private opts = { forceSimulate: false }) {}
 
-  async initialize(): Promise<CheckpointerInitializeReturn> {
+  async init(): Promise<CheckpointerInitializeReturn> {
     if (this.#initialized) {
-      return this.#getInitializeReturn();
+      return this.#getInitReturn(this.#canCheckpoint);
     }
 
     this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
 
     if (this.#dockerMode) {
-      try {
-        await $`criu --version`;
-      } catch (error) {
-        this.#logger.error("No checkpoint support: Missing CRIU binary");
-        this.#logger.error("Will simulate instead");
-        this.#canCheckpoint = false;
-        this.#initialized = true;
+      const testCheckpoint = await testDockerCheckpoint();
 
-        return this.#getInitializeReturn();
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
       }
 
-      try {
-        await $`docker checkpoint`;
-      } catch (error) {
-        this.#logger.error(
-          "No checkpoint support: Docker needs to have experimental features enabled"
-        );
-        this.#logger.error("Will simulate instead");
-        this.#canCheckpoint = false;
-        this.#initialized = true;
-
-        return this.#getInitializeReturn();
-      }
+      this.#logger.error(testCheckpoint.message, testCheckpoint.error ?? "");
+      return this.#getInitReturn(false);
     } else {
       try {
         await $`buildah login --get-login ${REGISTRY_HOST}`;
       } catch (error) {
         this.#logger.error(`No checkpoint support: Not logged in to registry ${REGISTRY_HOST}`);
-        this.#canCheckpoint = false;
-        this.#initialized = true;
-
-        return this.#getInitializeReturn();
+        return this.#getInitReturn(false);
       }
     }
 
-    this.#logger.log(
-      `Full checkpoint support${
-        this.#dockerMode && this.opts.forceSimulate ? " with forced simulation enabled." : "!"
-      }`
-    );
-
-    this.#initialized = true;
-    this.#canCheckpoint = true;
-
-    return this.#getInitializeReturn();
+    return this.#getInitReturn(true);
   }
 
-  #getInitializeReturn(): CheckpointerInitializeReturn {
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#initialized = true;
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      this.#logger.log("Full checkpoint support!");
+    }
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
     return {
-      canCheckpoint: this.#canCheckpoint,
-      willSimulate: this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate),
+      canCheckpoint,
+      willSimulate,
     };
   }
 
@@ -327,6 +318,11 @@ class Checkpointer {
           return result;
         }
 
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
         continue;
       } catch (error) {
         this.#logger.error("Checkpoint error", {
@@ -355,7 +351,7 @@ class Checkpointer {
     projectRef,
     deploymentVersion,
   }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
-    await this.initialize();
+    await this.init();
 
     const options = {
       runId,
@@ -473,7 +469,8 @@ class Checkpointer {
 
       // Create checkpoint (CRI)
       if (!this.#canCheckpoint) {
-        throw new Error("No checkpoint support in kubernetes mode.");
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
       }
 
       const containerId = this.#logger.debug(
@@ -484,7 +481,8 @@ class Checkpointer {
       );
 
       if (!containerId.stdout) {
-        throw new Error("could not find container id");
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
       }
 
       const start = performance.now();
@@ -617,7 +615,7 @@ class TaskCoordinator {
     private host = "0.0.0.0"
   ) {
     this.#httpServer = this.#createHttpServer();
-    this.#checkpointer.initialize();
+    this.#checkpointer.init();
     this.#delayThresholdInMs = this.#getDelayThreshold();
 
     if (process.env.DELAY_THRESHOLD_IN_MS) {
@@ -1034,7 +1032,7 @@ class TaskCoordinator {
             return;
           }
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 
@@ -1131,7 +1129,7 @@ class TaskCoordinator {
             return;
           }
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 
@@ -1185,7 +1183,7 @@ class TaskCoordinator {
         socket.on("WAIT_FOR_TASK", async (message, callback) => {
           logger.log("[WAIT_FOR_TASK]", message);
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 
@@ -1227,7 +1225,7 @@ class TaskCoordinator {
         socket.on("WAIT_FOR_BATCH", async (message, callback) => {
           logger.log("[WAIT_FOR_BATCH]", message);
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 

apps/docker-provider/Containerfile

@@ -1,16 +1,47 @@
-# syntax=docker/dockerfile:labs
-
-FROM node:18-slim AS base
-
-RUN apt-get update \
-  && apt-get install -y dumb-init
-
-FROM base
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-COPY --chown=node dist/index.mjs /app/
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init docker
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter docker-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
 
 EXPOSE 8000
 
-ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.mjs" ]
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/docker-provider/src/index.ts

@@ -6,6 +6,8 @@ import {
   TaskOperationsRestoreOptions,
   TaskOperationsCreateOptions,
   TaskOperationsIndexOptions,
+  isExecaChildProcess,
+  testDockerCheckpoint,
 } from "@trigger.dev/core-apps";
 import { setTimeout } from "node:timers/promises";
 import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
@@ -23,70 +25,58 @@ const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
 
 const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
 
-type InitializeReturn = {
+type TaskOperationsInitReturn = {
   canCheckpoint: boolean;
   willSimulate: boolean;
 };
 
-function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
-  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
-}
-
 class DockerTaskOperations implements TaskOperations {
   #initialized = false;
   #canCheckpoint = false;
 
   constructor(private opts = { forceSimulate: false }) {}
 
-  async #initialize(): Promise<InitializeReturn> {
+  async init(): Promise<TaskOperationsInitReturn> {
     if (this.#initialized) {
-      return this.#getInitializeReturn();
+      return this.#getInitReturn(this.#canCheckpoint);
     }
 
     logger.log("Initializing task operations");
 
-    if (this.opts.forceSimulate) {
-      logger.log("Forced simulation enabled. Will simulate regardless of checkpoint support.");
+    const testCheckpoint = await testDockerCheckpoint();
+
+    if (testCheckpoint.ok) {
+      return this.#getInitReturn(true);
     }
 
-    try {
-      await $`criu --version`;
-    } catch (error) {
-      logger.error("No checkpoint support: Missing CRIU binary. Will simulate instead.");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    try {
-      await $`docker checkpoint`;
-    } catch (error) {
-      logger.error("No checkpoint support: Docker needs to have experimental features enabled");
-      logger.error("Will simulate instead");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    logger.log("Full checkpoint support!");
-
-    this.#initialized = true;
-    this.#canCheckpoint = true;
-
-    return this.#getInitializeReturn();
+    logger.error(testCheckpoint.message, testCheckpoint.error);
+    return this.#getInitReturn(false);
   }
 
-  #getInitializeReturn(): InitializeReturn {
+  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
+    this.#initialized = true;
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      logger.log("Full checkpoint support!");
+    }
+
+    const willSimulate = !canCheckpoint || this.opts.forceSimulate;
+
+    if (willSimulate) {
+      logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
     return {
-      canCheckpoint: this.#canCheckpoint,
-      willSimulate: !this.#canCheckpoint || this.opts.forceSimulate,
+      canCheckpoint,
+      willSimulate,
     };
   }
 
   async index(opts: TaskOperationsIndexOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getIndexContainerName(opts.shortCode);
 
@@ -129,7 +119,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async create(opts: TaskOperationsCreateOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
 
@@ -165,7 +155,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async restore(opts: TaskOperationsRestoreOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
 
@@ -194,7 +184,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async delete(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
     await this.#sendPreStop(containerName);
@@ -203,7 +193,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async get(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     logger.log("noop: get");
   }

apps/kubernetes-provider/Containerfile

@@ -1,14 +1,14 @@
-FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-FROM node-18-alpine AS pruner
+FROM node-20-alpine AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18-alpine AS base
+FROM node-20-alpine AS base
 
 RUN apk add --no-cache dumb-init
 

apps/kubernetes-provider/src/index.ts

@@ -47,6 +47,10 @@ class KubernetesTaskOperations implements TaskOperations {
     this.#k8sApi = this.#createK8sApi();
   }
 
+  async init() {
+    // noop
+  }
+
   async index(opts: TaskOperationsIndexOptions) {
     await this.#createJob(
       {

apps/webapp/app/components/SetupCommands.tsx

@@ -8,6 +8,7 @@ import {
 } from "./primitives/ClientTabs";
 import { ClipboardField } from "./primitives/ClipboardField";
 import { Paragraph } from "./primitives/Paragraph";
+import { useAppOrigin } from "~/hooks/useAppOrigin";
 
 export function InitCommand({ appOrigin, apiKey }: { appOrigin: string; apiKey: string }) {
   return (
@@ -133,9 +134,38 @@ export function TriggerDevStep({ extra }: { extra?: string }) {
 // Trigger.dev version 3 setup commands
 const v3PackageTag = "beta";
 
+function getApiUrlArg() {
+  const appOrigin = useAppOrigin();
+
+  let apiUrl: string | undefined = undefined;
+
+  switch (appOrigin) {
+    case "https://cloud.trigger.dev":
+      // don't display the arg, use the CLI default
+      break;
+    case "https://test-cloud.trigger.dev":
+      apiUrl = "https://test-api.trigger.dev";
+      break;
+    case "https://internal.trigger.dev":
+      apiUrl = "https://internal-api.trigger.dev";
+      break;
+    default:
+      apiUrl = appOrigin;
+      break;
+  }
+
+  return apiUrl ? `-a ${apiUrl}` : undefined;
+}
+
 export function InitCommandV3() {
   const project = useProject();
   const projectRef = project.ref;
+
+  const apiUrlArg = getApiUrlArg();
+
+  const initCommandParts = [`trigger.dev@${v3PackageTag}`, "init", `-p ${projectRef}`, apiUrlArg];
+  const initCommand = initCommandParts.filter(Boolean).join(" ");
+
   return (
     <ClientTabs defaultValue="npm">
       <ClientTabsList>
@@ -148,7 +178,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`npx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`npx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"pnpm"}>
@@ -156,7 +186,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`pnpm dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`pnpm dlx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"yarn"}>
@@ -164,7 +194,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`yarn dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`yarn dlx ${initCommand}`}
         />
       </ClientTabsContent>
     </ClientTabs>

apps/webapp/app/env.server.ts

@@ -27,15 +27,17 @@ const EnvironmentSchema = z.object({
     .string()
     .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.")
     .optional(),
+  ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(),
   REMIX_APP_PORT: z.string().optional(),
   LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ENV: z.string().default(process.env.NODE_ENV),
   SERVICE_NAME: z.string().default("trigger.dev webapp"),
   SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"),
-  POSTHOG_PROJECT_KEY: z.string().optional(),
+  POSTHOG_PROJECT_KEY: z.string().default("phc_LFH7kJiGhdIlnO22hTAKgHpaKhpM8gkzWAFvHmf5vfS"),
   TELEMETRY_TRIGGER_API_KEY: z.string().optional(),
   TELEMETRY_TRIGGER_API_URL: z.string().optional(),
+  TRIGGER_TELEMETRY_DISABLED: z.string().optional(),
   HIGHLIGHT_PROJECT_ID: z.string().optional(),
   AUTH_GITHUB_CLIENT_ID: z.string().optional(),
   AUTH_GITHUB_CLIENT_SECRET: z.string().optional(),

apps/webapp/app/features.server.ts

@@ -7,20 +7,28 @@ export type TriggerFeatures = {
   alertsEnabled: boolean;
 };
 
-// If the request host is cloud.trigger.dev then we are on the managed cloud
-// or if env.NODE_ENV is development
-export function featuresForRequest(request: Request): TriggerFeatures {
-  const url = requestUrl(request);
-
-  const isManagedCloud =
-    url.host === "cloud.trigger.dev" ||
-    url.host === "test-cloud.trigger.dev" ||
-    url.host === "internal.trigger.dev" ||
-    process.env.CLOUD_ENV === "development";
+function isManagedCloud(host: string): boolean {
+  return (
+    host === "cloud.trigger.dev" ||
+    host === "test-cloud.trigger.dev" ||
+    host === "internal.trigger.dev" ||
+    process.env.CLOUD_ENV === "development"
+  );
+}
 
+function featuresForHost(host: string): TriggerFeatures {
   return {
-    isManagedCloud,
+    isManagedCloud: isManagedCloud(host),
     v3Enabled: env.V3_ENABLED === "true",
     alertsEnabled: env.ALERT_FROM_EMAIL !== undefined && env.ALERT_RESEND_API_KEY !== undefined,
   };
 }
+
+export function featuresForRequest(request: Request): TriggerFeatures {
+  const url = requestUrl(request);
+  return featuresForUrl(url);
+}
+
+export function featuresForUrl(url: URL): TriggerFeatures {
+  return featuresForHost(url.host);
+}

apps/webapp/app/models/organization.server.ts

@@ -8,10 +8,10 @@ import type {
 import { customAlphabet } from "nanoid";
 import slug from "slug";
 import { prisma, PrismaClientOrTransaction } from "~/db.server";
-import { createProject } from "./project.server";
 import { generate } from "random-words";
 import { createApiKeyForEnv, createPkApiKeyForEnv, envSlug } from "./api-key.server";
 import { env } from "~/env.server";
+import { featuresForUrl } from "~/features.server";
 
 export type { Organization };
 
@@ -52,6 +52,8 @@ export async function createOrganization(
     );
   }
 
+  const features = featuresForUrl(new URL(env.APP_ORIGIN));
+
   const organization = await prisma.organization.create({
     data: {
       title,
@@ -64,6 +66,7 @@ export async function createOrganization(
           role: "ADMIN",
         },
       },
+      v3Enabled: features.v3Enabled && !features.isManagedCloud,
     },
     include: {
       members: true,

apps/webapp/app/models/user.server.ts

@@ -47,12 +47,21 @@ export async function findOrCreateMagicLinkUser(
     },
   });
 
+  const adminEmailRegex = env.ADMIN_EMAILS ? new RegExp(env.ADMIN_EMAILS) : undefined;
+  const makeAdmin = adminEmailRegex ? adminEmailRegex.test(input.email) : false;
+
   const user = await prisma.user.upsert({
     where: {
       email: input.email,
     },
-    update: { email: input.email },
-    create: { email: input.email, authenticationMethod: "MAGIC_LINK" },
+    update: {
+      email: input.email,
+    },
+    create: {
+      email: input.email,
+      authenticationMethod: "MAGIC_LINK",
+      admin: makeAdmin, // only on create, to prevent automatically removing existing admins
+    },
   });
 
   return {

apps/webapp/app/services/telemetry.server.ts

@@ -22,6 +22,11 @@ class Telemetry {
   #triggerClient: TriggerClient | undefined = undefined;
 
   constructor({ postHogApiKey, trigger }: Options) {
+    if (env.TRIGGER_TELEMETRY_DISABLED !== undefined) {
+      console.log("📉 Telemetry disabled");
+      return;
+    }
+
     if (postHogApiKey) {
       this.#posthogClient = new PostHog(postHogApiKey, { host: "https://eu.posthog.com" });
     } else {

docs/v3/cli-deploy.mdx

@@ -68,7 +68,7 @@ Load the built image into your local docker.
 
 #### Self-hosted (builds locally) `--self-hosted`
 
-Builds and loads the image using your local docker. Use the `--registry` option to specify the registry to push the image to when using `--self-hosted`, or just use `--push-image` to push to the default registry.
+Builds and loads the image using your local docker. Use the `--registry` option to specify the registry to push the image to when using `--self-hosted`, or just use `--push` to push to the default registry.
 
 #### Registry `--registry`
 
@@ -76,9 +76,7 @@ Builds and loads the image using your local docker. Use the `--registry` option
 
 The registry to push the image to when using --self-hosted.
 
-#### Push image `--push-image`
-
-<Snippet file="coming-soon-slim.mdx" />
+#### Push image `--push`
 
 When using the --self-hosted flag, push the image to the default registry. (defaults to false when not using --registry)
 

docs/v3/open-source-self-hosting.mdx

@@ -3,4 +3,275 @@ title: "Self-hosting"
 description: "You can self-host Trigger.dev on your own infrastructure."
 ---
 
-<Snippet file="incomplete-docs.mdx" />
+## Overview
+
+<Frame>
+  <img src="/images/v3/self-hosting.png" alt="Self-hosting architecture" />
+</Frame>
+
+The self-hosting guide comes in two parts. The first part is a simple setup where you run everything on one server. In the second part, the webapp and worker components are split on two separate machines.
+
+You're going to need at least one Debian (or derivative) machine with Docker and Docker Compose installed. We'll also use Ngrok to expose the webapp to the internet.
+
+## Caveats
+
+<Note>The v3 worker components don't have ARM support yet.</Note>
+
+This guide outlines a quick way to start self-hosting Trigger.dev. Scaling, security, and reliability concerns are not fully addressed here. It's unlikely to result in a production-ready deployment on its own, but it's a good starting point.
+
+As self-hosted deployments tend to have unique requirements and configurations, we don't provide specific advice for scaling up or improving security and reliability.
+
+Should the burden ever get too much, we'd be happy to see you on [Trigger.dev cloud](https://trigger.dev/pricing) where we deal with these concerns for you.
+
+<Accordion title="Please consider these additional warnings">
+- The Docker [checkpoint command](https://docs.docker.com/reference/cli/docker/checkpoint/) is an experimental feature which may not work as expected. It won't be enabled by default. Instead, the containers will stay up and their processes frozen. They won't consume CPU but they _will_ consume RAM.
+- The Docker provider does not currently enforce any resource limits. This means your tasks can consume up to the total machine CPU and RAM. Having no limits may be preferable when self-hosting, but can impact the performance of other services.
+- The worker components (not the tasks!) have direct access to the Docker socket. This means they can run any Docker command. To restrict access, you may want to consider using [Docker Socket Proxy](https://github.com/Tecnativa/docker-socket-proxy).
+- The task containers are running with host networking. This means there is no network isolation between them and the host machine. They will be able to access any networked service on the host.
+- There is currently no support for adding multiple worker machines. This would require a more elaborate provider, or possibly a switch to Docker Swarm. This is not currently planned, but you are welcome to [contribute](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md).
+</Accordion>
+
+## Requirements
+
+- 4 CPU
+- 8 GB RAM
+- Debian or derivative
+- Optional: A separate machine for the worker components
+
+You will also need a way to expose the webapp to the internet. This can be done with a reverse proxy, or with a service like Ngrok. We will be using the latter in this guide.
+
+## Part 1: Single server
+
+This is the simplest setup. You run everything on one server. It's a good option if you have spare capacity on an existing machine, and have no need to independently scale worker capacity.
+
+### Server setup
+
+Some very basic steps to get started:
+
+1. [Install Docker](https://docs.docker.com/get-docker/)
+2. [Install Docker Compose](https://docs.docker.com/compose/install/)
+3. [Install Ngrok](https://ngrok.com/download)
+
+On a Debian server, you can install everything you need with the following commands:
+
+```bash
+curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | \
+    sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && \
+    echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | \
+    sudo tee /etc/apt/sources.list.d/ngrok.list
+
+sudo apt-get update
+sudo apt-get install -y \
+    docker.io \
+    docker-compose \
+    ngrok
+```
+
+### Trigger.dev setup
+
+1. Clone the [Trigger.dev docker repository](https://github.com/triggerdotdev/docker) and checkout the v3 branch
+```bash
+git clone https://github.com/triggerdotdev/docker
+cd docker
+git checkout v3
+```
+
+2. Run the start script and follow the prompts
+```bash
+./start.sh # hint: you can append -d to run in detached mode
+```
+
+### Manual setup
+
+Alternatively, you can follow these manual steps after cloning the docker repo:
+
+1. Create the `.env` file
+
+```bash
+cp .env.example .env
+```
+
+2. Generate the required secrets
+
+```bash
+echo MAGIC_LINK_SECRET=$(openssl rand -hex 16)
+echo SESSION_SECRET=$(openssl rand -hex 16)
+echo ENCRYPTION_KEY=$(openssl rand -hex 16)
+echo PROVIDER_SECRET=$(openssl rand -hex 32)
+echo COORDINATOR_SECRET=$(openssl rand -hex 32)
+```
+
+3. Replace the default secrets in the `.env` file with the generated ones
+
+4. Run docker compose to start the services
+
+```bash
+. lib.sh # source the helper function
+docker_compose -p=trigger up
+```
+
+### Tunnelling
+
+You will need to expose the webapp to the internet. You can use Ngrok for this. If you already have a working reverse proxy setup and a domain, you can skip to the last step.
+
+1. Start Ngrok. You may get prompted to sign up - it's free.
+
+```bash
+./tunnel.sh
+```
+
+2. Copy the domain from the output, for example: `1234-42-42-42-42.ngrok-free.app`
+
+3. Uncomment the `TRIGGER_PROTOCOL` and `TRIGGER_DOMAIN` lines in the `.env` file. Set it to the domain you copied.
+
+```bash
+TRIGGER_PROTOCOL=https
+TRIGGER_DOMAIN=1234-42-42-42-42.ngrok-free.app
+```
+
+4. Quit the start script and launch it again, or run this:
+
+```bash
+./stop.sh && ./start.sh
+```
+
+### Registry setup
+
+If you want to deploy v3 projects, you will need access to a Docker registry. The [CLI deploy](/v3/cli-deploy) command will push the images, and then the worker machine can pull them when needed. We will use Docker Hub as an example.
+
+1. Sign up for a free account at [Docker Hub](https://hub.docker.com/)
+
+2. Edit the `.env` file and add the registry details
+
+```bash
+DEPLOY_REGISTRY_HOST=docker.io
+DEPLOY_REGISTRY_NAMESPACE=<your_dockerhub_username>
+```
+
+3. Log in to Docker Hub both locally and your server. For the split setup, this will be the worker machine. You may want to create an [access token](https://hub.docker.com/settings/security) for this.
+
+```bash
+docker login -u <your_dockerhub_username>
+```
+
+4. Restart the services
+
+```bash
+./stop.sh && ./start.sh
+```
+
+5. You can now deploy v3 projects using the CLI with these flags:
+
+```bash
+trigger deploy --self-hosted --push
+```
+
+## Part 2: Split services
+
+With this setup, the webapp will run on a different machine than the worker components. This allows independent scaling of your workload capacity.
+
+### Webapp setup
+
+All steps are the same as in Part 1, except for the following:
+
+1. Run the start script with the `webapp` argument
+
+```bash
+./start.sh webapp
+```
+
+2. Tunnelling is now _required_. Please follow the tunnelling section from above.
+
+### Worker setup
+
+1. Copy your `.env` file from the webapp to the worker machine
+
+```bash
+# an example using scp
+scp -3 root@<webapp_machine>:docker/.env root@<worker_machine>:docker/.env
+```
+
+2. Run the start script with the `worker` argument
+
+```bash
+./start.sh worker
+```
+
+2. Tunnelling is _not_ required for the worker components.
+
+## Checkpoint support
+
+<Warning>This requires an _experimental Docker feature_. Successfully checkpointing a task today, does not mean you will be able to restore it tomorrow. Your data may be lost. You've been warned!</Warning>
+
+Checkpointing allows you to save the state of a running container to disk and restore it later. This can be useful for 
+long-running tasks that need to be paused and resumed without losing state. Think fan-out and fan-in, or long waits in email campaigns.
+
+### Requirements
+
+- Debian, **NOT** a derivative like Ubuntu
+- Additional storage space for the checkpointed containers
+
+### Setup
+
+Underneath the hood this uses Checkpoint and Restore in Userspace, or [CRIU](https://github.com/checkpoint-restore/criu) in short. We'll have to do a few things to get this working:
+
+1. Install CRIU
+
+```bash
+sudo apt-get update
+sudo apt-get install criu
+```
+
+2. Tweak the config so we can successfully checkpoint our workloads
+
+```bash
+mkdir /etc/criu
+
+cat << EOF >/etc/criu/runc.conf
+tcp-close
+EOF
+```
+
+3. Make sure everything works
+
+```bash
+sudo criu check
+```
+
+3. Enable Docker experimental features, by adding the following to `/etc/docker/daemon.json`
+
+```json
+{
+  "experimental": true
+}
+```
+
+4. Restart the Docker daemon
+
+```bash
+sudo systemctl restart docker
+```
+
+5. Uncomment `FORCE_CHECKPOINT_SIMULATION=0` in your `.env` file. Alternatively, run this:
+
+```bash
+echo "FORCE_CHECKPOINT_SIMULATION=0" >> .env
+```
+
+6. Restart the services
+
+```bash
+# if you're running everything on the same machine
+./stop.sh && ./start.sh
+
+# if you're running the worker on a different machine
+./stop.sh worker && ./start.sh worker
+```
+
+## Telemetry
+
+By default, the Trigger.dev webapp sends telemetry data to our servers. This data is used to improve the product and is not shared with third parties. If you would like to opt-out of this, you can set the `TRIGGER_TELEMETRY_DISABLED` environment variable in your `.env` file. The value doesn't matter, it just can't be empty. For example:
+
+```bash
+TRIGGER_TELEMETRY_DISABLED=1
+```

packages/cli-v3/src/commands/deploy.ts

@@ -107,7 +107,7 @@ export function configureDeployCommand(program: Command) {
     .addOption(
       new CommandOption(
         "--self-hosted",
-        "Build and load the image using your local Docker. Use the --registry option to specify the registry to push the image to when using --self-hosted, or just use --push-image to push to the default registry."
+        "Build and load the image using your local Docker. Use the --registry option to specify the registry to push the image to when using --self-hosted, or just use --push to push to the default registry."
       ).hideHelp()
     )
     .addOption(

packages/cli-v3/src/commands/init.ts

@@ -30,6 +30,8 @@ import { logger } from "../utilities/logger";
 import { cliRootPath } from "../utilities/resolveInternalFilePath";
 import { login } from "./login";
 import { spinner } from "../utilities/windows";
+import { CLOUD_API_URL } from "../consts";
+import * as packageJson from "../../package.json";
 
 const InitCommandOptions = CommonCommandOptions.extend({
   projectRef: z.string().optional(),
@@ -53,7 +55,7 @@ export function configureInitCommand(program: Command) {
       .option(
         "-t, --tag <package tag>",
         "The version of the @trigger.dev/sdk package to install",
-        "beta"
+        packageJson.version
       )
       .option("--skip-package-install", "Skip installing the @trigger.dev/sdk package")
       .option("--override-config", "Override the existing config file if it exists")
@@ -160,7 +162,9 @@ async function _initCommand(dir: string, options: InitCommandOptions) {
   log.info("Next steps:");
   log.info(
     `   1. To start developing, run ${chalk.green(
-      `npx trigger.dev@${options.tag} dev`
+      `npx trigger.dev@${options.tag} dev${
+        options.apiUrl === CLOUD_API_URL ? "" : ` -a ${options.apiUrl}`
+      }`
     )} in your project directory`
   );
   log.info(`   2. Visit your ${projectDashboard} to view your newly created tasks.`);
@@ -401,7 +405,8 @@ async function installPackages(dir: string, options: InitCommandOptions) {
         case "npm": {
           installSpinner.start(`Running npm install @trigger.dev/sdk@${options.tag}`);
 
-          await execa("npm", ["install", `@trigger.dev/sdk@${options.tag}`], {
+          // --save-exact: pin version, e.g. 3.0.0-beta.20 instead of ^3.0.0-beta.20
+          await execa("npm", ["install", "--save-exact", `@trigger.dev/sdk@${options.tag}`], {
             cwd: projectDir,
             stdio: options.logLevel === "debug" ? "inherit" : "ignore",
           });
@@ -411,6 +416,7 @@ async function installPackages(dir: string, options: InitCommandOptions) {
         case "pnpm": {
           installSpinner.start(`Running pnpm add @trigger.dev/sdk@${options.tag}`);
 
+          // pins version by default
           await execa("pnpm", ["add", `@trigger.dev/sdk@${options.tag}`], {
             cwd: projectDir,
             stdio: options.logLevel === "debug" ? "inherit" : "ignore",
@@ -421,6 +427,7 @@ async function installPackages(dir: string, options: InitCommandOptions) {
         case "yarn": {
           installSpinner.start(`Running yarn add @trigger.dev/sdk@${options.tag}`);
 
+          // pins version by default
           await execa("yarn", ["add", `@trigger.dev/sdk@${options.tag}`], {
             cwd: projectDir,
             stdio: options.logLevel === "debug" ? "inherit" : "ignore",

packages/cli-v3/src/telemetry/tracing.ts

@@ -9,9 +9,15 @@ import {
   SEMRESATTRS_SERVICE_NAME,
   SEMRESATTRS_SERVICE_VERSION,
 } from "@opentelemetry/semantic-conventions";
+import { logger } from "../utilities/logger";
 
 function initializeTracing(): NodeTracerProvider | undefined {
-  if (process.argv.includes("--skip-telemetry") || process.env.TRIGGER_DEV_SKIP_TELEMETRY) {
+  if (
+    process.argv.includes("--skip-telemetry") ||
+    process.env.TRIGGER_DEV_SKIP_TELEMETRY || // only for backwards compat
+    process.env.TRIGGER_TELEMETRY_DISABLED
+  ) {
+    logger.debug("📉 Telemetry disabled");
     return;
   }
 

packages/core-apps/package.json

@@ -28,10 +28,13 @@
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/tsconfig": "workspace:*",
     "@types/node": "18",
-    "typescript": "^5.3.0",
-    "socket.io-client": "^4.7.4"
+    "socket.io-client": "^4.7.4",
+    "typescript": "^5.3.0"
   },
   "engines": {
     "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "execa": "^8.0.1"
   }
 }

packages/core-apps/src/checkpoints.ts

@@ -0,0 +1,67 @@
+import { randomUUID } from "crypto";
+import { $, type ExecaChildProcess } from "execa";
+
+export function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
+  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
+}
+
+export type CheckpointTestResult =
+  | {
+      ok: true;
+    }
+  | {
+      ok: false;
+      message: string;
+      error?: unknown;
+    };
+
+export async function testDockerCheckpoint(): Promise<CheckpointTestResult> {
+  try {
+    // Create a dummy container
+    const container =
+      await $`docker run -d --rm --name init-dummy-${randomUUID()} docker.io/library/busybox sleep 10`;
+
+    // Checkpoint it
+    await $`docker checkpoint create ${container} init-check`;
+  } catch (error) {
+    if (!isExecaChildProcess(error)) {
+      return {
+        ok: false,
+        message: "No checkpoint support: Unknown error.",
+        error,
+      };
+    }
+
+    if (error.stderr.includes("criu")) {
+      if (error.stderr.includes("executable file not found")) {
+        return {
+          ok: false,
+          message: "No checkpoint support: Missing CRIU binary.",
+        };
+      }
+
+      return {
+        ok: false,
+        message: "No checkpoint support: Unknown CRIU error.",
+        error,
+      };
+    }
+
+    if (error.stderr.includes("experimental features enabled")) {
+      return {
+        ok: false,
+        message: "No checkpoint support: Please enable docker experimental features.",
+      };
+    }
+
+    return {
+      ok: false,
+      message: "No checkpoint support: Unknown execa error.",
+      error,
+    };
+  }
+
+  return {
+    ok: true,
+  };
+}

packages/core-apps/src/index.ts

@@ -1,3 +1,4 @@
 export * from "./http";
 export * from "./logger";
 export * from "./provider";
+export * from "./checkpoints";

packages/core-apps/src/provider.ts

@@ -62,10 +62,14 @@ export interface TaskOperationsRestoreOptions {
 }
 
 export interface TaskOperations {
+  init: () => Promise<any>;
+
+  // CRUD
   index: (opts: TaskOperationsIndexOptions) => Promise<any>;
   create: (opts: TaskOperationsCreateOptions) => Promise<any>;
   restore: (opts: TaskOperationsRestoreOptions) => Promise<any>;
 
+  // unimplemented
   delete: (...args: any[]) => Promise<any>;
   get: (...args: any[]) => Promise<any>;
 }
@@ -303,7 +307,8 @@ export class ProviderShell implements Provider {
     return httpServer;
   }
 
-  listen() {
+  async listen() {
     this.#httpServer.listen(this.#httpPort, this.options.host ?? "0.0.0.0");
+    await this.tasks.init();
   }
 }

pnpm-lock.yaml

@@ -1811,6 +1811,10 @@ importers:
         version: 5.3.2
 
   packages/core-apps:
+    dependencies:
+      execa:
+        specifier: ^8.0.1
+        version: 8.0.1
     devDependencies:
       '@trigger.dev/core':
         specifier: workspace:*