v3: cli build command, prod runs, checkpoints (#919)

* fix trailing slash in api url

* add maybe platform down error

* shorten config path

* extend protobuf compiler install instructions

* build command and image model

* new image triggers task indexing

* index support for prod builds

* shorten example provider secret

* multi-stage task prod build

* lock prod tasks to node 18 image

* extract shared build and dev command libs

* pull out more shared deps

* add coordinator and providers

* fix core-apps build

* add dev builds for new apps

* fix cwd

* enable corepack

* some build fixes

* enable buildkit for old docker versions

* coordinator image fixes

* update provider containerfile

* build dev images in parallel

* upgrade pgadmin

* fix prod facade build

* prod runs

* fix merge

* don't knock out platform on invalid attempt id

* fix prod facade

* rename to build.ts

* fix prod builds

* set to executing after fetching payload

* make prod worker listen on random port if in use

* remove experimental warnings in dev

* prod resume

* prevent execution after completion

* exit prod worker after completion

* always restart otel collector

* docker checkpoints and prod runtime messaging

* don't retry indexing without chance of success

* make platform checkpoint aware

* deploy with existing hash sets latest worker

* log restore requests

* only try to checkpoint long waits

* tidying up

* lockfile

* fix build

* prod worker merge fixes

* fix prod complete and cancel

* fix lua nil checks

* socket namespace abstraction

* cleanup

* make all build args optional

* add build script

* don't require env vars for dev

* fix schema

* prod merge

* small fix

* bind correct logger

* fix v3 ref catalog entry

* resume prod batch

* pass socket to error and disconnect handlers

* fix non-batch resume

* fix batch resume

* send connection env vars when not in dev

* create worker via socket

* move api client back into v3 cli

* fix lockfile

* fix resume with failures

* marqs replace message

* typecheck prior to build

* don't define api url in prod builds

* support prod retries after resume

* skip typecheck option

.env.example

@@ -45,4 +45,8 @@ CLOUD_LINEAR_CLIENT_ID=
 CLOUD_LINEAR_CLIENT_SECRET=
 CLOUD_SLACK_APP_HOST=
 CLOUD_SLACK_CLIENT_ID=
-CLOUD_SLACK_CLIENT_SECRET= 
+CLOUD_SLACK_CLIENT_SECRET= 
+
+# v3 variables
+PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32`
+COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32`

.github/workflows/publish-dev.yml

@@ -0,0 +1,83 @@
+name: "🚢 Publish Container Images (dev)"
+
+on:
+  push:
+    tags:
+      - "dev-*"
+    paths:
+      - ".github/workflows/publish.yml"
+      - "packages/**"
+      - "!packages/**/*.md"
+      - "!packages/**/*.eslintrc"
+      - "apps/**"
+      - "!apps/**/*.md"
+      - "!apps/**/*.eslintrc"
+      - "integrations/**"
+      - "!integrations/**/*.md"
+      - "!integrations/**/*.eslintrc"
+      - "pnpm-lock.yaml"
+      - "pnpm-workspace.yaml"
+      - "turbo.json"
+      - "docker/Dockerfile"
+      - "docker/scripts/**"
+      - "tests/**"
+
+permissions:
+  id-token: write
+  packages: write
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+env:
+  AWS_REGION: us-east-1
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        package: [coordinator, kubernetes-provider]
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate build ID
+        id: prep
+        run: |
+          sha=${GITHUB_SHA::7}
+          ts=$(date +%s)
+          echo "BUILD_ID=${{ matrix.package }}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      # ..to avoid rate limits when pulling images
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t dev_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag dev_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: dev
+          IMAGE_TAG: ${{ steps.prep.outputs.BUILD_ID }}

apps/coordinator/.env.example

@@ -0,0 +1,4 @@
+HTTP_SERVER_PORT=8020
+
+PLATFORM_ENABLED=true
+PLATFORM_WS_PORT=3030

apps/coordinator/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/coordinator/Containerfile

@@ -0,0 +1,63 @@
+# syntax=docker/dockerfile:labs
+
+FROM node:18.18.2-bullseye-slim@sha256:21479df46c3173ee0cefc6b264928e10239152c4f74df872ca9369be01a245b7 AS node-18
+
+WORKDIR /app
+
+FROM node-18 AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-18 AS base
+
+RUN apt-get update \
+  && apt-get install -y buildah ca-certificates dumb-init \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter '@trigger.dev/core*' build
+RUN pnpm run -r --filter coordinator build:bundle
+
+FROM alpine AS cri-tools
+
+WORKDIR /cri-tools
+
+ARG CRICTL_VERSION=v1.29.0
+ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
+ADD --checksum=${CRICTL_CHECKSUM} \
+  https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz .
+RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin
+COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.cjs ./index.cjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.cjs" ]

apps/coordinator/README.md

@@ -0,0 +1,3 @@
+# Coordinator
+
+Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things.

apps/coordinator/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "coordinator",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "type": "module",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.cjs --platform=node",
+    "build:image": "docker build -f Containerfile . -t coordinator",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/core-apps": "workspace:*",
+    "execa": "^8.0.1",
+    "prom-client": "^15.1.0",
+    "socket.io": "^4.7.4",
+    "socket.io-client": "^4.7.4"
+  },
+  "devDependencies": {
+    "@types/node": "^18",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/coordinator/src/index.ts

@@ -0,0 +1,506 @@
+import { randomUUID } from "node:crypto";
+import { createServer } from "node:http";
+import { $ } from "execa";
+import { Namespace } from "socket.io";
+import { Server } from "socket.io";
+import { Socket, io } from "socket.io-client";
+import { DefaultEventsMap } from "socket.io/dist/typed-events";
+import {
+  CoordinatorToPlatformEvents,
+  CoordinatorToProdWorkerEvents,
+  PlatformToCoordinatorEvents,
+  ProdWorkerSocketData,
+  ProdWorkerToCoordinatorEvents,
+} from "@trigger.dev/core/v3";
+import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps";
+
+import { collectDefaultMetrics, register, Gauge } from "prom-client";
+collectDefaultMetrics();
+
+const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020);
+const NODE_NAME = process.env.NODE_NAME || "coordinator";
+const REGISTRY_FQDN = process.env.REGISTRY_FQDN || "localhost:5000";
+const REPO_NAME = process.env.REPO_NAME || "checkpoints";
+const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints";
+const REGISTRY_TLS_VERIFY = process.env.REGISTRY_TLS_VERIFY === "false" ? "false" : "true";
+
+const PLATFORM_ENABLED = ["1", "true"].includes(process.env.PLATFORM_ENABLED ?? "true");
+const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
+const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
+const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret";
+
+const logger = new SimpleLogger(`[${NODE_NAME}]`);
+
+class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode = true;
+
+  #logger = new SimpleLogger("[checkptr]");
+
+  async initialize() {
+    if (this.#initialized) {
+      return;
+    }
+
+    try {
+      await $`criu --version`;
+    } catch (error) {
+      this.#logger.error("No checkpoint support: Missing CRIU binary");
+      this.#canCheckpoint = false;
+      this.#initialized = true;
+      return;
+    }
+
+    if (this.#dockerMode) {
+      try {
+        await $`docker checkpoint`;
+      } catch (error) {
+        this.#logger.error(
+          "No checkpoint support: Docker needs to have experimental features enabled"
+        );
+        this.#canCheckpoint = false;
+        this.#initialized = true;
+        return;
+      }
+    }
+
+    this.#logger.log(
+      `Full checkpoint support with docker ${this.#dockerMode ? "enabled" : "disabled"}`
+    );
+
+    this.#initialized = true;
+    this.#canCheckpoint = true;
+  }
+
+  async checkpointAndPush(podName: string) {
+    await this.initialize();
+
+    if (!this.#canCheckpoint) {
+      return;
+    }
+
+    try {
+      const { path } = await this.#checkpointContainer(podName);
+      const { tag } = await this.#buildImage(path, podName);
+      const { destination } = await this.#pushImage(tag);
+
+      if (this.#dockerMode) {
+        this.#logger.log("checkpoint created:", { podName, path });
+      } else {
+        this.#logger.log("checkpointed and pushed image to:", destination);
+      }
+
+      return {
+        path,
+        tag,
+        destination: this.#dockerMode ? path : destination,
+        docker: this.#dockerMode,
+      };
+    } catch (error) {
+      this.#logger.error("checkpoint failed", error);
+      return;
+    }
+  }
+
+  async #checkpointContainer(podName: string) {
+    await this.initialize();
+
+    if (!this.#canCheckpoint) {
+      throw new Error("No checkpoint support");
+    }
+
+    if (this.#dockerMode) {
+      this.#logger.log("Checkpointing:", podName);
+
+      const path = randomUUID();
+
+      try {
+        this.#logger.debug(await $`docker checkpoint create --leave-running ${podName} ${path}`);
+      } catch (error: any) {
+        this.#logger.error(error.stderr);
+      }
+
+      return { path };
+    }
+
+    const containerId = this.#logger.debug(
+      // @ts-expect-error
+      await $`crictl ps`
+        .pipeStdout($({ stdin: "pipe" })`grep ${podName}`)
+        .pipeStdout($({ stdin: "pipe" })`cut -f1 ${"-d "}`)
+    );
+
+    if (!containerId.stdout) {
+      throw new Error("could not find container id");
+    }
+
+    const exportPath = `${CHECKPOINT_PATH}/${podName}.tar`;
+
+    this.#logger.debug(await $`crictl checkpoint --export=${exportPath} ${containerId}`);
+
+    return {
+      path: exportPath,
+    };
+  }
+
+  async #buildImage(checkpointPath: string, tag: string) {
+    await this.initialize();
+
+    if (!this.#canCheckpoint) {
+      throw new Error("No checkpoint support");
+    }
+
+    if (this.#dockerMode) {
+      // Nothing to do here
+      return { tag };
+    }
+
+    const container = this.#logger.debug(await $`buildah from scratch`);
+    this.#logger.debug(await $`buildah add ${container} ${checkpointPath} /`);
+    this.#logger.debug(
+      await $`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
+    );
+    this.#logger.debug(await $`buildah commit ${container} ${REGISTRY_FQDN}/${REPO_NAME}:${tag}`);
+    this.#logger.debug(await $`buildah rm ${container}`);
+
+    return {
+      tag,
+    };
+  }
+
+  async #pushImage(tag: string) {
+    await this.initialize();
+
+    if (!this.#canCheckpoint) {
+      throw new Error("No checkpoint support");
+    }
+
+    if (this.#dockerMode) {
+      // Nothing to do here
+      return { destination: "" };
+    }
+
+    const destination = `${REGISTRY_FQDN}/${REPO_NAME}:${tag}`;
+    this.#logger.debug(await $`buildah push --tls-verify=${REGISTRY_TLS_VERIFY} ${destination}`);
+
+    return {
+      destination,
+    };
+  }
+}
+
+class TaskCoordinator {
+  #httpServer: ReturnType<typeof createServer>;
+  #checkpointer = new Checkpointer();
+
+  #prodWorkerNamespace: Namespace<
+    ProdWorkerToCoordinatorEvents,
+    CoordinatorToProdWorkerEvents,
+    DefaultEventsMap,
+    ProdWorkerSocketData
+  >;
+  #platformSocket?: Socket<PlatformToCoordinatorEvents, CoordinatorToPlatformEvents>;
+
+  constructor(
+    private port: number,
+    private host = "0.0.0.0"
+  ) {
+    this.#httpServer = this.#createHttpServer();
+    this.#checkpointer.initialize();
+
+    const io = new Server(this.#httpServer);
+    this.#prodWorkerNamespace = this.#createProdWorkerNamespace(io);
+
+    this.#platformSocket = this.#createPlatformSocket();
+
+    const connectedTasksTotal = new Gauge({
+      name: "daemon_connected_tasks_total", // don't change this without updating dashboard config
+      help: "The number of tasks currently connected.",
+      collect: () => {
+        connectedTasksTotal.set(this.#prodWorkerNamespace.sockets.size);
+      },
+    });
+    register.registerMetric(connectedTasksTotal);
+  }
+
+  #createPlatformSocket() {
+    if (!PLATFORM_ENABLED) {
+      console.log("INFO: platform connection disabled");
+      return;
+    }
+
+    const socket: Socket<PlatformToCoordinatorEvents, CoordinatorToPlatformEvents> = io(
+      `ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/coordinator`,
+      {
+        transports: ["websocket"],
+        auth: {
+          token: PLATFORM_SECRET,
+        },
+      }
+    );
+
+    const logger = new SimpleLogger(`[platform][${socket.id ?? "NO_ID"}]`);
+
+    socket.on("connect", () => {
+      logger.log("connect");
+    });
+
+    socket.on("connect_error", (err) => {
+      logger.error(`connect_error: ${err.message}`);
+    });
+
+    socket.on("disconnect", () => {
+      logger.log("disconnect");
+    });
+
+    socket.on("RESUME", async (message) => {
+      logger.log("[RESUME]", message);
+
+      const taskSocket = await this.#getAttemptSocket(message.attemptId);
+
+      if (!taskSocket) {
+        logger.log("Socket for attempt not found", { attemptId: message.attemptId });
+        return;
+      }
+
+      taskSocket.emit("RESUME", message);
+    });
+
+    return socket;
+  }
+
+  async #getAttemptSocket(attemptId: string) {
+    const sockets = await this.#prodWorkerNamespace.fetchSockets();
+
+    for (const socket of sockets) {
+      if (socket.data.attemptId === attemptId) {
+        return socket;
+      }
+    }
+  }
+
+  #createProdWorkerNamespace(io: Server) {
+    const namespace: Namespace<
+      ProdWorkerToCoordinatorEvents,
+      CoordinatorToProdWorkerEvents,
+      DefaultEventsMap,
+      ProdWorkerSocketData
+    > = io.of("/prod-worker");
+
+    namespace.on("connection", async (socket) => {
+      const logger = new SimpleLogger(`[task][${socket.id}]`);
+
+      this.#platformSocket?.emit("LOG", {
+        version: "v1",
+        metadata: {
+          projectRef: socket.data.projectRef,
+          attemptId: socket.data.attemptId,
+        },
+        text: "connected",
+      });
+
+      logger.log("connected");
+
+      socket.on("disconnect", (reason, description) => {
+        logger.log("disconnect", { reason, description });
+
+        this.#platformSocket?.emit("LOG", {
+          version: "v1",
+          metadata: {
+            projectRef: socket.data.projectRef,
+            attemptId: socket.data.attemptId,
+          },
+          text: "disconnect",
+        });
+      });
+
+      socket.on("error", (error) => {
+        logger.error({ error });
+      });
+
+      socket.on("LOG", (message, callback) => {
+        logger.log("[LOG]", message.text);
+
+        callback();
+
+        this.#platformSocket?.emit("LOG", {
+          version: "v1",
+          metadata: { attemptId: socket.data.attemptId },
+          text: message.text,
+        });
+      });
+
+      socket.on("READY_FOR_EXECUTION", async (message) => {
+        logger.log("[READY_FOR_EXECUTION]", message);
+
+        const executionAck = await this.#platformSocket?.emitWithAck("READY_FOR_EXECUTION", {
+          version: "v1",
+          attemptId: message.attemptId,
+        });
+
+        if (!executionAck) {
+          logger.error("no execution ack", { attemptId: socket.data.attemptId });
+          return;
+        }
+
+        if (!executionAck.success) {
+          logger.error("execution unsuccessful", { attemptId: socket.data.attemptId });
+          return;
+        }
+
+        // FIXME: shouldn't wait for completion here
+        const completionAck = await socket.emitWithAck("EXECUTE_TASK_RUN", {
+          version: "v1",
+          payload: executionAck.payload,
+        });
+
+        logger.log("completed task", { completionId: completionAck.completion.id });
+
+        this.#platformSocket?.emit("TASK_RUN_COMPLETED", {
+          version: "v1",
+          execution: executionAck.payload.execution,
+          completion: completionAck.completion,
+        });
+      });
+
+      socket.on("TASK_HEARTBEAT", (message) => {
+        logger.log("[TASK_HEARTBEAT]", message);
+
+        this.#platformSocket?.emit("TASK_HEARTBEAT", message);
+      });
+
+      socket.on("WAIT_FOR_BATCH", (message) => {
+        logger.log("[WAIT_FOR_BATCH]", message);
+
+        // this.#checkpointer.checkpointAndPush(socket.data.podName);
+      });
+
+      socket.on("WAIT_FOR_DURATION", async (message, callback) => {
+        logger.log("[WAIT_FOR_DURATION]", message);
+
+        const checkpoint = await this.#checkpointer.checkpointAndPush(socket.data.podName);
+
+        if (!checkpoint) {
+          logger.error("Failed to checkpoint", { podName: socket.data.podName });
+          callback({ success: false });
+          return;
+        }
+
+        this.#platformSocket?.emit("CHECKPOINT_CREATED", {
+          version: "v1",
+          attemptId: socket.data.attemptId,
+          docker: checkpoint.docker,
+          location: checkpoint.destination,
+          reason: "WAIT_FOR_DURATION",
+        });
+
+        callback({ success: true });
+      });
+
+      socket.on("WAIT_FOR_TASK", (message) => {
+        logger.log("[WAIT_FOR_TASK]", message);
+
+        // this.#checkpointer.checkpointAndPush(socket.data.podName);
+      });
+
+      socket.on("INDEX_TASKS", async (message, callback) => {
+        logger.log("[INDEX_TASKS]", message);
+
+        const workerAck = await this.#platformSocket?.emitWithAck("CREATE_WORKER", {
+          version: "v1",
+          projectRef: socket.data.projectRef,
+          envId: socket.data.envId,
+          metadata: {
+            cliPackageVersion: socket.data.cliPackageVersion,
+            contentHash: socket.data.contentHash,
+            packageVersion: message.packageVersion,
+            tasks: message.tasks,
+          },
+        });
+
+        if (!workerAck) {
+          logger.debug("no worker ack while indexing", message);
+        }
+
+        callback({ success: !!workerAck?.success });
+      });
+    });
+
+    // auth middleware
+    namespace.use(async (socket, next) => {
+      const logger = new SimpleLogger(`[task][${socket.id}][auth]`);
+
+      function setSocketDataFromHeader(dataKey: keyof typeof socket.data, headerName: string) {
+        const value = socket.handshake.headers[headerName];
+        if (!value) {
+          logger.error(`missing required header: ${headerName}`);
+          throw new Error("missing header");
+        }
+        socket.data[dataKey] = Array.isArray(value) ? value[0] : value;
+      }
+
+      try {
+        setSocketDataFromHeader("podName", "x-pod-name");
+        setSocketDataFromHeader("contentHash", "x-trigger-content-hash");
+        setSocketDataFromHeader("cliPackageVersion", "x-trigger-cli-package-version");
+        setSocketDataFromHeader("projectRef", "x-trigger-project-ref");
+        setSocketDataFromHeader("attemptId", "x-trigger-attempt-id");
+        setSocketDataFromHeader("envId", "x-trigger-env-id");
+      } catch (error) {
+        return socket.disconnect(true);
+      }
+
+      logger.log("success", socket.data);
+
+      next();
+    });
+
+    return namespace;
+  }
+
+  #createHttpServer() {
+    const httpServer = createServer(async (req, res) => {
+      logger.log(`[${req.method}]`, req.url);
+
+      const reply = new HttpReply(res);
+
+      switch (req.url) {
+        case "/health": {
+          return reply.text("ok");
+        }
+        case "/metrics": {
+          return reply.text(await register.metrics(), 200, register.contentType);
+        }
+        case "/whoami": {
+          return reply.text(NODE_NAME);
+        }
+        case "/checkpoint": {
+          const body = await getTextBody(req);
+          await this.#checkpointer.checkpointAndPush(body);
+          return reply.text(`sent restore request: ${body}`);
+        }
+        default: {
+          return reply.empty(404);
+        }
+      }
+    });
+
+    httpServer.on("clientError", (err, socket) => {
+      socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+    });
+
+    httpServer.on("listening", () => {
+      logger.log("server listening on port", HTTP_SERVER_PORT);
+    });
+
+    return httpServer;
+  }
+
+  listen() {
+    this.#httpServer.listen(this.port, this.host);
+  }
+}
+
+const coordinator = new TaskCoordinator(HTTP_SERVER_PORT);
+coordinator.listen();

apps/coordinator/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../core/src/v3/*"],
+      "@trigger.dev/core-apps": ["../core-apps/src"],
+      "@trigger.dev/core-apps/*": ["../core-apps/src/*"]
+    }
+  }
+}

apps/docker-provider/.env.example

@@ -0,0 +1,4 @@
+HTTP_SERVER_PORT=8050
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret

apps/docker-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/docker-provider/Containerfile

@@ -0,0 +1,16 @@
+# syntax=docker/dockerfile:labs
+
+FROM node:18-slim AS base
+
+RUN apt-get update \
+  && apt-get install -y dumb-init
+
+FROM base
+
+WORKDIR /app
+
+COPY --chown=node dist/index.cjs /app/
+
+EXPOSE 8000
+
+ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.cjs" ]

apps/docker-provider/README.md

@@ -0,0 +1,3 @@
+# Docker provider
+
+The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions.

apps/docker-provider/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "docker-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "type": "module",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.cjs --platform=node",
+    "build:image": "docker build -f Containerfile . -t docker-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/core-apps": "workspace:*",
+    "execa": "^8.0.1",
+    "socket.io-client": "^4.7.4"
+  },
+  "devDependencies": {
+    "@types/node": "^18.19.8",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/docker-provider/src/index.ts

@@ -0,0 +1,335 @@
+import { createServer } from "node:http";
+import { $ } from "execa";
+import { io, Socket } from "socket.io-client";
+import {
+  clientWebsocketMessages,
+  Machine,
+  MessageCatalogToSocketIoEvents,
+  ProviderClientToServerEvents,
+  ProviderServerToClientEvents,
+  serverWebsocketMessages,
+  ZodMessageHandler,
+  ZodMessageSender,
+} from "@trigger.dev/core/v3";
+import { HttpReply, SimpleLogger, getTextBody, getRandomPortNumber } from "@trigger.dev/core-apps";
+
+const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
+const MACHINE_NAME = process.env.MACHINE_NAME || "local";
+
+const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
+const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
+const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
+
+const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
+
+const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
+
+interface TaskOperations {
+  create: (...args: any[]) => Promise<any>;
+  restore: (...args: any[]) => Promise<any>;
+  delete: (...args: any[]) => Promise<any>;
+  get: (...args: any[]) => Promise<any>;
+  index: (...args: any[]) => Promise<any>;
+}
+
+class DockerTaskOperations implements TaskOperations {
+  async index(opts: { contentHash: string; imageTag: string; envId: string }) {
+    const containerName = this.#getIndexContainerName(opts.contentHash);
+
+    const { exitCode } = logger.debug(
+      await $`docker run --rm -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e INDEX_TASKS=true --network=host --pull=never --name=${containerName} ${opts.imageTag}`
+    );
+
+    if (exitCode !== 0) {
+      throw new Error("docker run command failed");
+    }
+  }
+
+  async create(opts: { attemptId: string; image: string; machine: Machine; envId: string }) {
+    const containerName = this.#getRunContainerName(opts.attemptId);
+
+    const { exitCode } = logger.debug(
+      await $`docker run -d -e COORDINATOR_PORT=${COORDINATOR_PORT} -e POD_NAME=${containerName} -e TRIGGER_ENV_ID=${opts.envId} -e TRIGGER_ATTEMPT_ID=${opts.attemptId} --network=host --pull=never --name=${containerName} ${opts.image}`
+    );
+
+    if (exitCode !== 0) {
+      throw new Error("docker run command failed");
+    }
+  }
+
+  async restore(opts: {
+    attemptId: string;
+    runId: string;
+    image: string;
+    name: string;
+    checkpointId: string;
+    machine: Machine;
+  }) {
+    const containerName = this.#getRunContainerName(opts.attemptId);
+
+    const { exitCode } = logger.debug(
+      await $`docker start --checkpoint=${opts.checkpointId} ${containerName}`
+    );
+
+    if (exitCode !== 0) {
+      throw new Error("docker start command failed");
+    }
+  }
+
+  async delete(opts: { runId: string }) {
+    logger.log("noop: delete");
+  }
+
+  async get(opts: { runId: string }) {
+    logger.log("noop: get");
+  }
+
+  #getIndexContainerName(contentHash: string) {
+    return `task-index-${contentHash}`;
+  }
+
+  #getRunContainerName(attemptId: string) {
+    return `task-run-${attemptId}`;
+  }
+}
+
+interface Provider {
+  tasks: TaskOperations;
+}
+
+type DockerProviderOptions = {
+  tasks: DockerTaskOperations;
+  host?: string;
+  port: number;
+};
+
+class DockerProvider implements Provider {
+  tasks: DockerTaskOperations;
+
+  #httpServer: ReturnType<typeof createServer>;
+  #platformSocket: Socket<ProviderServerToClientEvents, ProviderClientToServerEvents>;
+
+  constructor(private options: DockerProviderOptions) {
+    this.tasks = options.tasks;
+    this.#httpServer = this.#createHttpServer();
+    this.#platformSocket = this.#createPlatformSocket();
+    this.#createSharedQueueSocket();
+  }
+
+  #createSharedQueueSocket() {
+    const socket: Socket<
+      MessageCatalogToSocketIoEvents<typeof serverWebsocketMessages>,
+      MessageCatalogToSocketIoEvents<typeof clientWebsocketMessages>
+    > = io(`ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/shared-queue`, {
+      transports: ["websocket"],
+      auth: {
+        token: PLATFORM_SECRET,
+      },
+    });
+
+    const logger = new SimpleLogger(`[shared-queue][${socket.id ?? "NO_ID"}]`);
+
+    socket.on("connect_error", (err) => {
+      logger.error(`connect_error: ${err.message}`);
+    });
+
+    socket.on("connect", () => {
+      logger.log("connect");
+    });
+
+    socket.on("disconnect", () => {
+      logger.log("disconnect");
+    });
+
+    const sender = new ZodMessageSender({
+      schema: clientWebsocketMessages,
+      sender: async (message) => {
+        return new Promise((resolve, reject) => {
+          try {
+            const { type, ...payload } = message;
+            socket.emit(type, payload as any);
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+      },
+    });
+
+    const handler = new ZodMessageHandler({
+      schema: serverWebsocketMessages,
+      messages: {
+        SERVER_READY: async (payload) => {
+          logger.log("received SERVER_READY", payload);
+
+          // TODO: create new schema without worker requirement
+          await sender.send("READY_FOR_TASKS", {
+            backgroundWorkerId: "placeholder",
+          });
+        },
+        BACKGROUND_WORKER_MESSAGE: async (payload) => {
+          logger.log("received BACKGROUND_WORKER_MESSAGE", payload);
+
+          if (payload.data.type === "SCHEDULE_ATTEMPT") {
+            this.tasks.create({
+              envId: payload.data.envId,
+              attemptId: payload.data.id,
+              image: payload.data.image,
+              machine: {},
+            });
+          }
+        },
+      },
+    });
+    handler.registerHandlers(socket, logger.log.bind(logger));
+
+    return socket;
+  }
+
+  #createPlatformSocket() {
+    const socket: Socket<ProviderServerToClientEvents, ProviderClientToServerEvents> = io(
+      `ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/provider`,
+      {
+        transports: ["websocket"],
+        auth: {
+          token: PLATFORM_SECRET,
+        },
+        extraHeaders: {
+          "x-trigger-provider-type": "docker",
+        },
+      }
+    );
+
+    const logger = new SimpleLogger(`[platform][${socket.id ?? "NO_ID"}]`);
+
+    socket.on("connect_error", (err) => {
+      logger.error(`connect_error: ${err.message}`);
+    });
+
+    socket.on("connect", () => {
+      logger.log("connect");
+    });
+
+    socket.on("disconnect", () => {
+      logger.log("disconnect");
+    });
+
+    socket.on("GET", async (message) => {
+      logger.log("[GET]", message);
+
+      this.tasks.get({ runId: message.name });
+    });
+
+    socket.on("DELETE", async (message, callback) => {
+      logger.log("[DELETE]", message);
+
+      callback({
+        message: "delete request received",
+      });
+
+      this.tasks.delete({ runId: message.name });
+    });
+
+    socket.on("INDEX", async (message) => {
+      logger.log("[INDEX]", message);
+      try {
+        await this.tasks.index({
+          contentHash: message.contentHash,
+          imageTag: message.imageTag,
+          envId: message.envId,
+        });
+      } catch (error) {
+        logger.error("task index failed", error);
+      }
+    });
+
+    socket.on("RESTORE", async (message) => {
+      logger.log("[RESTORE]", message);
+      // await this.tasks.restore({});
+    });
+
+    socket.on("HEALTH", async (message) => {
+      logger.log("[HEALTH]", message);
+    });
+
+    return socket;
+  }
+
+  #createHttpServer() {
+    const httpServer = createServer(async (req, res) => {
+      logger.log(`[${req.method}]`, req.url);
+
+      const reply = new HttpReply(res);
+
+      switch (req.url) {
+        case "/health": {
+          return reply.text("ok");
+        }
+        case "/whoami": {
+          return reply.text(`${MACHINE_NAME}`);
+        }
+        case "/close": {
+          this.#platformSocket.close();
+          return reply.text("platform socket closed");
+        }
+        case "/delete": {
+          const body = await getTextBody(req);
+
+          await this.tasks.delete({ runId: body });
+
+          return reply.text(`sent delete request: ${body}`);
+        }
+        case "/invoke": {
+          const body = await getTextBody(req);
+
+          await this.tasks.create({
+            attemptId: body,
+            envId: "placeholder",
+            image: body,
+            machine: {
+              cpu: "1",
+              memory: "100Mi",
+            },
+          });
+
+          return reply.text(`sent restore request: ${body}`);
+        }
+        case "/restore": {
+          const body = await getTextBody(req);
+
+          const items = body.split("&");
+          const image = items[0];
+          const baseImageTag = items[1] ?? image;
+
+          // await this.tasks.restore({});
+
+          return reply.text(`sent restore request: ${body}`);
+        }
+        default: {
+          return reply.empty(404);
+        }
+      }
+    });
+
+    httpServer.on("clientError", (err, socket) => {
+      socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+    });
+
+    httpServer.on("listening", () => {
+      logger.log("server listening on port", this.options.port);
+    });
+
+    return httpServer;
+  }
+
+  listen() {
+    this.#httpServer.listen(this.options.port, this.options.host ?? "0.0.0.0");
+  }
+}
+
+const provider = new DockerProvider({
+  port: HTTP_SERVER_PORT,
+  tasks: new DockerTaskOperations(),
+});
+
+provider.listen();

apps/docker-provider/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../core/src/v3/*"],
+      "@trigger.dev/core-apps": ["../core-apps/src"],
+      "@trigger.dev/core-apps/*": ["../core-apps/src/*"]
+    }
+  }
+}

apps/kubernetes-provider/.env.example

@@ -0,0 +1,7 @@
+HTTP_SERVER_PORT=8060
+
+PLATFORM_WS_PORT=8003
+PLATFORM_SECRET=provider-secret
+
+REGISTRY_FQDN=docker.io
+REPO_NAME=task

apps/kubernetes-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/kubernetes-provider/Containerfile

@@ -0,0 +1,48 @@
+FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine
+
+WORKDIR /app
+
+FROM node-18-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-18-alpine AS base
+
+RUN apk add --no-cache dumb-init
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter '@trigger.dev/core*' build
+RUN pnpm run -r --filter kubernetes-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.cjs ./index.cjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.cjs" ]

apps/kubernetes-provider/README.md

@@ -0,0 +1,3 @@
+# Kubernetes provider
+
+The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions.

apps/kubernetes-provider/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "kubernetes-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "type": "module",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.cjs --platform=node",
+    "build:image": "docker build -f Containerfile . -t kubernetes-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@kubernetes/client-node": "^0.20.0",
+    "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/core-apps": "workspace:*",
+    "socket.io-client": "^4.7.4"
+  },
+  "devDependencies": {
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/kubernetes-provider/src/index.ts

@@ -0,0 +1,561 @@
+import { randomUUID } from "node:crypto";
+import { createServer } from "node:http";
+import k8s, { BatchV1Api, CoreV1Api, V1Job, V1Pod } from "@kubernetes/client-node";
+import { io, Socket } from "socket.io-client";
+import {
+  Machine,
+  ProviderClientToServerEvents,
+  ProviderServerToClientEvents,
+} from "@trigger.dev/core/v3";
+import { HttpReply, SimpleLogger, getTextBody } from "@trigger.dev/core-apps";
+
+const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
+
+const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8000);
+const NODE_NAME = process.env.NODE_NAME || "some-node";
+const POD_NAME = process.env.POD_NAME || "k8s-provider";
+
+const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
+const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 5080;
+const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
+
+const REGISTRY_FQDN = process.env.REGISTRY_FQDN || "localhost:5000";
+const REPO_NAME = process.env.REPO_NAME || "test";
+
+const logger = new SimpleLogger(`[${NODE_NAME}]`);
+
+type Namespace = {
+  metadata: {
+    name: string;
+  };
+};
+
+interface TaskOperations {
+  create: (...args: any[]) => Promise<any>;
+  restore: (...args: any[]) => Promise<any>;
+  delete: (...args: any[]) => Promise<any>;
+  get: (...args: any[]) => Promise<any>;
+  index: (...args: any[]) => Promise<any>;
+}
+
+class KubernetesTaskOperations implements TaskOperations {
+  #namespace: Namespace;
+  #k8sApi: {
+    core: CoreV1Api;
+    batch: BatchV1Api;
+  };
+
+  constructor(namespace = "default") {
+    this.#namespace = {
+      metadata: {
+        name: namespace,
+      },
+    };
+
+    this.#k8sApi = this.#createK8sApi();
+  }
+
+  async index(opts: { contentHash: string; imageTag: string }) {
+    await this.#createJob(
+      {
+        metadata: {
+          name: `task-index-${opts.contentHash}`,
+          namespace: this.#namespace.metadata.name,
+        },
+        spec: {
+          completions: 1,
+          template: {
+            metadata: {
+              labels: {
+                app: "task-index",
+              },
+            },
+            spec: {
+              restartPolicy: "Never",
+              imagePullSecrets: [
+                {
+                  name: "registry-trigger",
+                },
+              ],
+              containers: [
+                {
+                  name: opts.contentHash,
+                  image: opts.imageTag,
+                  ports: [
+                    {
+                      containerPort: 8000,
+                    },
+                  ],
+                  resources: {
+                    limits: {
+                      cpu: "100m",
+                      memory: "50Mi",
+                    },
+                  },
+                  env: [
+                    {
+                      name: "DEBUG",
+                      value: "true",
+                    },
+                    {
+                      name: "INDEX_TASKS",
+                      value: "true",
+                    },
+                    {
+                      name: "HTTP_SERVER_PORT",
+                      value: "8000",
+                    },
+                    {
+                      name: "POD_NAME",
+                      valueFrom: {
+                        fieldRef: {
+                          fieldPath: "metadata.name",
+                        },
+                      },
+                    },
+                    {
+                      name: "COORDINATOR_HOST",
+                      valueFrom: {
+                        fieldRef: {
+                          fieldPath: "status.hostIP",
+                        },
+                      },
+                    },
+                    {
+                      name: "MACHINE_NAME",
+                      valueFrom: {
+                        fieldRef: {
+                          fieldPath: "spec.nodeName",
+                        },
+                      },
+                    },
+                  ],
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async create(opts: { runId: string; image: string; machine: Machine }) {
+    await this.#createPod(
+      {
+        metadata: {
+          name: `${opts.runId}-${randomUUID().slice(0, 5)}`,
+          namespace: this.#namespace.metadata.name,
+        },
+        spec: {
+          restartPolicy: "Never",
+          containers: [
+            {
+              name: opts.runId,
+              image: this.#getImageFromRunId(opts.runId),
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              // resources: {
+              //   limits: opts.machine,
+              // },
+              env: [
+                {
+                  name: "DEBUG",
+                  value: "true",
+                },
+                {
+                  name: "POD_NAME",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "metadata.name",
+                    },
+                  },
+                },
+                {
+                  name: "COORDINATOR_HOST",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "status.hostIP",
+                    },
+                  },
+                },
+                {
+                  name: "NODE_NAME",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "spec.nodeName",
+                    },
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async restore(opts: {
+    runId: string;
+    image: string;
+    name: string;
+    checkpointId: string;
+    machine: Machine;
+  }) {
+    await this.#createPod(
+      {
+        metadata: {
+          name: opts.name,
+          namespace: this.#namespace.metadata.name,
+        },
+        spec: {
+          imagePullSecrets: [
+            {
+              name: "regcred",
+            },
+          ],
+          initContainers: [
+            {
+              name: "pull-base-image",
+              image: this.#getRestoreImage(opts.runId, opts.checkpointId),
+              command: ["sleep", "0"],
+            },
+          ],
+          containers: [
+            {
+              name: opts.runId,
+              image: this.#getImageFromRunId(opts.runId),
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              // resources: {
+              //   limits: opts.machine,
+              // },
+              lifecycle: {
+                postStart: {
+                  httpGet: {
+                    path: "/connect",
+                    port: 8000,
+                  },
+                },
+              },
+              env: [
+                {
+                  name: "DEBUG",
+                  value: "true",
+                },
+                {
+                  name: "POD_NAME",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "metadata.name",
+                    },
+                  },
+                },
+                {
+                  name: "COORDINATOR_HOST",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "status.hostIP",
+                    },
+                  },
+                },
+                {
+                  name: "NODE_NAME",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "spec.nodeName",
+                    },
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.#deletePod({
+      podName: opts.runId,
+      namespace: this.#namespace,
+    });
+  }
+
+  async get(opts: { runId: string }) {
+    await this.#getPod(opts.runId, this.#namespace);
+  }
+
+  #getImageFromRunId(runId: string) {
+    return `${REGISTRY_FQDN}/${REPO_NAME}:${runId}`;
+  }
+
+  #getRestoreImage(runId: string, checkpointId: string) {
+    return `${REGISTRY_FQDN}/${REPO_NAME}:${checkpointId}`;
+  }
+
+  #createK8sApi() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (RUNTIME_ENV === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (RUNTIME_ENV === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+    };
+  }
+
+  async #createPod(pod: V1Pod, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod);
+      logger.debug(res.body);
+    } catch (err: any) {
+      if ("body" in err) {
+        logger.error(err.body);
+      } else {
+        logger.error(err);
+      }
+    }
+  }
+
+  async #deletePod(opts: { podName: string; namespace: Namespace }) {
+    try {
+      const res = await this.#k8sApi.core.deleteNamespacedPod(
+        opts.podName,
+        opts.namespace.metadata.name
+      );
+      logger.debug(res.body);
+    } catch (err: any) {
+      if ("body" in err) {
+        logger.error(err.body);
+      } else {
+        logger.error(err);
+      }
+    }
+  }
+
+  async #getPod(podName: string, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.readNamespacedPod(podName, namespace.metadata.name);
+      logger.debug(res.body);
+      return res.body;
+    } catch (err: any) {
+      if ("body" in err) {
+        logger.error(err.body);
+      } else {
+        logger.error(err);
+      }
+    }
+  }
+
+  async #createJob(job: V1Job, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job);
+      logger.debug(res.body);
+    } catch (err: any) {
+      if ("body" in err) {
+        logger.error(err.body);
+      } else {
+        logger.error(err);
+      }
+    }
+  }
+}
+
+interface Provider {
+  tasks: TaskOperations;
+}
+
+type KubernetesProviderOptions = {
+  tasks: KubernetesTaskOperations;
+  host?: string;
+  port: number;
+};
+
+class KubernetesProvider implements Provider {
+  tasks: KubernetesTaskOperations;
+
+  #httpServer: ReturnType<typeof createServer>;
+  #platformSocket: Socket<ProviderServerToClientEvents, ProviderClientToServerEvents>;
+
+  constructor(private options: KubernetesProviderOptions) {
+    this.tasks = options.tasks;
+    this.#httpServer = this.#createHttpServer();
+    this.#platformSocket = this.#createPlatformSocket();
+  }
+
+  #createPlatformSocket() {
+    const socket: Socket<ProviderServerToClientEvents, ProviderClientToServerEvents> = io(
+      `ws://${PLATFORM_HOST}:${PLATFORM_WS_PORT}/provider`,
+      {
+        transports: ["websocket"],
+        auth: {
+          token: PLATFORM_SECRET,
+        },
+        extraHeaders: {
+          "x-trigger-provider-type": "kubernetes",
+        },
+      }
+    );
+
+    const logger = new SimpleLogger(`[platform][${socket.id ?? "NO_ID"}]`);
+
+    socket.on("connect_error", (err) => {
+      logger.error(`connect_error: ${err.message}`);
+    });
+
+    socket.on("connect", () => {
+      logger.log("connect");
+    });
+
+    socket.on("disconnect", () => {
+      logger.log("disconnect");
+    });
+
+    socket.on("GET", async (message) => {
+      logger.log("[GET]", message);
+      this.tasks.get({ runId: message.name });
+    });
+
+    socket.on("DELETE", async (message, callback) => {
+      logger.log("[DELETE]", message);
+
+      callback({
+        message: "delete request received",
+      });
+
+      this.tasks.delete({ runId: message.name });
+    });
+
+    socket.on("INDEX", async (message) => {
+      logger.log("[INDEX]", message);
+
+      await this.tasks.index({
+        contentHash: message.contentHash,
+        imageTag: message.imageTag,
+      });
+    });
+
+    socket.on("INVOKE", async (message) => {
+      logger.log("[INVOKE]", message);
+
+      await this.tasks.create({
+        runId: message.name,
+        image: message.name,
+        machine: message.machine,
+      });
+    });
+
+    socket.on("RESTORE", async (message) => {
+      logger.log("[RESTORE]", message);
+
+      // await this.tasks.restore({});
+    });
+
+    socket.on("HEALTH", async (message) => {
+      logger.log("[HEALTH]", message);
+    });
+
+    return socket;
+  }
+
+  #createHttpServer() {
+    const httpServer = createServer(async (req, res) => {
+      logger.log(`[${req.method}]`, req.url);
+
+      const reply = new HttpReply(res);
+
+      switch (req.url) {
+        case "/health": {
+          return reply.text("ok");
+        }
+        case "/whoami": {
+          return reply.text(`${POD_NAME}`);
+        }
+        case "/close": {
+          this.#platformSocket.close();
+          return reply.text("platform socket closed");
+        }
+        case "/delete": {
+          const body = await getTextBody(req);
+
+          await this.tasks.delete({ runId: body });
+
+          return reply.text(`sent delete request: ${body}`);
+        }
+        case "/invoke": {
+          const body = await getTextBody(req);
+
+          await this.tasks.create({
+            runId: body,
+            image: body,
+            machine: {
+              cpu: "1",
+              memory: "100Mi",
+            },
+          });
+
+          return reply.text(`sent restore request: ${body}`);
+        }
+        case "/restore": {
+          const body = await getTextBody(req);
+
+          const items = body.split("&");
+          const image = items[0];
+          const baseImageTag = items[1] ?? image;
+
+          await this.tasks.restore({
+            runId: image,
+            name: `${image}-restore`,
+            image,
+            checkpointId: baseImageTag,
+            machine: {
+              cpu: "1",
+              memory: "100Mi",
+            },
+          });
+
+          return reply.text(`sent restore request: ${body}`);
+        }
+        default: {
+          return reply.empty(404);
+        }
+      }
+    });
+
+    httpServer.on("clientError", (err, socket) => {
+      socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+    });
+
+    httpServer.on("listening", () => {
+      logger.log("server listening on port", this.options.port);
+    });
+
+    return httpServer;
+  }
+
+  listen() {
+    this.#httpServer.listen(this.options.port, this.options.host ?? "0.0.0.0");
+  }
+}
+
+const provider = new KubernetesProvider({
+  port: HTTP_SERVER_PORT,
+  tasks: new KubernetesTaskOperations(),
+});
+
+provider.listen();

apps/kubernetes-provider/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../core/src/v3/*"],
+      "@trigger.dev/core-apps": ["../core-apps/src"],
+      "@trigger.dev/core-apps/*": ["../core-apps/src/*"]
+    }
+  }
+}

apps/webapp/app/entry.server.tsx

@@ -196,3 +196,4 @@ function logError(error: unknown, request?: Request) {
 const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer);
 
 export { wss } from "./v3/handleWebsockets.server";
+export { socketIo } from "./v3/handleSocketIo.server";

apps/webapp/app/env.server.ts

@@ -72,6 +72,10 @@ const EnvironmentSchema = z.object({
   V3_ENABLED: z.string().default("false"),
   OTLP_EXPORTER_TRACES_URL: z.string().optional(),
   LOG_TELEMETRY: z.string().default("true"),
+  IMAGE_REGISTRY: z.string().default("docker.io"),
+  IMAGE_REPO: z.string().default("task"),
+  PROVIDER_SECRET: z.string().default("provider-secret"),
+  COORDINATOR_SECRET: z.string().default("coordinator-secret"),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/routes/api.v1.projects.$projectRef.image-details.ts

@@ -0,0 +1,54 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { CreateImageDetailsService } from "~/v3/services/createImageDetails.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  const { projectRef } = parsedParams.data;
+
+  const rawBody = await request.json();
+  const body = CreateImageDetailsRequestBody.safeParse(rawBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
+  }
+
+  const service = new CreateImageDetailsService();
+
+  const imageDetails = await service.call(projectRef, authenticatedEnv, body.data);
+
+  return json(
+    {
+      id: imageDetails.friendlyId,
+      contentHash: imageDetails.contentHash,
+    },
+    { status: 200 }
+  );
+}

apps/webapp/app/routes/api.v1.projects.$projectRef.prod.ts

@@ -0,0 +1,62 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { GetProjectDevResponse } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { logger } from "~/services/logger.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+});
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  logger.info("projects get prod env", { url: request.url });
+
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing Access Token" }, { status: 401 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid Params" }, { status: 400 });
+  }
+
+  const projectRef = parsedParams.data.projectRef;
+
+  const project = await prisma.project.findUnique({
+    where: {
+      externalRef: projectRef,
+      organization: {
+        members: {
+          some: {
+            userId: authenticationResult.userId,
+          },
+        },
+      },
+    },
+    include: {
+      environments: {
+        where: {
+          slug: "prod",
+        },
+      },
+    },
+  });
+
+  if (!project) {
+    return json({ error: "Project not found" }, { status: 404 });
+  }
+
+  const prodEnvironment = project.environments[0];
+
+  const result: GetProjectDevResponse = {
+    apiKey: prodEnvironment.apiKey,
+    name: project.name,
+  };
+
+  return json(result);
+}

apps/webapp/app/services/worker.server.ts

@@ -29,6 +29,7 @@ import { InvokeEphemeralDispatcherService } from "./dispatchers/invokeEphemeralE
 import { ResumeRunService } from "./runs/resumeRun.server";
 import { executionRateLimiter } from "./runExecutionRateLimiter.server";
 import { DeliverWebhookRequestService } from "./sources/deliverWebhookRequest.server";
+import { IndexTasksService } from "~/v3/services/indexTasks.server";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -102,6 +103,10 @@ const workerCatalog = {
   resumeRun: z.object({
     id: z.string(),
   }),
+  // v3 tasks
+  indexTasks: z.object({
+    id: z.string(),
+  }),
 };
 
 const executionWorkerCatalog = {
@@ -425,6 +430,16 @@ function getWorkerQueue() {
         handler: async (payload, job) => {
           const service = new ResumeRunService();
 
+          return await service.call(payload.id);
+        },
+      },
+      // v3 tasks
+      indexTasks: {
+        priority: 0,
+        maxAttempts: 5,
+        handler: async (payload, job) => {
+          const service = new IndexTasksService();
+
           return await service.call(payload.id);
         },
       },

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -4,6 +4,7 @@ import { $transaction, prisma } from "~/db.server";
 import { getSecretStore } from "~/services/secrets/secretStore.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { EnvironmentVariable, ProjectEnvironmentVariable, Repository, Result } from "./repository";
+import { env } from "~/env.server";
 
 function secretKeyProjectPrefix(projectId: string) {
   return `environmentvariable:${projectId}:`;
@@ -402,7 +403,34 @@ export class EnvironmentVariablesRepository implements Repository {
     return this.getEnvironmentVariables(projectId, environmentId);
   }
 
-  async getEnvironmentVariables(
+  async #getTriggerEnvironmentVariables(environmentId: string): Promise<EnvironmentVariable[]> {
+    const environment = await this.prismaClient.runtimeEnvironment.findFirst({
+      where: {
+        id: environmentId,
+      },
+    });
+
+    if (!environment) {
+      return [];
+    }
+
+    if (environment.type === "DEVELOPMENT") {
+      return [];
+    }
+
+    return [
+      {
+        key: "TRIGGER_API_KEY",
+        value: environment.apiKey,
+      },
+      {
+        key: "TRIGGER_API_URL",
+        value: env.APP_ORIGIN,
+      },
+    ];
+  }
+
+  async #getSecretEnvironmentVariables(
     projectId: string,
     environmentId: string
   ): Promise<EnvironmentVariable[]> {
@@ -424,6 +452,16 @@ export class EnvironmentVariablesRepository implements Repository {
     });
   }
 
+  async getEnvironmentVariables(
+    projectId: string,
+    environmentId: string
+  ): Promise<EnvironmentVariable[]> {
+    const secretEnvVars = await this.#getSecretEnvironmentVariables(projectId, environmentId);
+    const triggerEnvVars = await this.#getTriggerEnvironmentVariables(environmentId);
+
+    return [...secretEnvVars, ...triggerEnvVars];
+  }
+
   async delete(projectId: string, userId: string, options: { id: string }): Promise<Result> {
     const project = await this.prismaClient.project.findUnique({
       where: {

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -0,0 +1,132 @@
+import {
+  CoordinatorToPlatformMessages,
+  PlatformToCoordinatorMessages,
+  PlatformToProviderMessages,
+  ProviderToPlatformMessages,
+  ZodNamespace,
+  clientWebsocketMessages,
+  serverWebsocketMessages,
+} from "@trigger.dev/core/v3";
+import { Server } from "socket.io";
+import { env } from "~/env.server";
+import { singleton } from "~/utils/singleton";
+import { SharedSocketConnection } from "./sharedSocketConnection";
+import { CreateCheckpointService } from "./services/createCheckpoint.server";
+import { sharedQueueTasks } from "./marqs/sharedQueueConsumer.server";
+import { CompleteAttemptService } from "./services/completeAttempt.server";
+import { CreateBackgroundWorkerService } from "./services/createBackgroundWorker.server";
+import { logger } from "~/services/logger.server";
+import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+
+export const socketIo = singleton("socketIo", initalizeIoServer);
+
+function initalizeIoServer() {
+  const io = new Server();
+
+  io.on("connection", (socket) => {
+    console.log(`[socket.io][${socket.id}] connection at url: ${socket.request.url}`);
+  });
+
+  const coordinatorNamespace = createCoordinatorNamespace(io);
+  const providerNamespace = createProviderNamespace(io);
+  const sharedQueueConsumerNamespace = createSharedQueueConsumerNamespace(io);
+
+  return {
+    io,
+    coordinatorNamespace,
+    providerNamespace,
+    sharedQueueConsumerNamespace,
+  };
+}
+
+function createCoordinatorNamespace(io: Server) {
+  const coordinator = new ZodNamespace({
+    io,
+    name: "coordinator",
+    authToken: env.COORDINATOR_SECRET,
+    clientMessages: CoordinatorToPlatformMessages,
+    serverMessages: PlatformToCoordinatorMessages,
+    messageHandler: {
+      READY_FOR_EXECUTION: async (message) => {
+        const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt(message.attemptId);
+
+        if (!payload) {
+          return { success: false };
+        } else {
+          return { success: true, payload };
+        }
+      },
+      TASK_RUN_COMPLETED: async (message) => {
+        const completeAttempt = new CompleteAttemptService();
+        await completeAttempt.call(message.completion, message.execution);
+      },
+      TASK_HEARTBEAT: async (message) => {
+        await sharedQueueTasks.taskHeartbeat(message.attemptFriendlyId);
+      },
+      CHECKPOINT_CREATED: async (message) => {
+        const createCheckpoint = new CreateCheckpointService();
+        await createCheckpoint.call(message);
+      },
+      CREATE_WORKER: async (message) => {
+        try {
+          const environment = await findEnvironmentById(message.envId);
+
+          if (!environment) {
+            logger.error("Environment not found", { id: message.envId });
+            return { success: false };
+          }
+
+          const createCheckpoint = new CreateBackgroundWorkerService();
+          await createCheckpoint.call(message.projectRef, environment, {
+            localOnly: true,
+            metadata: message.metadata,
+          });
+
+          return { success: true };
+        } catch (error) {
+          logger.error("Error while creating worker", { error });
+          return { success: false };
+        }
+      },
+    },
+  });
+
+  return coordinator.namespace;
+}
+
+function createProviderNamespace(io: Server) {
+  const provider = new ZodNamespace({
+    io,
+    name: "provider",
+    authToken: env.PROVIDER_SECRET,
+    clientMessages: ProviderToPlatformMessages,
+    serverMessages: PlatformToProviderMessages,
+  });
+
+  return provider.namespace;
+}
+
+function createSharedQueueConsumerNamespace(io: Server) {
+  const sharedQueue = new ZodNamespace({
+    io,
+    name: "shared-queue",
+    authToken: env.PROVIDER_SECRET,
+    clientMessages: clientWebsocketMessages,
+    serverMessages: serverWebsocketMessages,
+    onConnection: async (socket, handler, sender, logger) => {
+      const sharedSocketConnection = new SharedSocketConnection(
+        sharedQueue.namespace,
+        socket,
+        logger
+      );
+
+      sharedSocketConnection.onClose.attach((closeEvent) => {
+        logger("Socket closed", { closeEvent });
+      });
+
+      await sharedSocketConnection.initialize();
+    },
+  });
+
+  return sharedQueue.namespace;
+}

apps/webapp/app/v3/marqs.server.ts

@@ -29,7 +29,7 @@ const constants = {
   QUEUE_PART: "queue",
   CONCURRENCY_KEY_PART: "ck",
   MESSAGE_PART: "message",
-};
+} as const;
 
 const MessagePayload = z.object({
   version: z.literal("1"),
@@ -259,6 +259,50 @@ export class MarQS {
     );
   }
 
+  public async replaceMessage(
+    messageId: string,
+    messageData: Record<string, unknown>,
+    timestamp?: number
+  ) {
+    return this.#trace(
+      "replaceMessage",
+      async (span) => {
+        const oldMessage = await this.#readMessage(messageId);
+
+        if (!oldMessage) {
+          return;
+        }
+
+        span.setAttributes({
+          [SemanticAttributes.QUEUE]: oldMessage.queue,
+          [SemanticAttributes.MESSAGE_ID]: oldMessage.messageId,
+          [SemanticAttributes.CONCURRENCY_KEY]: oldMessage.concurrencyKey,
+          [SemanticAttributes.PARENT_QUEUE]: oldMessage.parentQueue,
+        });
+
+        await this.#callAcknowledgeMessage({
+          messageKey: `${constants.MESSAGE_PART}:${messageId}`,
+          visibilityQueue: constants.MESSAGE_VISIBILITY_TIMEOUT_QUEUE,
+          concurrencyKey: `${oldMessage.queue}:${constants.CURRENT_CONCURRENCY_PART}`,
+          messageId,
+        });
+
+        const newMessage: MessagePayload = {
+          version: "1",
+          data: messageData,
+          queue: oldMessage.queue,
+          concurrencyKey: oldMessage.concurrencyKey,
+          timestamp: timestamp ?? Date.now(),
+          messageId,
+          parentQueue: oldMessage.parentQueue,
+        };
+
+        await this.#callEnqueueMessage(newMessage);
+      },
+      { kind: SpanKind.CONSUMER }
+    );
+  }
+
   async #trace<T>(
     name: string,
     fn: (span: Span, abort: () => void) => Promise<T>,
@@ -783,9 +827,9 @@ local currentTime = tonumber(ARGV[3])
 local messageScore = tonumber(ARGV[4])
 
 -- Check to see if the message is still in the visibilityQueue
-local messageVisibility = redis.call('ZSCORE', visibilityQueue, messageId)
+local messageVisibility = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0
 
-if messageVisibility == nil then
+if messageVisibility == 0 then
     return
 end
 
@@ -820,9 +864,9 @@ local milliseconds = tonumber(ARGV[2])
 local maxVisibilityTimeout = tonumber(ARGV[3])
 
 -- Get the current visibility timeout
-local currentVisibilityTimeout = redis.call('ZSCORE', visibilityQueue, messageId)
+local currentVisibilityTimeout = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0
 
-if currentVisibilityTimeout == nil then
+if currentVisibilityTimeout == 0 then
     return
 end
 

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -0,0 +1,851 @@
+import { Context, ROOT_CONTEXT, Span, SpanKind, context, trace } from "@opentelemetry/api";
+import {
+  ProdTaskRunExecution,
+  ProdTaskRunExecutionPayload,
+  TaskRunError,
+  TaskRunExecution,
+  TaskRunExecutionResult,
+  TaskRunFailedExecutionResult,
+  TaskRunSuccessfulExecutionResult,
+  ZodMessageSender,
+  serverWebsocketMessages,
+} from "@trigger.dev/core/v3";
+import { BackgroundWorker, BackgroundWorkerTask } from "@trigger.dev/database";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { logger } from "~/services/logger.server";
+import { generateFriendlyId } from "../friendlyIdentifiers";
+import { marqs } from "../marqs.server";
+import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
+import { CancelAttemptService } from "../services/cancelAttempt.server";
+import { socketIo } from "../handleSocketIo.server";
+import { singleton } from "~/utils/singleton";
+
+const tracer = trace.getTracer("sharedQueueConsumer");
+
+const MessageBody = z.discriminatedUnion("type", [
+  z.object({
+    type: z.literal("EXECUTE"),
+    taskIdentifier: z.string(),
+  }),
+  z.object({
+    type: z.literal("RESUME"),
+    completedAttemptIds: z.string().array(),
+  }),
+]);
+
+type BackgroundWorkerWithTasks = BackgroundWorker & { tasks: BackgroundWorkerTask[] };
+
+export type SharedQueueConsumerOptions = {
+  maximumItemsPerTrace?: number;
+  traceTimeoutSeconds?: number;
+};
+
+export class SharedQueueConsumer {
+  private _backgroundWorkers: Map<string, BackgroundWorkerWithTasks> = new Map();
+  private _deprecatedWorkers: Map<string, BackgroundWorkerWithTasks> = new Map();
+  private _enabled = false;
+  private _options: Required<SharedQueueConsumerOptions>;
+  private _perTraceCountdown: number | undefined;
+  private _lastNewTrace: Date | undefined;
+  private _currentSpanContext: Context | undefined;
+  private _taskFailures: number = 0;
+  private _taskSuccesses: number = 0;
+  private _currentSpan: Span | undefined;
+  private _endSpanInNextIteration = false;
+  private _tasks = sharedQueueTasks;
+  private _inProgressAttempts: Map<string, string> = new Map(); // Keys are task attempt friendly IDs, values are TaskRun ids/queue message ids
+
+  constructor(
+    private _sender: ZodMessageSender<typeof serverWebsocketMessages>,
+    options: SharedQueueConsumerOptions = {}
+  ) {
+    this._options = {
+      maximumItemsPerTrace: options.maximumItemsPerTrace ?? 1_000, // 1k items per trace
+      traceTimeoutSeconds: options.traceTimeoutSeconds ?? 60, // 60 seconds
+    };
+  }
+
+  // This method is called when a background worker is deprecated and will no longer be used unless a run is locked to it
+  public async deprecateBackgroundWorker(id: string) {
+    const backgroundWorker = this._backgroundWorkers.get(id);
+
+    if (!backgroundWorker) {
+      return;
+    }
+
+    this._deprecatedWorkers.set(id, backgroundWorker);
+    this._backgroundWorkers.delete(id);
+  }
+
+  public async registerBackgroundWorker(id: string, envId?: string) {
+    if (!envId) {
+      logger.error("Environment ID is required for background worker registration", {
+        backgroundWorkerId: id,
+      });
+      return;
+    }
+
+    const backgroundWorker = await prisma.backgroundWorker.findUnique({
+      where: {
+        friendlyId: id,
+        runtimeEnvironmentId: envId,
+      },
+      include: {
+        tasks: true,
+      },
+    });
+
+    if (!backgroundWorker) {
+      return;
+    }
+
+    this._backgroundWorkers.set(backgroundWorker.id, backgroundWorker);
+
+    logger.debug("Registered background worker", { backgroundWorker: backgroundWorker.id });
+
+    // Start reading from the queue if we haven't already
+    this.#enable();
+  }
+
+  public async start() {
+    this.#enable();
+  }
+
+  public async stop(reason: string = "Provider disconnected") {
+    if (!this._enabled) {
+      return;
+    }
+
+    logger.debug("Stopping shared queue consumer");
+    this._enabled = false;
+
+    // TODO: think about automatic prod cancellation
+
+    // We need to cancel all the in progress task run attempts and ack the messages so they will stop processing
+    // await this.#cancelInProgressAttempts(reason);
+  }
+
+  async #cancelInProgressAttempts(reason: string) {
+    const service = new CancelAttemptService();
+
+    const cancelledAt = new Date();
+
+    const inProgressAttempts = new Map(this._inProgressAttempts);
+
+    this._inProgressAttempts.clear();
+
+    for (const [attemptId, messageId] of inProgressAttempts) {
+      await this.#cancelInProgressAttempt(attemptId, messageId, service, cancelledAt, reason);
+    }
+  }
+
+  async #cancelInProgressAttempt(
+    attemptId: string,
+    messageId: string,
+    cancelAttemptService: CancelAttemptService,
+    cancelledAt: Date,
+    reason: string
+  ) {
+    try {
+      await cancelAttemptService.call(attemptId, messageId, cancelledAt, reason);
+    } catch (e) {
+      logger.error("Failed to cancel in progress attempt", {
+        attemptId,
+        messageId,
+        error: e,
+      });
+    }
+  }
+
+  #enable() {
+    if (this._enabled) {
+      return;
+    }
+
+    this._enabled = true;
+    this._perTraceCountdown = this._options.maximumItemsPerTrace;
+    this._lastNewTrace = new Date();
+    this._taskFailures = 0;
+    this._taskSuccesses = 0;
+
+    this.#doWork().finally(() => {});
+  }
+
+  async #doWork() {
+    if (!this._enabled) {
+      return;
+    }
+
+    // Check if the trace has expired
+    if (
+      this._perTraceCountdown === 0 ||
+      Date.now() - this._lastNewTrace!.getTime() > this._options.traceTimeoutSeconds * 1000 ||
+      this._currentSpanContext === undefined ||
+      this._endSpanInNextIteration
+    ) {
+      if (this._currentSpan) {
+        this._currentSpan.setAttribute("tasks.period.failures", this._taskFailures);
+        this._currentSpan.setAttribute("tasks.period.successes", this._taskSuccesses);
+
+        this._currentSpan.end();
+      }
+
+      // Create a new trace
+      this._currentSpan = tracer.startSpan(
+        "SharedQueueConsumer.doWork()",
+        {
+          kind: SpanKind.CONSUMER,
+        },
+        ROOT_CONTEXT
+      );
+
+      // Get the span trace context
+      this._currentSpanContext = trace.setSpan(ROOT_CONTEXT, this._currentSpan);
+
+      this._perTraceCountdown = this._options.maximumItemsPerTrace;
+      this._lastNewTrace = new Date();
+      this._taskFailures = 0;
+      this._taskSuccesses = 0;
+      this._endSpanInNextIteration = false;
+    }
+
+    return context.with(this._currentSpanContext ?? ROOT_CONTEXT, async () => {
+      await this.#doWorkInternal();
+      this._perTraceCountdown = this._perTraceCountdown! - 1;
+    });
+  }
+
+  async #doWorkInternal() {
+    // Attempt to dequeue a message from the shared queue
+    // If no message is available, reschedule the worker to run again in 1 second
+    // If a message is available, find the BackgroundWorkerTask that matches the message's taskIdentifier
+    // If no matching task is found, nack the message and reschedule the worker to run again in 1 second
+    // If the matching task is found, create the task attempt and lock the task run, then send the task run to the client
+    // Store the message as a processing message
+    // If the websocket connection disconnects before the task run is completed, nack the message
+    // When the task run completes, ack the message
+    // Using a heartbeat mechanism, if the client keeps responding with a heartbeat, we'll keep the message processing and increase the visibility timeout.
+
+    const message = await marqs?.dequeueMessageInSharedQueue();
+
+    if (!message) {
+      setTimeout(() => this.#doWork(), 1000);
+      return;
+    }
+
+    console.log("dequeueMessageInSharedQueue()", message);
+
+    const envId = this.#envIdFromQueue(message.queue);
+
+    const environment = await prisma.runtimeEnvironment.findUnique({
+      include: {
+        organization: true,
+        project: true,
+      },
+      where: {
+        id: envId,
+      },
+    });
+
+    if (!environment) {
+      logger.error("Environment not found", {
+        queueMessage: message.data,
+        envId,
+      });
+      await marqs?.acknowledgeMessage(message.messageId);
+      setTimeout(() => this.#doWork(), 100);
+      return;
+    }
+
+    const messageBody = MessageBody.safeParse(message.data);
+
+    if (!messageBody.success) {
+      logger.error("Failed to parse message", {
+        queueMessage: message.data,
+        error: messageBody.error,
+        env: environment,
+      });
+
+      await marqs?.acknowledgeMessage(message.messageId);
+
+      setTimeout(() => this.#doWork(), 100);
+      return;
+    }
+
+    switch (messageBody.data.type) {
+      case "EXECUTE": {
+        const existingTaskRun = await prisma.taskRun.findUnique({
+          where: {
+            id: message.messageId,
+          },
+        });
+
+        if (!existingTaskRun) {
+          logger.error("No existing task run", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+          });
+          await marqs?.acknowledgeMessage(message.messageId);
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const backgroundWorker = await prisma.backgroundWorker.findFirst({
+          where: {
+            runtimeEnvironmentId: existingTaskRun.runtimeEnvironmentId,
+            projectId: existingTaskRun.projectId,
+            imageDetails: {
+              some: {},
+            },
+          },
+          orderBy: {
+            updatedAt: "desc",
+          },
+          include: {
+            tasks: true,
+            imageDetails: true,
+          },
+        });
+
+        if (!backgroundWorker) {
+          logger.error("No matching background worker found for task run", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+          });
+          await marqs?.acknowledgeMessage(message.messageId);
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const backgroundTask = backgroundWorker.tasks.find(
+          (task) => task.slug === existingTaskRun.taskIdentifier
+        );
+
+        if (!backgroundTask) {
+          logger.warn("No matching background task found for task run", {
+            taskRun: existingTaskRun.id,
+            taskIdentifier: existingTaskRun.taskIdentifier,
+            backgroundWorker: backgroundWorker.id,
+            taskSlugs: backgroundWorker.tasks.map((task) => task.slug),
+          });
+
+          await marqs?.acknowledgeMessage(message.messageId);
+
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const lockedTaskRun = await prisma.taskRun.update({
+          where: {
+            id: message.messageId,
+          },
+          data: {
+            lockedAt: new Date(),
+            lockedById: backgroundTask.id,
+          },
+          include: {
+            attempts: {
+              take: 1,
+              orderBy: { number: "desc" },
+            },
+            tags: true,
+          },
+        });
+
+        if (!lockedTaskRun) {
+          logger.warn("Failed to lock task run", {
+            taskRun: existingTaskRun.id,
+            taskIdentifier: existingTaskRun.taskIdentifier,
+            backgroundWorker: backgroundWorker.id,
+            messageId: message.messageId,
+          });
+
+          await marqs?.acknowledgeMessage(message.messageId);
+
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const queue = await prisma.taskQueue.findUnique({
+          where: {
+            runtimeEnvironmentId_name: {
+              runtimeEnvironmentId: environment.id,
+              name: lockedTaskRun.queue,
+            },
+          },
+        });
+
+        if (!queue) {
+          await marqs?.nackMessage(message.messageId);
+          setTimeout(() => this.#doWork(), 1000);
+          return;
+        }
+
+        if (!this._enabled) {
+          await marqs?.nackMessage(message.messageId);
+          return;
+        }
+
+        const taskRunAttempt = await prisma.taskRunAttempt.create({
+          data: {
+            number: lockedTaskRun.attempts[0] ? lockedTaskRun.attempts[0].number + 1 : 1,
+            friendlyId: generateFriendlyId("attempt"),
+            taskRunId: lockedTaskRun.id,
+            startedAt: new Date(),
+            backgroundWorkerId: backgroundTask.workerId,
+            backgroundWorkerTaskId: backgroundTask.id,
+            status: "PENDING" as const,
+            queueId: queue.id,
+          },
+        });
+
+        try {
+          await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
+            backgroundWorkerId: backgroundWorker.friendlyId,
+            data: {
+              type: "SCHEDULE_ATTEMPT",
+              id: taskRunAttempt.id,
+              image: backgroundWorker.imageDetails[0].tag,
+              envId: environment.id,
+            },
+          });
+
+          this._inProgressAttempts.set(taskRunAttempt.friendlyId, message.messageId);
+        } catch (e) {
+          if (e instanceof Error) {
+            this._currentSpan?.recordException(e);
+          } else {
+            this._currentSpan?.recordException(new Error(String(e)));
+          }
+
+          this._endSpanInNextIteration = true;
+
+          // We now need to unlock the task run and delete the task run attempt
+          await prisma.$transaction([
+            prisma.taskRun.update({
+              where: {
+                id: lockedTaskRun.id,
+              },
+              data: {
+                lockedAt: null,
+                lockedById: null,
+              },
+            }),
+            prisma.taskRunAttempt.delete({
+              where: {
+                id: taskRunAttempt.id,
+              },
+            }),
+          ]);
+
+          // Finally we need to nack the message so it can be retried
+          await marqs?.nackMessage(message.messageId);
+        } finally {
+          setTimeout(() => this.#doWork(), 100);
+        }
+        break;
+      }
+      // Resume after dependency completed with no remaining retries
+      case "RESUME": {
+        if (messageBody.data.completedAttemptIds.length < 1) {
+          logger.error("No attempt IDs provided", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+          });
+          await marqs?.acknowledgeMessage(message.messageId);
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const resumableRun = await prisma.taskRun.findFirst({
+          where: {
+            id: message.messageId,
+          },
+          include: {
+            attempts: {
+              orderBy: {
+                createdAt: "desc",
+              },
+              take: 1,
+              include: {
+                checkpoints: {
+                  take: 1,
+                  orderBy: {
+                    createdAt: "desc",
+                  },
+                },
+              },
+            },
+          },
+        });
+
+        const resumableAttempt = resumableRun?.attempts[0];
+
+        if (!resumableAttempt) {
+          logger.error("Task run attempt to resume not found", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+          });
+          await marqs?.acknowledgeMessage(message.messageId);
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const backgroundWorker = await prisma.backgroundWorker.findFirst({
+          where: {
+            runtimeEnvironmentId: resumableRun.runtimeEnvironmentId,
+            projectId: resumableRun.projectId,
+            imageDetails: {
+              some: {},
+            },
+          },
+          orderBy: {
+            createdAt: "desc",
+          },
+          include: {
+            tasks: true,
+            imageDetails: {
+              take: 1,
+              orderBy: {
+                updatedAt: "desc",
+              },
+            },
+          },
+        });
+
+        if (!backgroundWorker) {
+          logger.error("No matching background worker found for task run", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+          });
+          await marqs?.acknowledgeMessage(message.messageId);
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const backgroundTask = backgroundWorker.tasks.find(
+          (task) => task.slug === resumableRun.taskIdentifier
+        );
+
+        if (!backgroundTask) {
+          logger.warn("No matching background task found for task run", {
+            taskRun: resumableRun.id,
+            taskIdentifier: resumableRun.taskIdentifier,
+            backgroundWorker: backgroundWorker.id,
+            taskSlugs: backgroundWorker.tasks.map((task) => task.slug),
+          });
+
+          await marqs?.acknowledgeMessage(message.messageId);
+
+          setTimeout(() => this.#doWork(), 100);
+          return;
+        }
+
+        const queue = await prisma.taskQueue.findUnique({
+          where: {
+            runtimeEnvironmentId_name: {
+              runtimeEnvironmentId: environment.id,
+              name: resumableRun.queue,
+            },
+          },
+        });
+
+        if (!queue) {
+          await marqs?.nackMessage(message.messageId);
+          setTimeout(() => this.#doWork(), 1000);
+          return;
+        }
+
+        if (!this._enabled) {
+          await marqs?.nackMessage(message.messageId);
+          return;
+        }
+
+        const completions: TaskRunExecutionResult[] = [];
+        const executions: TaskRunExecution[] = [];
+
+        for (const completedAttemptId of messageBody.data.completedAttemptIds) {
+          const completedAttempt = await prisma.taskRunAttempt.findUnique({
+            where: {
+              id: completedAttemptId,
+              taskRun: {
+                lockedAt: {
+                  not: null,
+                },
+                lockedById: {
+                  not: null,
+                },
+              },
+            },
+          });
+
+          if (!completedAttempt) {
+            logger.error("Completed attempt not found", {
+              queueMessage: message.data,
+              messageId: message.messageId,
+            });
+            await marqs?.acknowledgeMessage(message.messageId);
+            setTimeout(() => this.#doWork(), 100);
+            return;
+          }
+
+          const completion = await this._tasks.getCompletionPayloadFromAttempt(completedAttempt.id);
+
+          if (!completion) {
+            await marqs?.acknowledgeMessage(message.messageId);
+            setTimeout(() => this.#doWork(), 100);
+            return;
+          }
+
+          completions.push(completion);
+
+          const executionPayload = await this._tasks.getExecutionPayloadFromAttempt(
+            completedAttempt.id,
+            false
+          );
+
+          if (!executionPayload) {
+            await marqs?.acknowledgeMessage(message.messageId);
+            setTimeout(() => this.#doWork(), 100);
+            return;
+          }
+
+          executions.push(executionPayload.execution);
+        }
+
+        try {
+          const latestCheckpoint = resumableAttempt.checkpoints[0];
+
+          if (!latestCheckpoint) {
+            // No checkpoint means the task should still be running
+            // We can broadcast to all coordinators to resume immediately
+            socketIo.coordinatorNamespace.emit("RESUME", {
+              version: "v1",
+              attemptId: resumableAttempt.id,
+              image: backgroundWorker.imageDetails[0].tag,
+              completions,
+              executions,
+            });
+          } else {
+            // There's a checkpoint we need to restore first
+            // TODO: Send RESUME message once the restored task has checked in
+            socketIo.providerNamespace.emit("RESTORE", {
+              version: "v1",
+              id: latestCheckpoint.id,
+              attemptId: latestCheckpoint.attemptId,
+              type: latestCheckpoint.type,
+              location: latestCheckpoint.location,
+              reason: latestCheckpoint.reason ?? undefined,
+            });
+          }
+        } catch (e) {
+          if (e instanceof Error) {
+            this._currentSpan?.recordException(e);
+          } else {
+            this._currentSpan?.recordException(new Error(String(e)));
+          }
+
+          this._endSpanInNextIteration = true;
+
+          // Finally we need to nack the message so it can be retried
+          await marqs?.nackMessage(message.messageId);
+        } finally {
+          setTimeout(() => this.#doWork(), 100);
+        }
+        break;
+      }
+    }
+  }
+
+  #envIdFromQueue(queueName: string) {
+    return queueName.split(":")[1];
+  }
+}
+
+class SharedQueueTasks {
+  async getCompletionPayloadFromAttempt(id: string): Promise<TaskRunExecutionResult | undefined> {
+    const attempt = await prisma.taskRunAttempt.findUnique({
+      where: {
+        id,
+        status: {
+          in: ["COMPLETED", "FAILED"],
+        },
+      },
+      include: {
+        backgroundWorker: true,
+        backgroundWorkerTask: true,
+        taskRun: {
+          include: {
+            runtimeEnvironment: {
+              include: {
+                organization: true,
+                project: true,
+              },
+            },
+            tags: true,
+          },
+        },
+        queue: true,
+      },
+    });
+
+    if (!attempt) {
+      logger.error("No completed attempt found", { id });
+      return;
+    }
+
+    const ok = attempt.status === "COMPLETED";
+
+    if (ok) {
+      const success: TaskRunSuccessfulExecutionResult = {
+        ok,
+        id: attempt.friendlyId,
+        output: attempt.output ?? "",
+        outputType: attempt.outputType,
+      };
+      return success;
+    } else {
+      const failure: TaskRunFailedExecutionResult = {
+        ok,
+        id: attempt.friendlyId,
+        error: attempt.error as TaskRunError,
+      };
+      return failure;
+    }
+  }
+
+  async getExecutionPayloadFromAttempt(
+    id: string,
+    setToExecuting = true
+  ): Promise<ProdTaskRunExecutionPayload | undefined> {
+    const attempt = await prisma.taskRunAttempt.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        backgroundWorker: true,
+        backgroundWorkerTask: true,
+        taskRun: {
+          include: {
+            runtimeEnvironment: {
+              include: {
+                organization: true,
+                project: true,
+              },
+            },
+            tags: true,
+            batchItem: {
+              include: {
+                batchTaskRun: true,
+              },
+            },
+          },
+        },
+        queue: true,
+      },
+    });
+
+    if (!attempt) {
+      logger.error("No attempt found", { id });
+      return;
+    }
+
+    if (setToExecuting) {
+      await prisma.taskRunAttempt.update({
+        where: {
+          id,
+        },
+        data: {
+          status: "EXECUTING",
+        },
+      });
+    }
+
+    const { backgroundWorkerTask, taskRun, queue } = attempt;
+
+    const execution: ProdTaskRunExecution = {
+      task: {
+        id: backgroundWorkerTask.slug,
+        filePath: backgroundWorkerTask.filePath,
+        exportName: backgroundWorkerTask.exportName,
+      },
+      attempt: {
+        id: attempt.friendlyId,
+        number: attempt.number,
+        startedAt: attempt.startedAt ?? attempt.createdAt,
+        backgroundWorkerId: attempt.backgroundWorkerId,
+        backgroundWorkerTaskId: attempt.backgroundWorkerTaskId,
+        status: "EXECUTING" as const,
+      },
+      run: {
+        id: taskRun.friendlyId,
+        payload: taskRun.payload,
+        payloadType: taskRun.payloadType,
+        context: taskRun.context,
+        createdAt: taskRun.createdAt,
+        tags: taskRun.tags.map((tag) => tag.name),
+        isTest: taskRun.isTest,
+      },
+      queue: {
+        id: queue.friendlyId,
+        name: queue.name,
+      },
+      environment: {
+        id: taskRun.runtimeEnvironment.id,
+        slug: taskRun.runtimeEnvironment.slug,
+        type: taskRun.runtimeEnvironment.type,
+      },
+      organization: {
+        id: taskRun.runtimeEnvironment.organization.id,
+        slug: taskRun.runtimeEnvironment.organization.slug,
+        name: taskRun.runtimeEnvironment.organization.title,
+      },
+      project: {
+        id: taskRun.runtimeEnvironment.project.id,
+        ref: taskRun.runtimeEnvironment.project.externalRef,
+        slug: taskRun.runtimeEnvironment.project.slug,
+        name: taskRun.runtimeEnvironment.project.name,
+      },
+      batch: taskRun.batchItem?.batchTaskRun
+        ? { id: taskRun.batchItem.batchTaskRun.friendlyId }
+        : undefined,
+      worker: {
+        id: attempt.backgroundWorkerId,
+        contentHash: attempt.backgroundWorker.contentHash,
+        version: attempt.backgroundWorker.version,
+      },
+    };
+
+    const environmentRepository = new EnvironmentVariablesRepository();
+    const variables = await environmentRepository.getEnvironmentVariables(
+      attempt.taskRun.runtimeEnvironment.projectId,
+      attempt.taskRun.runtimeEnvironmentId
+    );
+
+    const payload: ProdTaskRunExecutionPayload = {
+      execution,
+      traceContext: taskRun.traceContext as Record<string, unknown>,
+      environment: variables.reduce((acc: Record<string, string>, curr) => {
+        acc[curr.key] = curr.value;
+        return acc;
+      }, {}),
+    };
+
+    return payload;
+  }
+
+  async taskHeartbeat(attemptFriendlyId: string, seconds: number = 60) {
+    const taskRunAttempt = await prisma.taskRunAttempt.findUnique({
+      where: { friendlyId: attemptFriendlyId },
+    });
+
+    if (!taskRunAttempt) {
+      return;
+    }
+
+    await marqs?.heartbeatMessage(taskRunAttempt.taskRunId, seconds);
+  }
+}
+
+export const sharedQueueTasks = singleton("sharedQueueTasks", () => new SharedQueueTasks());

apps/webapp/app/v3/services/cancelAttempt.server.ts

@@ -4,14 +4,26 @@ import { marqs } from "../marqs.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 
+import { PrismaClientOrTransaction, prisma } from "~/db.server";
+
 export class CancelAttemptService extends BaseService {
   public async call(
     attemptId: string,
     taskRunId: string,
     cancelledAt: Date,
     reason: string,
-    environment: AuthenticatedEnvironment
+    env?: AuthenticatedEnvironment
   ) {
+    let environment: AuthenticatedEnvironment | undefined = env;
+
+    if (!environment) {
+      environment = await getAuthenticatedEnvironmentFromAttempt(attemptId);
+
+      if (!environment) {
+        return;
+      }
+    }
+
     return await this.traceWithEnv("call()", environment, async (span) => {
       span.setAttribute("taskRunId", taskRunId);
       span.setAttribute("attemptId", attemptId);
@@ -56,3 +68,32 @@ export class CancelAttemptService extends BaseService {
     });
   }
 }
+
+async function getAuthenticatedEnvironmentFromAttempt(
+  friendlyId: string,
+  prismaClient?: PrismaClientOrTransaction
+) {
+  const taskRunAttempt = await (prismaClient ?? prisma).taskRunAttempt.findUnique({
+    where: {
+      friendlyId,
+    },
+    include: {
+      taskRun: {
+        include: {
+          runtimeEnvironment: {
+            include: {
+              organization: true,
+              project: true,
+            },
+          },
+        },
+      },
+    },
+  });
+
+  if (!taskRunAttempt) {
+    return;
+  }
+
+  return taskRunAttempt?.taskRun.runtimeEnvironment;
+}

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -11,13 +11,14 @@ import { eventRepository } from "../eventRepository.server";
 import { marqs } from "../marqs.server";
 import { BaseService } from "./baseService.server";
 import { Attributes } from "@opentelemetry/api";
+import { logger } from "~/services/logger.server";
 
 export class CompleteAttemptService extends BaseService {
   public async call(
     completion: TaskRunExecutionResult,
     execution: TaskRunExecution,
-    env: AuthenticatedEnvironment
-  ): Promise<"ACKNOWLEDGED" | "RETRIED"> {
+    env?: AuthenticatedEnvironment
+  ): Promise<"ACKNOWLEDGED" | "RETRIED" | "FAILED"> {
     const taskRunAttempt = completion.ok
       ? await this._prisma.taskRunAttempt.update({
           where: { friendlyId: completion.id },
@@ -28,7 +29,17 @@ export class CompleteAttemptService extends BaseService {
             outputType: completion.outputType,
           },
           include: {
-            taskRun: true,
+            taskRun: {
+              include: {
+                batchItem: true,
+                dependency: {
+                  include: {
+                    dependentAttempt: true,
+                    dependentBatchRun: true,
+                  },
+                },
+              },
+            },
             backgroundWorkerTask: true,
           },
         })
@@ -40,7 +51,17 @@ export class CompleteAttemptService extends BaseService {
             error: completion.error,
           },
           include: {
-            taskRun: true,
+            taskRun: {
+              include: {
+                batchItem: true,
+                dependency: {
+                  include: {
+                    dependentAttempt: true,
+                    dependentBatchRun: true,
+                  },
+                },
+              },
+            },
             backgroundWorkerTask: true,
           },
         });
@@ -53,7 +74,10 @@ export class CompleteAttemptService extends BaseService {
           }
         : undefined;
 
+      const environment = env ?? (await this.#getEnvironment(execution.environment.id));
+
       const retryAt = new Date(completion.retry.timestamp);
+
       // Retry the task run
       await eventRepository.recordEvent(
         retryConfig?.maxAttempts
@@ -61,7 +85,7 @@ export class CompleteAttemptService extends BaseService {
           : `Retry #${execution.attempt.number} delay`,
         {
           taskSlug: taskRunAttempt.taskRun.taskIdentifier,
-          environment: env,
+          environment,
           attributes: {
             metadata: this.#generateMetadataAttributesForNextAttempt(execution),
             properties: {
@@ -85,10 +109,29 @@ export class CompleteAttemptService extends BaseService {
         }
       );
 
-      await marqs?.nackMessage(taskRunAttempt.taskRunId, completion.retry.timestamp);
+      logger.debug("Retrying", { taskRun: taskRunAttempt.taskRun.friendlyId });
+
+      if (environment.type === "DEVELOPMENT") {
+        // This is already an EXECUTE message so we can just NACK
+        await marqs?.nackMessage(taskRunAttempt.taskRunId, completion.retry.timestamp);
+      } else {
+        // We have to replace a potential RESUME with EXECUTE to correctly retry the attempt
+        await marqs?.replaceMessage(
+          taskRunAttempt.taskRunId,
+          {
+            type: "EXECUTE",
+            taskIdentifier: taskRunAttempt.taskRun.taskIdentifier,
+          },
+          completion.retry.timestamp
+        );
+      }
 
       return "RETRIED";
-    } else {
+    }
+    // Attempt succeeded or this was the last retry
+    else {
+      logger.debug("Completed attempt, ACKing message", taskRunAttempt);
+
       await marqs?.acknowledgeMessage(taskRunAttempt.taskRunId);
 
       // Now we need to "complete" the task run event/span
@@ -109,6 +152,124 @@ export class CompleteAttemptService extends BaseService {
         });
       }
 
+      const { batchItem, dependency } = taskRunAttempt.taskRun;
+
+      // This run is part of a batch so we should update its status
+      if (batchItem) {
+        logger.debug("Completing attempt with batch item", { batchItem });
+
+        await this._prisma.batchTaskRunItem.update({
+          where: {
+            id: batchItem.id,
+          },
+          data: {
+            status: completion.ok ? "COMPLETED" : "FAILED",
+          },
+        });
+
+        const finalizedBatchRun = await this._prisma.batchTaskRun.findFirst({
+          where: {
+            id: batchItem.batchTaskRunId,
+            dependentTaskAttemptId: {
+              not: null,
+            },
+            items: {
+              every: {
+                status: {
+                  not: "PENDING",
+                },
+              },
+            },
+          },
+          include: {
+            dependentTaskAttempt: true,
+            items: {
+              include: {
+                taskRun: {
+                  include: {
+                    attempts: {
+                      orderBy: {
+                        completedAt: "desc",
+                      },
+                      take: 1,
+                      select: {
+                        id: true,
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        });
+
+        // This batch has a dependent attempt and just finalized, we should resume that attempt
+        if (finalizedBatchRun && finalizedBatchRun.dependentTaskAttempt) {
+          const environment =
+            env ?? (await this.#getEnvironment(taskRunAttempt.taskRun.runtimeEnvironmentId));
+
+          if (!environment) {
+            logger.error("Environment not found", {
+              attemptId: taskRunAttempt.id,
+              envId: taskRunAttempt.taskRun.runtimeEnvironmentId,
+            });
+            return "FAILED";
+          }
+
+          if (environment.type === "DEVELOPMENT") {
+            return "ACKNOWLEDGED";
+          }
+
+          await marqs?.replaceMessage(finalizedBatchRun.dependentTaskAttempt.taskRunId, {
+            type: "RESUME",
+            completedAttemptIds: finalizedBatchRun.items.map(
+              (item) => item.taskRun.attempts[0]?.id
+            ),
+          });
+        }
+      }
+
+      if (dependency) {
+        logger.debug("Completing attempt with dependency", { dependency });
+
+        const environment =
+          env ?? (await this.#getEnvironment(taskRunAttempt.taskRun.runtimeEnvironmentId));
+
+        if (!environment) {
+          logger.error("Environment not found", {
+            attemptId: taskRunAttempt.id,
+            envId: taskRunAttempt.taskRun.runtimeEnvironmentId,
+          });
+          return "FAILED";
+        }
+
+        if (environment.type === "DEVELOPMENT") {
+          return "ACKNOWLEDGED";
+        }
+
+        if (dependency.dependentAttempt) {
+          const dependentRun = await this._prisma.taskRun.findFirst({
+            where: {
+              id: dependency.dependentAttempt.taskRunId,
+            },
+          });
+
+          if (!dependentRun) {
+            logger.error("Dependent task run does not exist", {
+              attemptId: taskRunAttempt.id,
+              envId: taskRunAttempt.taskRun.runtimeEnvironmentId,
+              taskRunId: dependency.taskRunId,
+            });
+            return "FAILED";
+          }
+
+          await marqs?.replaceMessage(dependentRun.id, {
+            type: "RESUME",
+            completedAttemptIds: [taskRunAttempt.id],
+          });
+        }
+      }
+
       return "ACKNOWLEDGED";
     }
   }
@@ -123,4 +284,16 @@ export class CompleteAttemptService extends BaseService {
 
     return flattenAttributes(context, "ctx");
   }
+
+  async #getEnvironment(id: string) {
+    return await this._prisma.runtimeEnvironment.findUniqueOrThrow({
+      where: {
+        id,
+      },
+      include: {
+        project: true,
+        organization: true,
+      },
+    });
+  }
 }

apps/webapp/app/v3/services/createBackgroundWorker.server.ts

@@ -5,6 +5,7 @@ import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "../marqs.server";
 import { BaseService } from "./baseService.server";
+import { $transaction } from "~/db.server";
 
 export class CreateBackgroundWorkerService extends BaseService {
   public async call(
@@ -15,100 +16,123 @@ export class CreateBackgroundWorkerService extends BaseService {
     return this.traceWithEnv("call", environment, async (span) => {
       span.setAttribute("projectRef", projectRef);
 
-      const project = await this._prisma.project.findUniqueOrThrow({
-        where: {
-          externalRef: projectRef,
-          environments: {
-            some: {
-              id: environment.id,
+      const backgroundWorker = await $transaction(this._prisma, async (tx) => {
+        const project = await this._prisma.project.findUniqueOrThrow({
+          where: {
+            externalRef: projectRef,
+            environments: {
+              some: {
+                id: environment.id,
+              },
             },
           },
-        },
-        include: {
-          backgroundWorkers: {
-            where: {
-              runtimeEnvironmentId: environment.id,
+          include: {
+            backgroundWorkers: {
+              where: {
+                runtimeEnvironmentId: environment.id,
+              },
+              orderBy: {
+                createdAt: "desc",
+              },
+              take: 1,
             },
-            orderBy: {
-              createdAt: "desc",
-            },
-            take: 1,
           },
-        },
-      });
+        });
 
-      const latestBackgroundWorker = project.backgroundWorkers[0];
+        const latestBackgroundWorker = project.backgroundWorkers[0];
 
-      if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) {
-        return latestBackgroundWorker;
-      }
+        if (latestBackgroundWorker?.contentHash === body.metadata.contentHash) {
+          return latestBackgroundWorker;
+        }
 
-      const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version);
+        const nextVersion = calculateNextBuildVersion(project.backgroundWorkers[0]?.version);
 
-      logger.debug(`Creating background worker`, {
-        nextVersion,
-        lastVersion: project.backgroundWorkers[0]?.version,
-      });
+        logger.debug(`Creating background worker`, {
+          nextVersion,
+          lastVersion: project.backgroundWorkers[0]?.version,
+        });
 
-      const backgroundWorker = await this._prisma.backgroundWorker.create({
-        data: {
-          friendlyId: generateFriendlyId("worker"),
-          version: nextVersion,
-          runtimeEnvironmentId: environment.id,
-          projectId: project.id,
-          metadata: body.metadata,
-          contentHash: body.metadata.contentHash,
+        const backgroundWorker = await this._prisma.backgroundWorker.create({
+          data: {
+            friendlyId: generateFriendlyId("worker"),
+            version: nextVersion,
+            runtimeEnvironmentId: environment.id,
+            projectId: project.id,
+            metadata: body.metadata,
+            contentHash: body.metadata.contentHash,
           cliVersion: body.metadata.cliPackageVersion,
           sdkVersion: body.metadata.packageVersion,
-        },
+          },
+        });
+
+        if (environment.type !== "DEVELOPMENT") {
+          await this._prisma.imageDetails.update({
+            where: {
+              projectId_runtimeEnvironmentId_contentHash: {
+                projectId: environment.projectId,
+                runtimeEnvironmentId: environment.id,
+                contentHash: backgroundWorker.contentHash,
+              },
+            },
+            data: {
+              backgroundWorkerId: backgroundWorker.id,
+            },
+          });
+        }
+
+        for (const task of body.metadata.tasks) {
+          await this._prisma.backgroundWorkerTask.create({
+            data: {
+              friendlyId: generateFriendlyId("task"),
+              projectId: project.id,
+              runtimeEnvironmentId: environment.id,
+              workerId: backgroundWorker.id,
+              slug: task.id,
+              filePath: task.filePath,
+              exportName: task.exportName,
+              retryConfig: task.retry,
+              queueConfig: task.queue,
+            },
+          });
+
+          const queueName = task.queue?.name ?? `task/${task.id}`;
+
+          const taskQueue = await this._prisma.taskQueue.upsert({
+            where: {
+              runtimeEnvironmentId_name: {
+                runtimeEnvironmentId: environment.id,
+                name: queueName,
+              },
+            },
+            update: {
+              concurrencyLimit: task.queue?.concurrencyLimit,
+              rateLimit: task.queue?.rateLimit,
+            },
+            create: {
+              friendlyId: generateFriendlyId("queue"),
+              name: queueName,
+              concurrencyLimit: task.queue?.concurrencyLimit,
+              runtimeEnvironmentId: environment.id,
+              projectId: project.id,
+              rateLimit: task.queue?.rateLimit,
+              type: task.queue?.name ? "NAMED" : "VIRTUAL",
+            },
+          });
+
+          if (taskQueue.concurrencyLimit) {
+            await marqs?.updateQueueConcurrency(
+              environment,
+              taskQueue.name,
+              taskQueue.concurrencyLimit
+            );
+          }
+        }
+
+        return backgroundWorker;
       });
 
-      for (const task of body.metadata.tasks) {
-        await this._prisma.backgroundWorkerTask.create({
-          data: {
-            friendlyId: generateFriendlyId("task"),
-            projectId: project.id,
-            runtimeEnvironmentId: environment.id,
-            workerId: backgroundWorker.id,
-            slug: task.id,
-            filePath: task.filePath,
-            exportName: task.exportName,
-            retryConfig: task.retry,
-            queueConfig: task.queue,
-          },
-        });
-
-        const queueName = task.queue?.name ?? `task/${task.id}`;
-
-        const taskQueue = await this._prisma.taskQueue.upsert({
-          where: {
-            runtimeEnvironmentId_name: {
-              runtimeEnvironmentId: environment.id,
-              name: queueName,
-            },
-          },
-          update: {
-            concurrencyLimit: task.queue?.concurrencyLimit,
-            rateLimit: task.queue?.rateLimit,
-          },
-          create: {
-            friendlyId: generateFriendlyId("queue"),
-            name: queueName,
-            concurrencyLimit: task.queue?.concurrencyLimit,
-            runtimeEnvironmentId: environment.id,
-            projectId: project.id,
-            rateLimit: task.queue?.rateLimit,
-            type: task.queue?.name ? "NAMED" : "VIRTUAL",
-          },
-        });
-
-        if (taskQueue.concurrencyLimit) {
-          await marqs?.updateQueueConcurrency(
-            environment,
-            taskQueue.name,
-            taskQueue.concurrencyLimit
-          );
-        }
+      if (!backgroundWorker) {
+        throw new Error("Failed to create background worker");
       }
 
       return backgroundWorker;

apps/webapp/app/v3/services/createCheckpoint.server.ts

@@ -0,0 +1,46 @@
+import { CoordinatorToPlatformEvents } from "@trigger.dev/core/v3";
+import type { Checkpoint } from "@trigger.dev/database";
+import { PrismaClient, prisma } from "~/db.server";
+import { logger } from "~/services/logger.server";
+import { generateFriendlyId } from "../friendlyIdentifiers";
+import { marqs } from "../marqs.server";
+
+export class CreateCheckpointService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    params: Parameters<CoordinatorToPlatformEvents["CHECKPOINT_CREATED"]>[0]
+  ): Promise<Checkpoint> {
+    const attempt = await this.#prismaClient.taskRunAttempt.findUniqueOrThrow({
+      where: {
+        id: params.attemptId,
+      },
+      include: {
+        taskRun: true,
+      },
+    });
+
+    logger.debug(`Creating checkpoint`, params);
+
+    const checkpoint = await this.#prismaClient.checkpoint.create({
+      data: {
+        friendlyId: generateFriendlyId("checkpoint"),
+        runtimeEnvironmentId: attempt.taskRun.runtimeEnvironmentId,
+        projectId: attempt.taskRun.projectId,
+        attemptId: attempt.id,
+        location: params.location,
+        type: params.docker ? "DOCKER" : "KUBERNETES",
+        reason: params.reason,
+      },
+    });
+
+    // TODO: Can't heartbeat when checkpointed, so we ACK to prevent automatic requeue
+    // await marqs?.acknowledgeMessage(attempt.taskRunId);
+
+    return checkpoint;
+  }
+}

apps/webapp/app/v3/services/createImageDetails.server.ts

@@ -0,0 +1,75 @@
+import { CreateImageDetailsRequestBody } from "@trigger.dev/core/v3";
+import type { ImageDetails } from "@trigger.dev/database";
+import { PrismaClient, prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { generateFriendlyId } from "../friendlyIdentifiers";
+import { env } from "~/env.server";
+import { workerQueue } from "~/services/worker.server";
+
+function escapeStringForRegex(rawString: string) {
+  return rawString.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&").replace(/-/g, "\\x2d");
+}
+
+export class CreateImageDetailsService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    projectRef: string,
+    environment: AuthenticatedEnvironment,
+    body: CreateImageDetailsRequestBody
+  ): Promise<ImageDetails> {
+    const allowedTagPrefix = escapeStringForRegex(`${env.IMAGE_REGISTRY}/${env.IMAGE_REPO}:`);
+
+    if (!body.metadata.imageTag.match(`^${allowedTagPrefix}`)) {
+      if (env.NODE_ENV !== "development") {
+        throw new Error("Forbidden image tag");
+      }
+    }
+
+    const project = await this.#prismaClient.project.findUniqueOrThrow({
+      where: {
+        externalRef: projectRef,
+        environments: {
+          some: {
+            id: environment.id,
+          },
+        },
+      },
+    });
+
+    logger.debug(`Creating image details`, {
+      imageTag: body.metadata.imageTag,
+    });
+
+    const imageDetails = await this.#prismaClient.imageDetails.upsert({
+      where: {
+        projectId_runtimeEnvironmentId_contentHash: {
+          contentHash: body.metadata.contentHash,
+          runtimeEnvironmentId: environment.id,
+          projectId: project.id,
+        },
+      },
+      create: {
+        contentHash: body.metadata.contentHash,
+        friendlyId: generateFriendlyId("image"),
+        tag: body.metadata.imageTag,
+        runtimeEnvironmentId: environment.id,
+        projectId: project.id,
+        metadata: body.metadata,
+      },
+      update: {
+        tag: body.metadata.imageTag,
+        metadata: body.metadata,
+      },
+    });
+
+    await workerQueue.enqueue("indexTasks", { id: imageDetails.id });
+
+    return imageDetails;
+  }
+}

apps/webapp/app/v3/services/indexTasks.server.ts

@@ -0,0 +1,53 @@
+import { PrismaClient, prisma } from "~/db.server";
+import { socketIo } from "../handleSocketIo.server";
+import { logger } from "~/services/logger.server";
+
+export type IndexTasksServiceOptions = {
+  idempotencyKey?: string;
+  triggerVersion?: string;
+  traceContext?: Record<string, string | undefined>;
+};
+
+export class IndexTasksService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(imageDetailsId: string) {
+    const imageDetails = await this.#prismaClient.imageDetails.findUnique({
+      where: {
+        id: imageDetailsId,
+      },
+    });
+
+    if (!imageDetails) {
+      logger.error(`No image details with this ID: ${imageDetailsId}`);
+      return;
+    }
+
+    if (imageDetails.backgroundWorkerId) {
+      logger.debug(
+        `Image details have already been indexed for ${imageDetails.friendlyId}. Refreshing worker timestamp.`
+      );
+      await this.#prismaClient.backgroundWorker.update({
+        where: {
+          id: imageDetails.backgroundWorkerId,
+        },
+        data: {
+          updatedAt: new Date(),
+        },
+      });
+      return;
+    }
+
+    // just broadcast for now - there should only ever be one provider connected
+    socketIo.providerNamespace.emit("INDEX", {
+      version: "v1",
+      contentHash: imageDetails.contentHash,
+      imageTag: imageDetails.tag,
+      envId: imageDetails.runtimeEnvironmentId,
+    });
+  }
+}

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -123,7 +123,7 @@ export class TriggerTaskService extends BaseService {
             span.setAttribute("runId", taskRun.friendlyId);
 
             if (body.options?.dependentAttempt) {
-              const dependentAttempt = await tx.taskRun.findUnique({
+              const dependentAttempt = await tx.taskRunAttempt.findUnique({
                 where: { friendlyId: body.options.dependentAttempt },
               });
 

apps/webapp/app/v3/sharedSocketConnection.ts

@@ -0,0 +1,94 @@
+import {
+  MessageCatalogToSocketIoEvents,
+  ZodMessageHandler,
+  ZodMessageSender,
+  clientWebsocketMessages,
+  serverWebsocketMessages,
+} from "@trigger.dev/core/v3";
+import { Evt } from "evt";
+import { randomUUID } from "node:crypto";
+import { logger } from "~/services/logger.server";
+import { SharedQueueConsumer } from "./marqs/sharedQueueConsumer.server";
+import { DisconnectReason, Namespace, Socket } from "socket.io";
+
+export class SharedSocketConnection {
+  public id: string;
+  public onClose: Evt<DisconnectReason> = new Evt();
+
+  private _sender: ZodMessageSender<typeof serverWebsocketMessages>;
+  private _sharedConsumer: SharedQueueConsumer;
+  private _messageHandler: ZodMessageHandler<typeof clientWebsocketMessages>;
+
+  constructor(
+    namespace: Namespace<
+      MessageCatalogToSocketIoEvents<typeof clientWebsocketMessages>,
+      MessageCatalogToSocketIoEvents<typeof serverWebsocketMessages>
+    >,
+    private socket: Socket<
+      MessageCatalogToSocketIoEvents<typeof clientWebsocketMessages>,
+      MessageCatalogToSocketIoEvents<typeof serverWebsocketMessages>
+    >,
+    logger?: (...args: any[]) => void
+  ) {
+    this.id = randomUUID();
+
+    this._sender = new ZodMessageSender({
+      schema: serverWebsocketMessages,
+      sender: async (message) => {
+        return new Promise((resolve, reject) => {
+          try {
+            const { type, ...payload } = message;
+            namespace.emit(type, payload as any);
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+      },
+    });
+
+    this._sharedConsumer = new SharedQueueConsumer(this._sender);
+
+    socket.on("disconnect", this.#handleClose.bind(this));
+    socket.on("error", this.#handleError.bind(this));
+
+    this._messageHandler = new ZodMessageHandler({
+      schema: clientWebsocketMessages,
+      messages: {
+        READY_FOR_TASKS: async (payload) => {
+          this._sharedConsumer.start();
+        },
+        BACKGROUND_WORKER_DEPRECATED: async (payload) => {
+          // await this._sharedConsumer.deprecateBackgroundWorker(payload.backgroundWorkerId);
+        },
+        BACKGROUND_WORKER_MESSAGE: async (payload) => {
+          switch (payload.data.type) {
+            case "TASK_RUN_COMPLETED": {
+              // handled in coordinator namespace
+              break;
+            }
+            case "TASK_HEARTBEAT": {
+              // handled in coordinator namespace
+              break;
+            }
+          }
+        },
+      },
+    });
+    this._messageHandler.registerHandlers(this.socket, logger);
+  }
+
+  async initialize() {
+    this._sender.send("SERVER_READY", { id: this.id });
+  }
+
+  async #handleClose(ev: DisconnectReason) {
+    await this._sharedConsumer.stop();
+
+    this.onClose.post(ev);
+  }
+
+  async #handleError(ev: Error) {
+    logger.error("Websocket error", { ev });
+  }
+}

apps/webapp/package.json

@@ -142,6 +142,7 @@
     "simple-oauth2": "^5.0.0",
     "simplur": "^3.0.1",
     "slug": "^6.0.0",
+    "socket.io": "^4.7.4",
     "sonner": "^1.0.3",
     "sqs-consumer": "^7.4.0",
     "tailwind-merge": "^1.12.0",
@@ -201,6 +202,7 @@
     "@typescript-eslint/parser": "^5.59.6",
     "autoprefixer": "^10.4.13",
     "datepicker": "link:@types/@react-aria/datepicker",
+    "engine.io": "^6.5.4",
     "esbuild": "^0.15.10",
     "eslint": "^8.24.0",
     "eslint-config-prettier": "^8.5.0",

apps/webapp/server.ts

@@ -5,6 +5,8 @@ import morgan from "morgan";
 import { createRequestHandler } from "@remix-run/express";
 import { WebSocketServer } from "ws";
 import { broadcastDevReady, logDevReady } from "@remix-run/server-runtime";
+import type { Server as IoServer } from "socket.io";
+import type { Server as EngineServer } from "engine.io";
 
 const app = express();
 
@@ -53,6 +55,7 @@ app.all(
 const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000;
 
 if (process.env.HTTP_SERVER_DISABLED !== "true") {
+  const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo;
   const wss: WebSocketServer | undefined = build.entry.module.wss;
 
   const server = app.listen(port, () => {
@@ -77,6 +80,9 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") {
     });
   });
 
+  socketIo?.io.attach(server);
+  server.removeAllListeners("upgrade"); // prevent duplicate upgrades from listeners created by io.attach()
+
   server.on("upgrade", async (req, socket, head) => {
     console.log(
       `Attemping to upgrade connection at url ${req.url} with headers: ${JSON.stringify(
@@ -86,6 +92,15 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") {
 
     const url = new URL(req.url ?? "", "http://localhost");
 
+    // Upgrade socket.io connection
+    if (url.pathname.startsWith("/socket.io/")) {
+      console.log(`Socket.io client connected, upgrading their connection...`);
+
+      // https://github.com/socketio/socket.io/issues/4693
+      (socketIo?.io.engine as EngineServer).handleUpgrade(req, socket, head);
+      return;
+    }
+
     // Only upgrade the connecting if the path is `/ws`
     if (url.pathname !== "/ws") {
       socket.destroy(

docker/docker-compose.yml

@@ -27,7 +27,7 @@ services:
 
   pgadmin:
     container_name: pgadmin
-    image: dpage/pgadmin4:7
+    image: dpage/pgadmin4:8
     restart: always
     environment:
       PGADMIN_DEFAULT_EMAIL: admin@example.com
@@ -55,7 +55,9 @@ services:
       - 6379:6379
 
   otel-collector:
+    container_name: otel-collector
     image: otel/opentelemetry-collector-contrib:latest
+    restart: always
     command: ["--config", "/etc/otel-collector-config.yaml"]
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml

packages/cli-v3/package.json

@@ -43,7 +43,9 @@
     "@types/object-hash": "^3.0.6",
     "@types/react": "^18.2.48",
     "@types/ws": "^8.5.3",
+    "cpy-cli": "^5.0.0",
     "npm-run-all": "^4.1.5",
+    "npm-watch": "^0.11.0",
     "open": "^10.0.3",
     "p-retry": "^6.1.0",
     "rimraf": "^3.0.2",
@@ -53,14 +55,23 @@
     "vitest": "^0.34.4",
     "xdg-app-paths": "^8.3.0"
   },
+  "watch": {
+    "build:prod-containerfile": "src/Containerfile.prod"
+  },
   "scripts": {
     "typecheck": "tsc",
     "build": "npm run clean && run-p build:**",
     "build:main": "tsup",
     "build:facade": "tsup --config tsup.facade.config.ts",
+    "build:prod-facade": "tsup --config tsup.prod-facade.config.ts",
+    "build:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts",
+    "build:prod-containerfile": "cpy --flat src/Containerfile.prod dist/",
     "dev": "npm run clean && run-p dev:**",
     "dev:main": "tsup --watch",
     "dev:facade": "tsup --config tsup.facade.config.ts --watch",
+    "dev:prod-facade": "tsup --config tsup.prod-facade.config.ts --watch",
+    "dev:prod-worker": "esbuild --platform=node --bundle --format=esm --target=esnext --outfile=dist/prod-worker.mjs --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);const path = require('path');const __dirname = path.resolve();\" ./src/prod-worker.ts --watch",
+    "dev:prod-containerfile": "npm-watch",
     "clean": "rimraf dist",
     "start": "node dist/index.js",
     "test": "vitest"
@@ -111,6 +122,7 @@
     "react": "^18.2.0",
     "react-error-boundary": "^4.0.12",
     "simple-git": "^3.19.0",
+    "socket.io-client": "^4.7.4",
     "source-map-support": "^0.5.21",
     "supports-color": "^9.4.0",
     "terminal-link": "^3.0.0",

packages/cli-v3/src/Containerfile.prod

@@ -0,0 +1,18 @@
+FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS base
+
+RUN apk add --no-cache dumb-init
+
+FROM base
+
+ENV TRIGGER_CONTENT_HASH=__CONTENT_HASH__
+ENV TRIGGER_PROJECT_DIR=__PROJECT_DIR__
+ENV TRIGGER_PROJECT_REF=__PROJECT_REF__
+ENV TRIGGER_CLI_PACKAGE_VERSION=__CLI_PACKAGE_VERSION__
+
+WORKDIR /app
+
+COPY --chown=node:node . .
+
+USER node
+
+CMD [ "dumb-init", "node", "index.mjs" ]

packages/cli-v3/src/apiClient.ts

@@ -2,18 +2,24 @@ import { z } from "zod";
 import {
   CreateAuthorizationCodeResponseSchema,
   GetPersonalAccessTokenResponseSchema,
-  GetProjectDevResponse,
-  CreateBackgroundWorkerRequestBody,
   WhoAmIResponseSchema,
+  CreateBackgroundWorkerRequestBody,
   CreateBackgroundWorkerResponse,
+  CreateImageDetailsRequestBody,
+  CreateImageDetailsResponse,
+  GetProjectDevResponse,
   GetEnvironmentVariablesResponseBody,
 } from "@trigger.dev/core/v3";
 
-export class ApiClient {
+export class CliApiClient {
+  private readonly apiURL: string;
+
   constructor(
-    private readonly apiURL: string,
+    apiURL: string,
     private readonly accessToken?: string
-  ) {}
+  ) {
+    this.apiURL = apiURL.replace(/\/$/, "");
+  }
 
   async createAuthorizationCode() {
     return zodfetch(
@@ -49,7 +55,7 @@ export class ApiClient {
 
   async createBackgroundWorker(projectRef: string, body: CreateBackgroundWorkerRequestBody) {
     if (!this.accessToken) {
-      throw new Error("indexProject: No access token");
+      throw new Error("createBackgroundWorker: No access token");
     }
 
     return zodfetch(
@@ -66,6 +72,25 @@ export class ApiClient {
     );
   }
 
+  async createImageDetails(projectRef: string, body: CreateImageDetailsRequestBody) {
+    if (!this.accessToken) {
+      throw new Error("createImageDetails: No access token");
+    }
+
+    return zodfetch(
+      CreateImageDetailsResponse,
+      `${this.apiURL}/api/v1/projects/${projectRef}/image-details`,
+      {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${this.accessToken}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(body),
+      }
+    );
+  }
+
   async getProjectDevEnv({ projectRef }: { projectRef: string }) {
     if (!this.accessToken) {
       throw new Error("getProjectDevEnv: No access token");
@@ -79,6 +104,19 @@ export class ApiClient {
     });
   }
 
+  async getProjectProdEnv({ projectRef }: { projectRef: string }) {
+    if (!this.accessToken) {
+      throw new Error("getProjectDevEnv: No access token");
+    }
+
+    return zodfetch(GetProjectDevResponse, `${this.apiURL}/api/v1/projects/${projectRef}/prod`, {
+      headers: {
+        Authorization: `Bearer ${this.accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+  }
+
   async getEnvironmentVariables(projectRef: string) {
     if (!this.accessToken) {
       throw new Error("getEnvironmentVariables: No access token");

packages/cli-v3/src/cli/index.ts

@@ -8,6 +8,7 @@ import { configureWhoamiCommand } from "../commands/whoami.js";
 import { COMMAND_NAME } from "../consts.js";
 import { getVersion } from "../utilities/getVersion.js";
 import { printInitialBanner } from "../utilities/initialBanner.js";
+import { configureBuildCommand } from "../commands/build.js";
 
 export const program = new Command();
 
@@ -55,6 +56,8 @@ program
     }
   });
 
+
+configureBuildCommand(program);
 configureDevCommand(program);
 
 program

packages/cli-v3/src/commands/build.ts

@@ -0,0 +1,316 @@
+import chalk from "chalk";
+import { Command } from "commander";
+import { build } from "esbuild";
+import { execa } from "execa";
+import { resolve as importResolve } from "import-meta-resolve";
+import { createHash } from "node:crypto";
+import fs, { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { z } from "zod";
+import * as packageJson from "../../package.json";
+import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
+import { logger } from "../utilities/logger.js";
+import { isLoggedIn } from "../utilities/session.js";
+import { CommonCommandOptions } from "../cli/common.js";
+import { getConfigPath, readConfig } from "../utilities/configFiles.js";
+import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles.js";
+import { ResolvedConfig } from "@trigger.dev/core/v3";
+import { CliApiClient } from "../apiClient";
+
+const BuildCommandOptions = CommonCommandOptions.extend({
+  registry: z.string(),
+  repo: z.string(),
+  skipTypecheck: z.boolean().optional(),
+});
+
+type BuildCommandOptions = z.infer<typeof BuildCommandOptions>;
+
+export function configureBuildCommand(program: Command) {
+  program
+    .command("build")
+    .description("Build your Trigger.dev tasks locally")
+    .argument("[path]", "The path to the project", ".")
+    .option("-r, --repo <repo_name>", "The repo to push images to", "task")
+    .option("-R, --registry <registry_address>", "The registry to push images to", "")
+    .option("-T, --skip-typecheck", "Whether to skip the pre-build typecheck")
+    .option(
+      "-l, --log-level <level>",
+      "The log level to use (debug, info, log, warn, error, none)",
+      "log"
+    )
+    .action(async (path, options) => {
+      try {
+        await buildCommand(path, options);
+      } catch (e) {
+        //todo error reporting
+        throw e;
+      }
+    });
+}
+
+export async function buildCommand(dir: string, anyOptions: unknown) {
+  const options = BuildCommandOptions.safeParse(anyOptions);
+
+  if (!options.success) {
+    throw new Error(`Invalid options: ${options.error}`);
+  }
+
+  const authorization = await isLoggedIn();
+
+  if (!authorization.ok) {
+    if (authorization.error === "fetch failed") {
+      logger.error("Fetch failed. Platform down?");
+    } else {
+      logger.error("You must login first. Use `trigger.dev login` to login.");
+    }
+    process.exitCode = 1;
+    return;
+  }
+
+  await startBuild(dir, options.data, authorization.config);
+}
+
+async function startBuild(
+  dir: string,
+  options: BuildCommandOptions,
+  authorization: { apiUrl: string; accessToken: string }
+) {
+  try {
+    if (options.logLevel) {
+      logger.loggerLevel = options.logLevel;
+    }
+
+    await printStandloneInitialBanner(true);
+
+    const configPath = await getConfigPath(dir);
+    const config = await readConfig(configPath);
+
+    const apiClient = new CliApiClient(authorization.apiUrl, authorization.accessToken);
+
+    const prodEnv = await apiClient.getProjectProdEnv({ projectRef: config.project });
+
+    if (!prodEnv.success) {
+      throw new Error(prodEnv.error);
+    }
+
+    const buildResult = await runBuild(config, options, {
+      apiUrl: authorization.apiUrl,
+      apiKey: prodEnv.data.apiKey,
+    });
+
+    const envClient = new CliApiClient(authorization.apiUrl, prodEnv.data.apiKey);
+    await envClient.createImageDetails(config.project, {
+      metadata: {
+        contentHash: buildResult.contentHash,
+        imageTag: buildResult.imageTag,
+      },
+    });
+  } catch (e) {
+    throw e;
+  }
+}
+
+async function runBuild(
+  config: ResolvedConfig,
+  options: BuildCommandOptions,
+  auth: { apiUrl: string; apiKey: string }
+) {
+  const taskFiles = await gatherTaskFiles(config);
+
+  const prodFacade = readFileSync(
+    new URL(importResolve("./prod-facade.js", import.meta.url)).href.replace("file://", ""),
+    "utf-8"
+  );
+  const entryPointContents = prodFacade.replace("__TASKS__", createTaskFileImports(taskFiles));
+
+  if (!options.skipTypecheck) {
+    logger.log(chalk.dim("⎔ Typecheck..."));
+
+    const tscTypecheck = execa("npm", ["exec", "tsc", "--", "--noEmit"]);
+
+    tscTypecheck.stdout?.on("data", (chunk) => logger.log(chunk.toString()));
+    tscTypecheck.stderr?.on("data", (chunk) => logger.error(chunk.toString()));
+
+    try {
+      await new Promise((resolve, reject) => {
+        tscTypecheck.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code)));
+      });
+    } catch (error) {
+      throw new Error("Typecheck failed.");
+    }
+
+    logger.log(chalk.green(`Typecheck succeeded.\n`));
+  }
+
+  logger.log(chalk.dim("⎔ Bundling tasks..."));
+
+  const result = await build({
+    stdin: {
+      contents: entryPointContents,
+      resolveDir: process.cwd(),
+      sourcefile: "__entryPoint.ts",
+    },
+    bundle: true,
+    metafile: true,
+    write: false,
+    minify: false,
+    sourcemap: true,
+    logLevel: "warning",
+    platform: "node",
+    format: "esm",
+    target: ["node18", "es2020"],
+    outdir: "out",
+    banner: {
+      js: "import { createRequire } from 'module';const require = createRequire(import.meta.url);",
+    },
+  });
+
+  if (result.errors.length > 0) {
+    logger.error(result.errors);
+    throw new Error("Build failed");
+  }
+  if (!result || !result.outputFiles) {
+    throw new Error("Build failed: no result");
+  }
+
+  const metaOutputKey = join("out", `stdin.js`);
+
+  const metaOutput = result.metafile!.outputs[metaOutputKey];
+
+  if (!metaOutput) {
+    throw new Error(`Could not find metafile`);
+  }
+
+  const outputFileKey = join(config.projectDir, metaOutputKey);
+  const outputFile = result.outputFiles.find((file) => file.path === outputFileKey);
+
+  if (!outputFile) {
+    throw new Error(`Could not find output file for entry point ${metaOutput.entryPoint}`);
+  }
+
+  const sourceMapFileKey = join(config.projectDir, `${metaOutputKey}.map`);
+  const sourceMapFile = result.outputFiles.find((file) => file.path === sourceMapFileKey);
+
+  if (!sourceMapFile) {
+    throw new Error(`Could not find source map file for entry point ${metaOutput.entryPoint}`);
+  }
+
+  const md5Hasher = createHash("md5");
+  md5Hasher.update(Buffer.from(outputFile.contents.buffer));
+
+  const contentHash = md5Hasher.digest("hex");
+  const buildContextPath = join(config.projectDir, ".trigger");
+
+  try {
+    // Clean build context dir first
+    await fs.promises.rm(buildContextPath, { recursive: true, force: true });
+  } catch (err) {
+  } finally {
+    // ..then ensure it exists
+    await fs.promises.mkdir(buildContextPath, { recursive: true });
+  }
+
+  // Create a file at join(dir, ".trigger", path) with the fileContents
+  const fullPath = join(buildContextPath, `${contentHash}.mjs`);
+  await fs.promises.writeFile(fullPath, outputFile.text);
+  const sourceMapPath = `${fullPath}.map`;
+  await fs.promises.writeFile(sourceMapPath, sourceMapFile.text);
+
+  const prodWorkerPath = new URL(importResolve("./prod-worker.mjs", import.meta.url)).href.replace(
+    "file://",
+    ""
+  );
+  await fs.promises.copyFile(prodWorkerPath, join(buildContextPath, "index.mjs"));
+
+  logger.log(chalk.green(`Bundling finished.\n`));
+
+  let localOnly = false;
+
+  if (!options.registry) {
+    logger.log(chalk.yellow(`No registry specified, enabling local only mode.\n`));
+    localOnly = true;
+  }
+
+  const registryWithRepo = localOnly ? options.repo : `${options.registry}/${options.repo}`;
+
+  if (!localOnly) {
+    logger.log(chalk.dim("⎔ Checking repo login..."));
+
+    const dockerLogin = execa("docker", ["login", registryWithRepo]);
+
+    dockerLogin.stdout?.on("data", (chunk) => logger.debug(chunk.toString()));
+    dockerLogin.stderr?.on("data", (chunk) => logger.error(chunk.toString()));
+
+    try {
+      await new Promise((resolve, reject) => {
+        dockerLogin.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code)));
+      });
+    } catch (error) {
+      throw new Error("Login failed. Please run `docker login` to authenticate.");
+    }
+
+    logger.log(chalk.green(`Login succeeded.\n`));
+  }
+
+  logger.log(chalk.dim("⎔ Starting docker build..."));
+
+  const containerfile = await fs.promises.readFile(
+    new URL(importResolve("./Containerfile.prod", import.meta.url)).href.replace("file://", ""),
+    "utf-8"
+  );
+
+  const containerfileContents = containerfile
+    .replace("__CONTENT_HASH__", contentHash)
+    .replace("__PROJECT_DIR__", config.projectDir)
+    .replace("__PROJECT_REF__", config.project)
+    .replace("__CLI_PACKAGE_VERSION__", packageJson.version);
+
+  const containerfilePath = join(buildContextPath, "Containerfile");
+  const imageTag = `${registryWithRepo}:${contentHash}`;
+
+  await fs.promises.writeFile(containerfilePath, containerfileContents);
+
+  const dockerBuild = execa("docker", [
+    "build",
+    "-f",
+    containerfilePath,
+    "-t",
+    imageTag,
+    join(config.projectDir, ".trigger"),
+  ]);
+  dockerBuild.stdout?.pipe(process.stdout);
+  dockerBuild.stderr?.pipe(process.stderr);
+
+  try {
+    await new Promise((resolve, reject) => {
+      dockerBuild.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code)));
+    });
+  } catch (error) {
+    throw new Error("Build failed.");
+  }
+
+  logger.log(chalk.green(`Build finished.\n`));
+
+  if (!localOnly) {
+    logger.log(chalk.dim("⎔ Pushing image..."));
+
+    const dockerPush = execa("docker", ["push", imageTag]);
+    dockerPush.stdout?.pipe(process.stdout);
+    dockerPush.stderr?.pipe(process.stderr);
+
+    try {
+      await new Promise((resolve, reject) => {
+        dockerPush.addListener("exit", (code) => (code === 0 ? resolve(code) : reject(code)));
+      });
+    } catch (error) {
+      throw new Error("Push failed.");
+    }
+
+    logger.log(chalk.green(`Push complete.\n`));
+  }
+
+  return {
+    contentHash,
+    imageTag,
+  };
+}

packages/cli-v3/src/commands/dev.tsx

@@ -1,5 +1,6 @@
 import {
   CreateBackgroundWorkerRequestBody,
+  ResolvedConfig,
   TaskResource,
   ZodMessageHandler,
   ZodMessageSender,
@@ -10,48 +11,28 @@ import chalk from "chalk";
 import { watch } from "chokidar";
 import { Command } from "commander";
 import { BuildContext, context } from "esbuild";
-import { findUp } from "find-up";
 import { resolve as importResolve } from "import-meta-resolve";
 import { Box, Text, render, useApp, useInput } from "ink";
 import { createHash } from "node:crypto";
 import fs, { readFileSync } from "node:fs";
 import { ClientRequestArgs } from "node:http";
-import { basename, dirname, join, relative, resolve } from "node:path";
-import { pathToFileURL } from "node:url";
+import { basename, dirname, join } from "node:path";
 import pThrottle from "p-throttle";
 import { WebSocket } from "partysocket";
 import React, { Suspense, useEffect } from "react";
 import { ClientOptions, WebSocket as wsWebSocket } from "ws";
 import { z } from "zod";
 import * as packageJson from "../../package.json";
-import { ApiClient } from "../apiClient.js";
-import { CLOUD_API_URL } from "../consts.js";
 import { BackgroundWorker, BackgroundWorkerCoordinator } from "../dev/backgroundWorker.js";
 import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
 import { logger } from "../utilities/logger.js";
-import { RequireKeys } from "../utilities/requiredKeys.js";
 import { isLoggedIn } from "../utilities/session.js";
 import { CommonCommandOptions } from "../cli/common.js";
+import { getConfigPath, readConfig } from "../utilities/configFiles";
+import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
+import { CliApiClient } from "../apiClient";
 
-const CONFIG_FILES = ["trigger.config.js", "trigger.config.mjs"];
-
-const ConfigSchema = z.object({
-  project: z.string(),
-  triggerDirectories: z.string().array().optional(),
-  triggerUrl: z.string().optional(),
-  projectDir: z.string().optional(),
-});
-
-type Config = z.infer<typeof ConfigSchema>;
-type ResolvedConfig = RequireKeys<Config, "triggerDirectories" | "triggerUrl" | "projectDir">;
-type TaskFile = {
-  triggerDir: string;
-  filePath: string;
-  importPath: string;
-  importName: string;
-};
-
-let apiClient: ApiClient | undefined;
+let apiClient: CliApiClient | undefined;
 
 const DevCommandOptions = CommonCommandOptions.extend({
   debugger: z.boolean().default(false),
@@ -92,7 +73,11 @@ export async function devCommand(dir: string, anyOptions: unknown) {
   const authorization = await isLoggedIn();
 
   if (!authorization.ok) {
-    logger.error("You must login first. Use `trigger.dev login` to login.");
+    if (authorization.error === "fetch failed") {
+      logger.error("Fetch failed. Platform down?");
+    } else {
+      logger.error("You must login first. Use `trigger.dev login` to login.");
+    }
     process.exitCode = 1;
     return;
   }
@@ -143,7 +128,7 @@ async function startDev(
       const accessToken = authorization.accessToken;
       const apiUrl = authorization.apiUrl;
 
-      apiClient = new ApiClient(apiUrl, accessToken);
+      apiClient = new CliApiClient(apiUrl, accessToken);
 
       const devEnv = await apiClient.getProjectDevEnv({ projectRef: config.project });
 
@@ -151,7 +136,7 @@ async function startDev(
         throw new Error(devEnv.error);
       }
 
-      const environmentClient = new ApiClient(apiUrl, devEnv.data.apiKey);
+      const environmentClient = new CliApiClient(apiUrl, devEnv.data.apiKey);
 
       return (
         <DevUI
@@ -188,7 +173,7 @@ type DevProps = {
   config: ResolvedConfig;
   apiUrl: string;
   apiKey: string;
-  environmentClient: ApiClient;
+  environmentClient: CliApiClient;
   projectName: string;
   debuggerOn: boolean;
   debugOtel: boolean;
@@ -623,119 +608,3 @@ function WebsocketFactory(apiKey: string) {
     }
   };
 }
-
-function createTaskFileImports(taskFiles: TaskFile[]) {
-  return taskFiles
-    .map(
-      (taskFile) =>
-        `import * as ${taskFile.importName} from "./${taskFile.importPath}"; TaskFileImports["${
-          taskFile.importName
-        }"] = ${taskFile.importName}; TaskFiles["${taskFile.importName}"] = ${JSON.stringify(
-          taskFile
-        )};`
-    )
-    .join("\n");
-}
-
-// Find all the top-level .js or .ts files in the trigger directories
-async function gatherTaskFiles(config: ResolvedConfig): Promise<Array<TaskFile>> {
-  const taskFiles: Array<TaskFile> = [];
-
-  for (const triggerDir of config.triggerDirectories) {
-    const files = await fs.promises.readdir(triggerDir, { withFileTypes: true });
-    for (const file of files) {
-      if (!file.isFile()) continue;
-      if (!file.name.endsWith(".js") && !file.name.endsWith(".ts")) continue;
-
-      const fullPath = join(triggerDir, file.name);
-
-      const filePath = relative(config.projectDir, fullPath);
-      const importPath = filePath.replace(/\.(js|ts)$/, "");
-      const importName = importPath.replace(/\//g, "_");
-
-      taskFiles.push({ triggerDir, importPath, importName, filePath });
-    }
-  }
-
-  return taskFiles;
-}
-
-async function getConfigPath(dir: string): Promise<string> {
-  const path = await findUp(CONFIG_FILES, { cwd: dir });
-
-  if (!path) {
-    throw new Error("No config file found.");
-  }
-
-  return path;
-}
-
-async function readConfig(path: string): Promise<ResolvedConfig> {
-  try {
-    // import the config file
-    const userConfigModule = await import(`${pathToFileURL(path).href}?_ts=${Date.now()}`);
-    const rawConfig = await normalizeConfig(userConfigModule ? userConfigModule.default : {});
-    const config = ConfigSchema.parse(rawConfig);
-
-    return resolveConfig(path, config);
-  } catch (error) {
-    console.error(`Failed to load config file at ${path}`);
-    throw error;
-  }
-}
-
-async function resolveConfig(path: string, config: Config): Promise<ResolvedConfig> {
-  if (!config.triggerDirectories) {
-    config.triggerDirectories = await findTriggerDirectories(path);
-  }
-
-  config.triggerDirectories = resolveTriggerDirectories(config.triggerDirectories);
-
-  if (!config.triggerUrl) {
-    config.triggerUrl = CLOUD_API_URL;
-  }
-
-  if (!config.projectDir) {
-    config.projectDir = dirname(path);
-  }
-
-  return config as ResolvedConfig;
-}
-
-async function normalizeConfig(config: any): Promise<any> {
-  if (typeof config === "function") {
-    config = config();
-  }
-
-  return await config;
-}
-
-function resolveTriggerDirectories(dirs: string[]): string[] {
-  return dirs.map((dir) => resolve(dir));
-}
-
-const IGNORED_DIRS = ["node_modules", ".git", "dist", "build"];
-
-async function findTriggerDirectories(filePath: string): Promise<string[]> {
-  const dirPath = dirname(filePath);
-  return getTriggerDirectories(dirPath);
-}
-
-async function getTriggerDirectories(dirPath: string): Promise<string[]> {
-  const entries = await fs.promises.readdir(dirPath, { withFileTypes: true });
-  const triggerDirectories: string[] = [];
-
-  for (const entry of entries) {
-    if (!entry.isDirectory() || IGNORED_DIRS.includes(entry.name)) continue;
-
-    const fullPath = join(dirPath, entry.name);
-
-    if (entry.name === "trigger") {
-      triggerDirectories.push(fullPath);
-    }
-
-    triggerDirectories.push(...(await getTriggerDirectories(fullPath)));
-  }
-
-  return triggerDirectories;
-}

packages/cli-v3/src/commands/login.ts

@@ -1,12 +1,12 @@
 import { intro, log, outro, select, spinner } from "@clack/prompts";
 import open from "open";
 import pRetry, { AbortError } from "p-retry";
-import { ApiClient } from "../apiClient.js";
 import { ApiUrlOptionsSchema } from "../cli/index.js";
 import { chalkLink } from "../utilities/colors.js";
 import { readAuthConfigFile, writeAuthConfigFile } from "../utilities/configFiles.js";
 import { logger } from "../utilities/logger.js";
 import { whoAmI } from "./whoami.js";
+import { CliApiClient } from "../apiClient.js";
 
 export async function loginCommand(options: any) {
   const result = ApiUrlOptionsSchema.safeParse(options);
@@ -29,7 +29,7 @@ export type LoginResult =
     };
 
 export async function login(apiUrl: string): Promise<LoginResult> {
-  const apiClient = new ApiClient(apiUrl);
+  const apiClient = new CliApiClient(apiUrl);
 
   intro("Logging in to Trigger.dev");
 
@@ -118,7 +118,7 @@ export async function login(apiUrl: string): Promise<LoginResult> {
   }
 }
 
-async function getPersonalAccessToken(apiClient: ApiClient, authorizationCode: string) {
+async function getPersonalAccessToken(apiClient: CliApiClient, authorizationCode: string) {
   const token = await apiClient.getPersonalAccessToken(authorizationCode);
 
   if (!token.success) {

packages/cli-v3/src/commands/whoami.ts

@@ -1,5 +1,4 @@
 import { note, spinner } from "@clack/prompts";
-import { ApiClient } from "../apiClient.js";
 import { chalkLink } from "../utilities/colors.js";
 import { logger } from "../utilities/logger.js";
 import { isLoggedIn } from "../utilities/session.js";
@@ -7,6 +6,7 @@ import { Command } from "commander";
 import { printInitialBanner } from "../utilities/initialBanner.js";
 import { CommonCommandOptions } from "../cli/common.js";
 import { z } from "zod";
+import { CliApiClient } from "../apiClient.js";
 
 type WhoAmIResult =
   | {
@@ -55,7 +55,11 @@ export async function whoAmI(options?: WhoamiCommandOptions): Promise<WhoAmIResu
   const authentication = await isLoggedIn();
 
   if (!authentication.ok) {
-    loadingSpinner.stop("You must login first. Use `trigger.dev login` to login.");
+    if (authentication.error === "fetch failed") {
+      loadingSpinner.stop("Fetch failed. Platform down?");
+    } else {
+      loadingSpinner.stop("You must login first. Use `trigger.dev login` to login.");
+    }
 
     return {
       success: false,
@@ -63,7 +67,10 @@ export async function whoAmI(options?: WhoamiCommandOptions): Promise<WhoAmIResu
     };
   }
 
-  const apiClient = new ApiClient(authentication.config.apiUrl, authentication.config.accessToken);
+  const apiClient = new CliApiClient(
+    authentication.config.apiUrl,
+    authentication.config.accessToken
+  );
   const userData = await apiClient.whoAmI();
 
   if (!userData.success) {

packages/cli-v3/src/consts.ts

@@ -10,3 +10,4 @@ export const PKG_ROOT = path.join(distPath, "../");
 export const COMMAND_NAME = "trigger.dev";
 export const CLOUD_WEB_URL = "https://cloud.trigger.dev";
 export const CLOUD_API_URL = "https://api.trigger.dev";
+export const CONFIG_FILES = ["trigger.config.js", "trigger.config.mjs"];

packages/cli-v3/src/dev/backgroundWorker.ts

@@ -532,7 +532,7 @@ class TaskRunProcess {
   }
 
   taskRunCompletedNotification(completion: TaskRunExecutionResult, execution: TaskRunExecution) {
-    if (!completion.ok && typeof completion.retry === "undefined") {
+    if (!completion.ok && typeof completion.retry !== "undefined") {
       return;
     }
 

packages/cli-v3/src/prod-facade.ts

@@ -0,0 +1,378 @@
+// import "source-map-support/register";
+import { TracingSDK } from "@trigger.dev/core/v3/otel";
+// import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
+
+// IMPORTANT: this needs to be the first import to work properly
+// WARNING: [WARNING] Constructing "ImportInTheMiddle" will crash at run-time because it's an import namespace object, not a constructor [call-import-namespace]
+// TODO: https://github.com/open-telemetry/opentelemetry-js/issues/3954
+const tracingSDK = new TracingSDK({
+  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
+  resource: new Resource({
+    [SemanticInternalAttributes.CLI_VERSION]: packageJson.version,
+  }),
+  instrumentations: [
+    // new OpenAIInstrumentation(),
+  ],
+  diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
+});
+
+const otelTracer = tracingSDK.getTracer("trigger-prod-worker", packageJson.version);
+const otelLogger = tracingSDK.getLogger("trigger-prod-worker", packageJson.version);
+
+import { SpanKind } from "@opentelemetry/api";
+import {
+  ConsoleInterceptor,
+  ProdRuntimeManager,
+  OtelTaskLogger,
+  SemanticInternalAttributes,
+  TaskMetadataWithFilePath,
+  TaskRunContext,
+  TaskRunErrorCodes,
+  TaskRunExecution,
+  TaskRunExecutionRetry,
+  TriggerTracer,
+  ZodMessageHandler,
+  ZodMessageSender,
+  accessoryAttributes,
+  calculateNextRetryDelay,
+  childToWorkerMessages,
+  logger,
+  parseError,
+  runtime,
+  taskContextManager,
+  workerToChildMessages,
+  type BackgroundWorkerProperties,
+} from "@trigger.dev/core/v3";
+import * as packageJson from "../package.json";
+
+import { Resource } from "@opentelemetry/resources";
+import { flattenAttributes } from "@trigger.dev/core/v3";
+import { TaskMetadataWithFunctions } from "./types";
+import { TracingDiagnosticLogLevel } from "@trigger.dev/core/v3/otel/tracingSDK";
+
+const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger });
+const consoleInterceptor = new ConsoleInterceptor(otelLogger);
+
+const sender = new ZodMessageSender({
+  schema: childToWorkerMessages,
+  sender: async (message) => {
+    process.send?.(message);
+  },
+});
+
+const prodRuntimeManager = new ProdRuntimeManager(sender);
+
+runtime.setGlobalRuntimeManager(prodRuntimeManager);
+
+const otelTaskLogger = new OtelTaskLogger({
+  logger: otelLogger,
+  tracer: tracer,
+  level: "info",
+});
+
+logger.setGlobalTaskLogger(otelTaskLogger);
+
+type TaskFileImport = Record<string, unknown>;
+
+const TaskFileImports: Record<string, TaskFileImport> = {};
+const TaskFiles: Record<string, string> = {};
+
+__TASKS__;
+declare const __TASKS__: Record<string, string>;
+
+class TaskExecutor {
+  constructor(public task: TaskMetadataWithFunctions) {}
+
+  async determineRetrying(
+    execution: TaskRunExecution,
+    error: unknown
+  ): Promise<TaskRunExecutionRetry | undefined> {
+    if (!this.task.retry) {
+      return;
+    }
+
+    const retry = this.task.retry;
+
+    const delay = calculateNextRetryDelay(retry, execution.attempt.number);
+
+    return typeof delay === "undefined" ? undefined : { timestamp: Date.now() + delay, delay };
+  }
+
+  async execute(
+    execution: TaskRunExecution,
+    worker: BackgroundWorkerProperties,
+    traceContext: Record<string, unknown>
+  ) {
+    const parsedPayload = JSON.parse(execution.run.payload);
+    const ctx = TaskRunContext.parse(execution);
+    const attemptMessage = `Attempt ${execution.attempt.number}`;
+
+    const output = await taskContextManager.runWith(
+      {
+        ctx,
+        payload: parsedPayload,
+        worker,
+      },
+      async () => {
+        tracingSDK.asyncResourceDetector.resolveWithAttributes({
+          ...taskContextManager.attributes,
+          [SemanticInternalAttributes.SDK_VERSION]: this.task.packageVersion,
+          [SemanticInternalAttributes.SDK_LANGUAGE]: "typescript",
+        });
+
+        return await tracer.startActiveSpan(
+          attemptMessage,
+          async (span) => {
+            return await consoleInterceptor.intercept(console, async () => {
+              const init = await this.#callTaskInit(parsedPayload, ctx);
+
+              try {
+                const output = await this.#callRun(parsedPayload, ctx, init);
+
+                span.setAttributes(flattenAttributes(output, SemanticInternalAttributes.OUTPUT));
+
+                return output;
+              } finally {
+                await this.#callTaskCleanup(parsedPayload, ctx, init);
+              }
+            });
+          },
+          {
+            kind: SpanKind.CONSUMER,
+            attributes: {
+              [SemanticInternalAttributes.STYLE_ICON]: "attempt",
+              ...flattenAttributes(parsedPayload, SemanticInternalAttributes.PAYLOAD),
+              ...accessoryAttributes({
+                items: [
+                  {
+                    text: ctx.task.filePath,
+                  },
+                  {
+                    text: `${ctx.task.exportName}.run()`,
+                  },
+                ],
+                style: "codepath",
+              }),
+            },
+          },
+          tracer.extractContext(traceContext)
+        );
+      }
+    );
+
+    return { output: JSON.stringify(output), outputType: "application/json" };
+  }
+
+  async #callRun(payload: unknown, ctx: TaskRunContext, init: unknown) {
+    const runFn = this.task.fns.run;
+    const middlewareFn = this.task.fns.middleware;
+
+    if (!runFn) {
+      throw new Error("Task does not have a run function");
+    }
+
+    if (!middlewareFn) {
+      return runFn({ payload, ctx });
+    }
+
+    return middlewareFn({ payload, ctx, next: async () => runFn({ payload, ctx, init }) });
+  }
+
+  async #callTaskInit(payload: unknown, ctx: TaskRunContext) {
+    const initFn = this.task.fns.init;
+
+    if (!initFn) {
+      return {};
+    }
+
+    return tracer.startActiveSpan("init", async (span) => {
+      return await initFn({ payload, ctx });
+    });
+  }
+
+  async #callTaskCleanup(payload: unknown, ctx: TaskRunContext, init: unknown) {
+    const cleanupFn = this.task.fns.cleanup;
+
+    if (!cleanupFn) {
+      return;
+    }
+
+    return tracer.startActiveSpan("cleanup", async (span) => {
+      return await cleanupFn({ payload, ctx, init });
+    });
+  }
+}
+
+function getTasks(): Array<TaskMetadataWithFunctions> {
+  const result: Array<TaskMetadataWithFunctions> = [];
+
+  for (const [importName, taskFile] of Object.entries(TaskFiles)) {
+    const fileImports = TaskFileImports[importName];
+
+    for (const [exportName, task] of Object.entries(fileImports ?? {})) {
+      if ((task as any).__trigger) {
+        result.push({
+          id: (task as any).__trigger.id,
+          exportName,
+          packageVersion: (task as any).__trigger.packageVersion,
+          filePath: (taskFile as any).filePath,
+          queue: (task as any).__trigger.queue,
+          retry: (task as any).__trigger.retry,
+          fns: (task as any).__trigger.fns,
+        });
+      }
+    }
+  }
+
+  return result;
+}
+
+function getTaskMetadata(): Array<TaskMetadataWithFilePath> {
+  const result = getTasks();
+
+  // Remove the functions from the metadata
+  return result.map((task) => {
+    const { fns, ...metadata } = task;
+
+    return metadata;
+  });
+}
+
+const tasks = getTasks();
+
+runtime.registerTasks(tasks);
+
+const taskExecutors: Map<string, TaskExecutor> = new Map();
+
+for (const task of tasks) {
+  taskExecutors.set(task.id, new TaskExecutor(task));
+}
+
+let _execution: TaskRunExecution | undefined;
+let _isRunning = false;
+
+const handler = new ZodMessageHandler({
+  schema: workerToChildMessages,
+  messages: {
+    EXECUTE_TASK_RUN: async ({ execution, traceContext, metadata }) => {
+      if (_isRunning) {
+        console.error("Worker is already running a task");
+
+        await sender.send("TASK_RUN_COMPLETED", {
+          execution,
+          result: {
+            ok: false,
+            id: execution.attempt.id,
+            error: {
+              type: "INTERNAL_ERROR",
+              code: TaskRunErrorCodes.TASK_ALREADY_RUNNING,
+            },
+          },
+        });
+
+        return;
+      }
+      process.title = `trigger-prod-worker: ${execution.task.id} ${execution.run.id}`;
+
+      const executor = taskExecutors.get(execution.task.id);
+
+      if (!executor) {
+        console.error(`Could not find executor for task ${execution.task.id}`);
+
+        await sender.send("TASK_RUN_COMPLETED", {
+          execution,
+          result: {
+            ok: false,
+            id: execution.attempt.id,
+            error: {
+              type: "INTERNAL_ERROR",
+              code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR,
+            },
+          },
+        });
+
+        return;
+      }
+
+      try {
+        _execution = execution;
+        _isRunning = true;
+
+        const result = await executor.execute(execution, metadata, traceContext);
+
+        return sender.send("TASK_RUN_COMPLETED", {
+          execution,
+          result: {
+            id: execution.attempt.id,
+            ok: true,
+            ...result,
+          },
+        });
+      } catch (e) {
+        return sender.send("TASK_RUN_COMPLETED", {
+          execution,
+          result: {
+            id: execution.attempt.id,
+            ok: false,
+            error: parseError(e),
+            retry: await executor.determineRetrying(execution, e),
+          },
+        });
+      } finally {
+        _execution = undefined;
+        _isRunning = false;
+      }
+    },
+    TASK_RUN_COMPLETED_NOTIFICATION: async ({ completion, execution }) => {
+      prodRuntimeManager.resumeTask(completion, execution);
+    },
+    CLEANUP: async ({ flush, kill }) => {
+      if (kill) {
+        await tracingSDK.flush();
+        // Now we need to exit the process
+        await sender.send("READY_TO_DISPOSE", undefined);
+      } else {
+        if (flush) {
+          await tracingSDK.flush();
+        }
+      }
+    },
+  },
+});
+
+process.on("message", async (msg: any) => {
+  await handler.handleMessage(msg);
+});
+
+sender.send("TASKS_READY", { tasks: getTaskMetadata() }).catch((err) => {
+  console.error("Failed to send TASKS_READY message", err);
+});
+
+process.title = "trigger-prod-worker";
+
+async function asyncHeartbeat(initialDelayInSeconds: number = 30, intervalInSeconds: number = 5) {
+  async function _doHeartbeat() {
+    while (true) {
+      if (_isRunning && _execution) {
+        try {
+          await sender.send("TASK_HEARTBEAT", { id: _execution.attempt.id });
+        } catch (err) {
+          console.error("Failed to send HEARTBEAT message", err);
+        }
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 1000 * intervalInSeconds));
+    }
+  }
+
+  // Wait for the initial delay
+  await new Promise((resolve) => setTimeout(resolve, 1000 * initialDelayInSeconds));
+
+  // Wait for 5 seconds before the next execution
+  return _doHeartbeat();
+}
+
+// Start the async interval after initial delay
+asyncHeartbeat(5).catch((err) => {
+  console.error("Failed to start asyncHeartbeat", err);
+});

packages/cli-v3/src/prod-worker.ts

@@ -0,0 +1,317 @@
+import {
+  CoordinatorToProdWorkerEvents,
+  ProdWorkerToCoordinatorEvents,
+  TaskResource,
+} from "@trigger.dev/core/v3";
+import { HttpReply, getTextBody, SimpleLogger, getRandomPortNumber } from "@trigger.dev/core-apps";
+import { createServer } from "node:http";
+import { io, Socket } from "socket.io-client";
+import { ProdBackgroundWorker } from "./prod/backgroundWorker";
+
+const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber());
+const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+const COORDINATOR_PORT = Number(process.env.COORDINATOR_PORT || 50080);
+const MACHINE_NAME = process.env.MACHINE_NAME || "local";
+const POD_NAME = process.env.POD_NAME || "some-pod";
+const SHORT_HASH = process.env.TRIGGER_CONTENT_HASH!.slice(0, 9);
+
+const logger = new SimpleLogger(`[${MACHINE_NAME}][${SHORT_HASH}]`);
+
+class ProdWorker {
+  private apiUrl = process.env.TRIGGER_API_URL!;
+  private apiKey = process.env.TRIGGER_API_KEY!;
+  private contentHash = process.env.TRIGGER_CONTENT_HASH!;
+  private projectDir = process.env.TRIGGER_PROJECT_DIR!;
+  private projectRef = process.env.TRIGGER_PROJECT_REF!;
+  private envId = process.env.TRIGGER_ENV_ID!;
+  private cliPackageVersion = process.env.TRIGGER_CLI_PACKAGE_VERSION!;
+  private attemptId = process.env.TRIGGER_ATTEMPT_ID || "index-only";
+
+  private executing = false;
+  private completed = false;
+
+  #httpPort: number;
+  #backgroundWorker: ProdBackgroundWorker;
+  #httpServer: ReturnType<typeof createServer>;
+  #coordinatorSocket: Socket<CoordinatorToProdWorkerEvents, ProdWorkerToCoordinatorEvents>;
+
+  constructor(
+    port: number,
+    private host = "0.0.0.0"
+  ) {
+    this.#coordinatorSocket = this.#createCoordinatorSocket();
+
+    this.#backgroundWorker = new ProdBackgroundWorker(this.#getWorkerEntryPath(this.contentHash), {
+      projectDir: this.projectDir,
+      env: {
+        TRIGGER_API_URL: this.apiUrl,
+        TRIGGER_API_KEY: this.apiKey,
+      },
+      contentHash: this.contentHash,
+    });
+    this.#backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
+      this.#coordinatorSocket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
+    });
+    this.#backgroundWorker.onWaitForBatch.attach((message) => {
+      this.#coordinatorSocket.emit("WAIT_FOR_BATCH", { version: "v1", ...message });
+    });
+    this.#backgroundWorker.onWaitForDuration.attach((message) => {
+      this.#coordinatorSocket.emit(
+        "WAIT_FOR_DURATION",
+        { version: "v1", ...message },
+        ({ success }) => {
+          logger.log("WAIT_FOR_DURATION", { success });
+        }
+      );
+    });
+    this.#backgroundWorker.onWaitForTask.attach((message) => {
+      this.#coordinatorSocket.emit("WAIT_FOR_TASK", { version: "v1", ...message });
+    });
+
+    this.#httpPort = port;
+    this.#httpServer = this.#createHttpServer();
+  }
+
+  #createCoordinatorSocket() {
+    const socket: Socket<CoordinatorToProdWorkerEvents, ProdWorkerToCoordinatorEvents> = io(
+      `ws://${COORDINATOR_HOST}:${COORDINATOR_PORT}/prod-worker`,
+      {
+        transports: ["websocket"],
+        extraHeaders: {
+          "x-machine-name": MACHINE_NAME,
+          "x-pod-name": POD_NAME,
+          "x-trigger-content-hash": this.contentHash,
+          "x-trigger-cli-package-version": this.cliPackageVersion,
+          "x-trigger-project-ref": this.projectRef,
+          "x-trigger-attempt-id": this.attemptId,
+          "x-trigger-env-id": this.envId,
+        },
+      }
+    );
+
+    const logger = new SimpleLogger(`[coordinator][${socket.id ?? "NO_ID"}]`);
+
+    socket.on("connect_error", (err) => {
+      logger.error(`connect_error: ${err.message}`);
+    });
+
+    socket.on("connect", async () => {
+      logger.log("connect");
+
+      if (process.env.INDEX_TASKS === "true") {
+        const taskResources = await this.#initializeWorker();
+        const { success } = await socket.emitWithAck("INDEX_TASKS", {
+          version: "v1",
+          ...taskResources,
+        });
+        if (success) {
+          logger.log("indexing done, shutting down..");
+          process.exit(0);
+        } else {
+          logger.log("indexing failure, shutting down..");
+          process.exit(1);
+        }
+      } else {
+        socket.emit("READY_FOR_EXECUTION", {
+          version: "v1",
+          attemptId: process.env.TRIGGER_ATTEMPT_ID!,
+        });
+      }
+    });
+
+    socket.on("disconnect", () => {
+      logger.log("disconnect");
+    });
+
+    socket.on("RESUME", async (message) => {
+      logger.log("[RESUME]", message);
+
+      for (let i = 0; i < message.completions.length; i++) {
+        const completion = message.completions[i];
+        const execution = message.executions[i];
+
+        if (!completion || !execution) continue;
+
+        this.#backgroundWorker.taskRunCompletedNotification(completion, execution);
+      }
+    });
+
+    socket.on("EXECUTE_TASK_RUN", async (message, callback) => {
+      logger.log("[EXECUTE_TASK_RUN]", { attempt: message.payload.execution.attempt });
+
+      if (this.executing || this.completed) {
+        return;
+      }
+
+      this.executing = true;
+      const completion = await this.#backgroundWorker.executeTaskRun(message.payload);
+
+      logger.log("completed", completion);
+
+      // TODO: replace ack with emit
+      callback({ completion });
+
+      this.completed = true;
+      this.executing = false;
+
+      setTimeout(() => {
+        process.exit(0);
+      }, 1000);
+    });
+
+    return socket;
+  }
+
+  #createHttpServer() {
+    const httpServer = createServer(async (req, res) => {
+      logger.log(`[${req.method}]`, req.url);
+
+      const reply = new HttpReply(res);
+
+      switch (req.url) {
+        case "/complete":
+          setTimeout(() => process.exit(0), 1000);
+          return reply.text("ok");
+
+        case "/date":
+          const date = new Date();
+          return reply.text(date.toString());
+
+        case "/fail":
+          setTimeout(() => process.exit(1), 1000);
+          return reply.text("ok");
+
+        case "/health":
+          return reply.text("ok");
+
+        case "/whoami":
+          return reply.text(this.contentHash);
+
+        case "/wait":
+          this.#coordinatorSocket.emit(
+            "WAIT_FOR_DURATION",
+            {
+              version: "v1",
+              ms: 60_000,
+            },
+            ({ success }) => {
+              logger.log("WAIT_FOR_DURATION", { success });
+            }
+          );
+          // this is required when C/Ring established connections
+          this.#coordinatorSocket.close();
+          return reply.text("sent WAIT");
+
+        case "/connect":
+          this.#coordinatorSocket.connect();
+          return reply.empty();
+
+        case "/close":
+          this.#coordinatorSocket.emitWithAck("LOG", {
+            version: "v1",
+            text: "close without delay",
+          });
+          this.#coordinatorSocket.close();
+          return reply.empty();
+
+        case "/close-delay":
+          this.#coordinatorSocket.emitWithAck("LOG", {
+            version: "v1",
+            text: "close with delay",
+          });
+          setTimeout(() => {
+            this.#coordinatorSocket.close();
+          }, 200);
+          return reply.empty();
+
+        case "/log":
+          this.#coordinatorSocket.emitWithAck("LOG", {
+            version: "v1",
+            text: await getTextBody(req),
+          });
+          return reply.empty();
+
+        case "/preStop":
+          logger.log("should do preStop stuff, e.g. checkpoint and graceful shutdown");
+          return reply.text("got preStop request");
+
+        case "/ready":
+          this.#coordinatorSocket.emit("READY_FOR_EXECUTION", {
+            version: "v1",
+            attemptId: this.attemptId,
+          });
+          return reply.empty();
+
+        default:
+          return reply.empty(404);
+      }
+    });
+
+    httpServer.on("clientError", (err, socket) => {
+      socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+    });
+
+    httpServer.on("listening", () => {
+      logger.log("http server listening on port", this.#httpPort);
+    });
+
+    httpServer.on("error", (error) => {
+      // @ts-expect-error
+      if (error.code != "EADDRINUSE") {
+        return;
+      }
+
+      logger.error(`port ${this.#httpPort} already in use, retrying with random port..`);
+
+      this.#httpPort = getRandomPortNumber();
+
+      setTimeout(() => {
+        this.start();
+      }, 100);
+    });
+
+    return httpServer;
+  }
+
+  #getWorkerEntryPath(contentHash: string) {
+    return `${contentHash}.mjs`;
+  }
+
+  async #initializeWorker() {
+    await this.#backgroundWorker.initialize();
+
+    let packageVersion: string | undefined;
+
+    const taskResources: Array<TaskResource> = [];
+
+    if (!this.#backgroundWorker.tasks) {
+      throw new Error(`Background Worker started without tasks`);
+    }
+
+    for (const task of this.#backgroundWorker.tasks) {
+      taskResources.push({
+        id: task.id,
+        filePath: task.filePath,
+        exportName: task.exportName,
+      });
+
+      packageVersion = task.packageVersion;
+    }
+
+    if (!packageVersion) {
+      throw new Error(`Background Worker started without package version`);
+    }
+
+    return {
+      packageVersion,
+      tasks: taskResources,
+    };
+  }
+
+  start() {
+    this.#httpServer.listen(this.#httpPort, this.host);
+  }
+}
+
+const prodWorker = new ProdWorker(HTTP_SERVER_PORT);
+prodWorker.start();

packages/cli-v3/src/prod/backgroundWorker.ts

@@ -0,0 +1,514 @@
+import {
+  BackgroundWorkerProperties,
+  CreateBackgroundWorkerResponse,
+  ProdTaskRunExecutionPayload,
+  SemanticInternalAttributes,
+  TaskMetadataWithFilePath,
+  TaskRunBuiltInError,
+  TaskRunErrorCodes,
+  TaskRunExecution,
+  TaskRunExecutionPayload,
+  TaskRunExecutionResult,
+  ZodMessageHandler,
+  ZodMessageSender,
+  childToWorkerMessages,
+  correctErrorStackTrace,
+  workerToChildMessages,
+} from "@trigger.dev/core/v3";
+import { Evt } from "evt";
+import { ChildProcess, fork } from "node:child_process";
+import { safeDeleteFileSync } from "../utilities/fileSystem";
+
+class UnexpectedExitError extends Error {
+  constructor(public code: number) {
+    super(`Unexpected exit with code ${code}`);
+
+    this.name = "UnexpectedExitError";
+  }
+}
+
+class CleanupProcessError extends Error {
+  constructor() {
+    super("Cancelled");
+
+    this.name = "CleanupProcessError";
+  }
+}
+
+type BackgroundWorkerParams = {
+  env: Record<string, string>;
+  projectDir: string;
+  contentHash: string;
+  debugOtel?: boolean;
+};
+
+export class ProdBackgroundWorker {
+  private _initialized: boolean = false;
+  private _handler = new ZodMessageHandler({
+    schema: childToWorkerMessages,
+  });
+
+  public onTaskHeartbeat: Evt<string> = new Evt();
+
+  public onWaitForBatch: Evt<{ version?: "v1"; id: string; runs: string[] }> = new Evt();
+  public onWaitForDuration: Evt<{ version?: "v1"; ms: number }> = new Evt();
+  public onWaitForTask: Evt<{ version?: "v1"; id: string }> = new Evt();
+
+  private _onClose: Evt<void> = new Evt();
+
+  public tasks: Array<TaskMetadataWithFilePath> = [];
+
+  _taskRunProcesses: Map<string, TaskRunProcess> = new Map();
+
+  private _closed: boolean = false;
+
+  constructor(
+    public path: string,
+    private params: BackgroundWorkerParams
+  ) {}
+
+  close() {
+    if (this._closed) {
+      return;
+    }
+
+    this._closed = true;
+
+    this.onTaskHeartbeat.detach();
+
+    // We need to close all the task run processes
+    for (const taskRunProcess of this._taskRunProcesses.values()) {
+      taskRunProcess.cleanup(true);
+    }
+
+    // Delete worker files
+    this._onClose.post();
+
+    safeDeleteFileSync(this.path);
+    safeDeleteFileSync(`${this.path}.map`);
+  }
+
+  async initialize() {
+    if (this._initialized) {
+      throw new Error("Worker already initialized");
+    }
+
+    let resolved = false;
+
+    this.tasks = await new Promise<Array<TaskMetadataWithFilePath>>((resolve, reject) => {
+      const child = fork(this.path, {
+        stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
+        env: {
+          ...this.params.env,
+        },
+      });
+
+      // Set a timeout to kill the child process if it doesn't respond
+      const timeout = setTimeout(() => {
+        if (resolved) {
+          return;
+        }
+
+        resolved = true;
+        child.kill();
+        reject(new Error("Worker timed out"));
+      }, 1000);
+
+      child.on("message", async (msg: any) => {
+        const message = this._handler.parseMessage(msg);
+
+        if (message.type === "TASKS_READY" && !resolved) {
+          clearTimeout(timeout);
+          resolved = true;
+          resolve(message.payload.tasks);
+          child.kill();
+        }
+      });
+
+      child.stdout?.on("data", (data) => {
+        console.log(data.toString());
+      });
+
+      child.stderr?.on("data", (data) => {
+        console.error(data.toString());
+      });
+
+      child.on("exit", (code) => {
+        if (!resolved) {
+          clearTimeout(timeout);
+          resolved = true;
+          reject(new Error(`Worker exited with code ${code}`));
+        }
+      });
+    });
+
+    this._initialized = true;
+  }
+
+  getMetadata(workerId: string, version: string): CreateBackgroundWorkerResponse {
+    return {
+      contentHash: this.params.contentHash,
+      id: workerId,
+      version: version,
+    };
+  }
+
+  // We need to notify all the task run processes that a task run has completed,
+  // in case they are waiting for it through triggerAndWait
+  async taskRunCompletedNotification(
+    completion: TaskRunExecutionResult,
+    execution: TaskRunExecution
+  ) {
+    for (const taskRunProcess of this._taskRunProcesses.values()) {
+      taskRunProcess.taskRunCompletedNotification(completion, execution);
+    }
+  }
+
+  async #initializeTaskRunProcess(payload: ProdTaskRunExecutionPayload): Promise<TaskRunProcess> {
+    const metadata = this.getMetadata(
+      payload.execution.worker.id,
+      payload.execution.worker.version
+    );
+
+    if (!this._taskRunProcesses.has(payload.execution.run.id)) {
+      const taskRunProcess = new TaskRunProcess(
+        this.path,
+        {
+          ...this.params.env,
+          ...(payload.environment ?? {}),
+        },
+        metadata,
+        this.params
+      );
+
+      taskRunProcess.onExit.attach(() => {
+        this._taskRunProcesses.delete(payload.execution.run.id);
+      });
+
+      taskRunProcess.onTaskHeartbeat.attach((id) => {
+        this.onTaskHeartbeat.post(id);
+      });
+
+      taskRunProcess.onWaitForBatch.attach((message) => {
+        this.onWaitForBatch.post(message);
+      });
+
+      taskRunProcess.onWaitForDuration.attach((message) => {
+        this.onWaitForDuration.post(message);
+      });
+
+      taskRunProcess.onWaitForTask.attach((message) => {
+        this.onWaitForTask.post(message);
+      });
+
+      await taskRunProcess.initialize();
+
+      this._taskRunProcesses.set(payload.execution.run.id, taskRunProcess);
+    }
+
+    return this._taskRunProcesses.get(payload.execution.run.id) as TaskRunProcess;
+  }
+
+  // We need to fork the process before we can execute any tasks
+  async executeTaskRun(payload: ProdTaskRunExecutionPayload): Promise<TaskRunExecutionResult> {
+    try {
+      const taskRunProcess = await this.#initializeTaskRunProcess(payload);
+
+      const result = await taskRunProcess.executeTaskRun(payload);
+
+      // Kill the worker if the task was successful or if it's not going to be retried);
+      await taskRunProcess.cleanup(result.ok || result.retry === undefined);
+
+      if (result.ok) {
+        return result;
+      }
+
+      const error = result.error;
+
+      if (error.type === "BUILT_IN_ERROR") {
+        const mappedError = await this.#correctError(error, payload.execution);
+
+        return {
+          ...result,
+          error: mappedError,
+        };
+      }
+
+      return result;
+    } catch (e) {
+      if (e instanceof CleanupProcessError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: undefined,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.TASK_EXECUTION_ABORTED,
+          },
+        };
+      }
+
+      if (e instanceof UnexpectedExitError) {
+        return {
+          id: payload.execution.attempt.id,
+          ok: false,
+          retry: undefined,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE,
+          },
+        };
+      }
+
+      return {
+        id: payload.execution.attempt.id,
+        ok: false,
+        retry: undefined,
+        error: {
+          type: "INTERNAL_ERROR",
+          code: TaskRunErrorCodes.TASK_EXECUTION_FAILED,
+        },
+      };
+    }
+  }
+
+  async #correctError(
+    error: TaskRunBuiltInError,
+    execution: TaskRunExecution
+  ): Promise<TaskRunBuiltInError> {
+    return {
+      ...error,
+      stackTrace: correctErrorStackTrace(error.stackTrace, this.params.projectDir),
+    };
+  }
+}
+
+class TaskRunProcess {
+  private _handler = new ZodMessageHandler({
+    schema: childToWorkerMessages,
+  });
+  private _sender: ZodMessageSender<typeof workerToChildMessages>;
+  private _child: ChildProcess | undefined;
+  private _attemptPromises: Map<
+    string,
+    { resolver: (value: TaskRunExecutionResult) => void; rejecter: (err?: any) => void }
+  > = new Map();
+  private _attemptStatuses: Map<string, "PENDING" | "REJECTED" | "RESOLVED"> = new Map();
+  private _currentExecution: TaskRunExecution | undefined;
+  private _isBeingKilled: boolean = false;
+
+  public onTaskHeartbeat: Evt<string> = new Evt();
+  public onExit: Evt<number> = new Evt();
+
+  public onWaitForBatch: Evt<{ version?: "v1"; id: string; runs: string[] }> = new Evt();
+  public onWaitForDuration: Evt<{ version?: "v1"; ms: number }> = new Evt();
+  public onWaitForTask: Evt<{ version?: "v1"; id: string }> = new Evt();
+
+  constructor(
+    private path: string,
+    private env: NodeJS.ProcessEnv,
+    private metadata: BackgroundWorkerProperties,
+    private worker: BackgroundWorkerParams
+  ) {
+    this._sender = new ZodMessageSender({
+      schema: workerToChildMessages,
+      sender: async (message) => {
+        if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
+          this._child?.send?.(message);
+        }
+      },
+    });
+  }
+
+  async initialize() {
+    this._child = fork(this.path, {
+      stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
+      env: {
+        ...this.env,
+        OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
+          [SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectDir,
+        }),
+        ...(this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}),
+      },
+    });
+
+    this._child.on("message", this.#handleMessage.bind(this));
+    this._child.on("exit", this.#handleExit.bind(this));
+    this._child.stdout?.on("data", this.#handleLog.bind(this));
+    this._child.stderr?.on("data", this.#handleStdErr.bind(this));
+  }
+
+  async cleanup(kill: boolean = false) {
+    if (kill && this._isBeingKilled) {
+      return;
+    }
+
+    await this._sender.send("CLEANUP", {
+      flush: true,
+      kill,
+    });
+
+    this._isBeingKilled = kill;
+  }
+
+  async executeTaskRun(payload: TaskRunExecutionPayload): Promise<TaskRunExecutionResult> {
+    let resolver: (value: TaskRunExecutionResult) => void;
+    let rejecter: (err?: any) => void;
+
+    const promise = new Promise<TaskRunExecutionResult>((resolve, reject) => {
+      resolver = resolve;
+      rejecter = reject;
+    });
+
+    this._attemptStatuses.set(payload.execution.attempt.id, "PENDING");
+
+    // @ts-expect-error - We know that the resolver and rejecter are defined
+    this._attemptPromises.set(payload.execution.attempt.id, { resolver, rejecter });
+
+    const { execution, traceContext } = payload;
+
+    this._currentExecution = execution;
+
+    await this._sender.send("EXECUTE_TASK_RUN", {
+      execution,
+      traceContext,
+      metadata: this.metadata,
+    });
+
+    const result = await promise;
+
+    this._currentExecution = undefined;
+
+    return result;
+  }
+
+  taskRunCompletedNotification(completion: TaskRunExecutionResult, execution: TaskRunExecution) {
+    if (!completion.ok && typeof completion.retry !== "undefined") {
+      return;
+    }
+
+    this._sender.send("TASK_RUN_COMPLETED_NOTIFICATION", {
+      completion,
+      execution,
+    });
+  }
+
+  async #handleMessage(msg: any) {
+    const message = this._handler.parseMessage(msg);
+
+    switch (message.type) {
+      case "TASK_RUN_COMPLETED": {
+        const { result, execution } = message.payload;
+
+        const promiseStatus = this._attemptStatuses.get(execution.attempt.id);
+
+        if (promiseStatus !== "PENDING") {
+          return;
+        }
+
+        this._attemptStatuses.set(execution.attempt.id, "RESOLVED");
+
+        const attemptPromise = this._attemptPromises.get(execution.attempt.id);
+
+        if (!attemptPromise) {
+          return;
+        }
+
+        const { resolver } = attemptPromise;
+
+        resolver(result);
+
+        break;
+      }
+      case "READY_TO_DISPOSE": {
+        this.#kill();
+
+        break;
+      }
+      case "TASK_HEARTBEAT": {
+        this.onTaskHeartbeat.post(message.payload.id);
+
+        break;
+      }
+      case "TASKS_READY": {
+        break;
+      }
+      case "WAIT_FOR_BATCH": {
+        this.onWaitForBatch.post(message.payload);
+
+        break;
+      }
+      case "WAIT_FOR_DURATION": {
+        this.onWaitForDuration.post(message.payload);
+
+        break;
+      }
+      case "WAIT_FOR_TASK": {
+        this.onWaitForTask.post(message.payload);
+
+        break;
+      }
+    }
+  }
+
+  async #handleExit(code: number) {
+    // Go through all the attempts currently pending and reject them
+    for (const [id, status] of this._attemptStatuses.entries()) {
+      if (status === "PENDING") {
+        this._attemptStatuses.set(id, "REJECTED");
+
+        const attemptPromise = this._attemptPromises.get(id);
+
+        if (!attemptPromise) {
+          continue;
+        }
+
+        const { rejecter } = attemptPromise;
+
+        if (this._isBeingKilled) {
+          rejecter(new CleanupProcessError());
+        } else {
+          rejecter(new UnexpectedExitError(code));
+        }
+      }
+    }
+
+    this.onExit.post(code);
+  }
+
+  #handleLog(data: Buffer) {
+    if (!this._currentExecution) {
+      return;
+    }
+
+    console.log(
+      `[${this.metadata.version}][${this._currentExecution.run.id}.${
+        this._currentExecution.attempt.number
+      }] ${data.toString()}`
+    );
+  }
+
+  #handleStdErr(data: Buffer) {
+    if (this._isBeingKilled) {
+      return;
+    }
+
+    if (!this._currentExecution) {
+      console.error(`[${this.metadata.version}] ${data.toString()}`);
+
+      return;
+    }
+
+    console.error(
+      `[${this.metadata.version}][${this._currentExecution.run.id}.${
+        this._currentExecution.attempt.number
+      }] ${data.toString()}`
+    );
+  }
+
+  #kill() {
+    if (this._child && !this._child.killed) {
+      this._child?.kill();
+    }
+  }
+}

packages/cli-v3/src/types.ts

@@ -8,3 +8,10 @@ export type TaskMetadataWithFunctions = TaskMetadataWithFilePath & {
     middleware?: (params: any) => Promise<void>;
   };
 };
+
+export type TaskFile = {
+  triggerDir: string;
+  filePath: string;
+  importPath: string;
+  importName: string;
+};

packages/cli-v3/src/utilities/configFiles.ts

@@ -1,12 +1,17 @@
 import { mkdirSync, writeFileSync } from "node:fs";
-import path from "node:path";
+import path, { dirname } from "node:path";
 import xdgAppPaths from "xdg-app-paths";
 import { z } from "zod";
 import { readJSONFileSync } from "./fileSystem.js";
 import { logger } from "./logger.js";
+import { findUp } from "find-up";
+import { CLOUD_API_URL, CONFIG_FILES } from "../consts.js";
+import { pathToFileURL } from "node:url";
+import { findTriggerDirectories, resolveTriggerDirectories } from "./taskFiles.js";
+import { Config, ResolvedConfig } from "@trigger.dev/core/v3";
 
 function getGlobalConfigFolderPath() {
-  const configDir = xdgAppPaths(".trigger").config();
+  const configDir = xdgAppPaths("trigger").config();
 
   return configDir;
 }
@@ -20,7 +25,7 @@ export const UserAuthConfigSchema = z.object({
 export type UserAuthConfig = z.infer<typeof UserAuthConfigSchema>;
 
 function getAuthConfigFilePath() {
-  return path.join(getGlobalConfigFolderPath(), "config", "default.json");
+  return path.join(getGlobalConfigFolderPath(), "default.json");
 }
 
 export function writeAuthConfigFile(config: UserAuthConfig) {
@@ -45,3 +50,53 @@ export function readAuthConfigFile(): UserAuthConfig | undefined {
     return undefined;
   }
 }
+
+export async function getConfigPath(dir: string): Promise<string> {
+  const path = await findUp(CONFIG_FILES, { cwd: dir });
+
+  if (!path) {
+    throw new Error("No config file found.");
+  }
+
+  return path;
+}
+
+export async function readConfig(path: string): Promise<ResolvedConfig> {
+  try {
+    // import the config file
+    const userConfigModule = await import(`${pathToFileURL(path).href}?_ts=${Date.now()}`);
+    const rawConfig = await normalizeConfig(userConfigModule ? userConfigModule.default : {});
+    const config = Config.parse(rawConfig);
+
+    return resolveConfig(path, config);
+  } catch (error) {
+    console.error(`Failed to load config file at ${path}`);
+    throw error;
+  }
+}
+
+export async function resolveConfig(path: string, config: Config): Promise<ResolvedConfig> {
+  if (!config.triggerDirectories) {
+    config.triggerDirectories = await findTriggerDirectories(path);
+  }
+
+  config.triggerDirectories = resolveTriggerDirectories(config.triggerDirectories);
+
+  if (!config.triggerUrl) {
+    config.triggerUrl = CLOUD_API_URL;
+  }
+
+  if (!config.projectDir) {
+    config.projectDir = dirname(path);
+  }
+
+  return config as ResolvedConfig;
+}
+
+export async function normalizeConfig(config: any): Promise<any> {
+  if (typeof config === "function") {
+    config = config();
+  }
+
+  return await config;
+}

packages/cli-v3/src/utilities/session.ts

@@ -1,4 +1,4 @@
-import { ApiClient } from "../apiClient.js";
+import { CliApiClient } from "../apiClient.js";
 import { readAuthConfigFile } from "./configFiles.js";
 
 export async function isLoggedIn() {
@@ -8,7 +8,7 @@ export async function isLoggedIn() {
     return { ok: false as const, error: "You must login first" };
   }
 
-  const apiClient = new ApiClient(config.apiUrl, config.accessToken);
+  const apiClient = new CliApiClient(config.apiUrl, config.accessToken);
   const userData = await apiClient.whoAmI();
 
   if (!userData.success) {

packages/cli-v3/src/utilities/taskFiles.ts

@@ -0,0 +1,70 @@
+import fs from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { TaskFile } from "../types";
+import { ResolvedConfig } from "@trigger.dev/core/v3";
+
+export function createTaskFileImports(taskFiles: TaskFile[]) {
+  return taskFiles
+    .map(
+      (taskFile) =>
+        `import * as ${taskFile.importName} from "./${taskFile.importPath}"; TaskFileImports["${
+          taskFile.importName
+        }"] = ${taskFile.importName}; TaskFiles["${taskFile.importName}"] = ${JSON.stringify(
+          taskFile
+        )};`
+    )
+    .join("\n");
+}
+
+// Find all the top-level .js or .ts files in the trigger directories
+export async function gatherTaskFiles(config: ResolvedConfig): Promise<Array<TaskFile>> {
+  const taskFiles: Array<TaskFile> = [];
+
+  for (const triggerDir of config.triggerDirectories) {
+    const files = await fs.promises.readdir(triggerDir, { withFileTypes: true });
+    for (const file of files) {
+      if (!file.isFile()) continue;
+      if (!file.name.endsWith(".js") && !file.name.endsWith(".ts")) continue;
+
+      const fullPath = join(triggerDir, file.name);
+
+      const filePath = relative(config.projectDir, fullPath);
+      const importPath = filePath.replace(/\.(js|ts)$/, "");
+      const importName = importPath.replace(/\//g, "_");
+
+      taskFiles.push({ triggerDir, importPath, importName, filePath });
+    }
+  }
+
+  return taskFiles;
+}
+
+export function resolveTriggerDirectories(dirs: string[]): string[] {
+  return dirs.map((dir) => resolve(dir));
+}
+
+const IGNORED_DIRS = ["node_modules", ".git", "dist", "build"];
+
+export async function findTriggerDirectories(filePath: string): Promise<string[]> {
+  const dirPath = dirname(filePath);
+  return getTriggerDirectories(dirPath);
+}
+
+async function getTriggerDirectories(dirPath: string): Promise<string[]> {
+  const entries = await fs.promises.readdir(dirPath, { withFileTypes: true });
+  const triggerDirectories: string[] = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || IGNORED_DIRS.includes(entry.name)) continue;
+
+    const fullPath = join(dirPath, entry.name);
+
+    if (entry.name === "trigger") {
+      triggerDirectories.push(fullPath);
+    }
+
+    triggerDirectories.push(...(await getTriggerDirectories(fullPath)));
+  }
+
+  return triggerDirectories;
+}

packages/cli-v3/tsconfig.json

@@ -25,7 +25,9 @@
     "jsx": "react",
     "paths": {
       "@trigger.dev/core/v3": ["../core/src/v3"],
-      "@trigger.dev/core/v3/*": ["../core/src/v3/*"]
+      "@trigger.dev/core/v3/*": ["../core/src/v3/*"],
+      "@trigger.dev/core-apps": ["../core-apps/src"],
+      "@trigger.dev/core-apps/*": ["../core-apps/src/*"]
     }
   },
   "exclude": ["node_modules"]

packages/cli-v3/tsup.prod-facade.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from "tsup";
+
+export default defineConfig({
+  clean: false,
+  dts: true,
+  tsconfig: "tsconfig.json",
+  splitting: false,
+  entry: ["src/prod-facade.ts"],
+  format: ["esm"],
+  minify: false,
+  metafile: false,
+  sourcemap: true,
+  target: "esnext",
+  outDir: "dist",
+  noExternal: ["zod", /traceloop/, /opentelemetry/],
+});

packages/cli/package.json

@@ -39,7 +39,7 @@
     "@types/gradient-string": "^1.1.2",
     "@types/inquirer": "^9.0.3",
     "@types/mock-fs": "^4.13.1",
-    "@types/node": "16",
+    "@types/node": "^18",
     "@types/node-fetch": "^2.6.2",
     "@types/ws": "^8.5.3",
     "rimraf": "^3.0.2",

packages/core-apps/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@trigger.dev/core-apps",
+  "description": "Backend core code used across apps",
+  "private": true,
+  "license": "MIT",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "module": "./dist/index.mjs",
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "require": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "clean": "rimraf dist",
+    "build": "npm run clean && npm run build:tsup",
+    "build:tsup": "tsup --dts-resolve",
+    "typecheck": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@trigger.dev/tsconfig": "workspace:*",
+    "@trigger.dev/tsup": "workspace:*",
+    "@types/node": "18",
+    "rimraf": "^3.0.2",
+    "tsup": "^8.0.1",
+    "typescript": "^5.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

packages/core-apps/src/http.ts

@@ -0,0 +1,39 @@
+import { IncomingMessage, RequestListener } from "http";
+
+export const getTextBody = (req: IncomingMessage) =>
+  new Promise<string>((resolve) => {
+    let body = "";
+    req.on("readable", () => {
+      const chunk = req.read();
+      if (chunk) {
+        body += chunk;
+      }
+    });
+    req.on("end", () => {
+      resolve(body);
+    });
+  });
+
+export class HttpReply {
+  constructor(private response: Parameters<RequestListener>[1]) {}
+
+  empty(status?: number) {
+    return this.response.writeHead(status ?? 200).end();
+  }
+
+  text(text: string, status?: number, contentType?: string) {
+    return this.response
+      .writeHead(status ?? 200, { "Content-Type": contentType || "text/plain" })
+      .end(text.endsWith("\n") ? text : `${text}\n`);
+  }
+}
+
+function getRandomInteger(min: number, max: number) {
+  const intMin = Math.ceil(min);
+  const intMax = Math.floor(max);
+  return Math.floor(Math.random() * (intMax - intMin + 1)) + intMin;
+}
+
+export function getRandomPortNumber() {
+  return getRandomInteger(8000, 9999);
+}

packages/core-apps/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./http";
+export * from "./logger";

packages/core-apps/src/logger.ts

@@ -0,0 +1,35 @@
+export class SimpleLogger {
+  #debugEnabled = ["1", "true"].includes(process.env.DEBUG ?? "");
+
+  constructor(private prefix?: string) {}
+
+  log<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
+    console.log(...this.#getPrefixedArgs(arg0, ...argN));
+
+    return arg0;
+  }
+
+  debug<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
+    if (!this.#debugEnabled) {
+      return arg0;
+    }
+
+    console.debug(...this.#getPrefixedArgs("DEBUG", arg0, ...argN));
+
+    return arg0;
+  }
+
+  error<TFirstArg>(arg0: TFirstArg, ...argN: any[]) {
+    console.error(...this.#getPrefixedArgs(arg0, ...argN));
+
+    return arg0;
+  }
+
+  #getPrefixedArgs(...args: any[]) {
+    if (!this.prefix) {
+      return args;
+    }
+
+    return [this.prefix, ...args];
+  }
+}

packages/core-apps/tsconfig.build.json

@@ -0,0 +1,11 @@
+{
+  "extends": "@trigger.dev/tsconfig/node18.json",
+  "include": ["src/globals.d.ts", "./src/**/*.ts", "tsup.config.ts"],
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "declaration": false,
+    "declarationMap": false
+  },
+  "exclude": ["node_modules"]
+}

packages/core-apps/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "extends": "@trigger.dev/tsconfig/node18.json",
+  "include": ["src/globals.d.ts", "./src/**/*.ts", "tsup.config.ts", "./test/**/*.ts"],
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "declaration": false,
+    "declarationMap": false,
+    "lib": ["DOM", "DOM.Iterable"],
+    "paths": {
+      "@trigger.dev/core/v3": ["../core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../core/src/v3/*"],
+      "@trigger.dev/tsup/*": ["../../config-packages/tsup/src/*"],
+      "@trigger.dev/tsup": ["../../config-packages/tsup/src/index"]
+    }
+  },
+  "exclude": ["node_modules"]
+}

packages/core-apps/tsup.config.ts

@@ -0,0 +1,6 @@
+import { packageOptions, defineConfig } from "@trigger.dev/tsup";
+
+export default defineConfig({
+  ...packageOptions,
+  config: "tsconfig.build.json",
+});

packages/core/package.json

@@ -74,6 +74,7 @@
     "@opentelemetry/sdk-trace-node": "^1.21.0",
     "@opentelemetry/semantic-conventions": "^1.21.0",
     "humanize-duration": "^3.27.3",
+    "socket.io": "^4.7.4",
     "ulidx": "^2.2.1",
     "zod": "3.22.3",
     "zod-error": "1.5.0"
@@ -83,7 +84,7 @@
     "@trigger.dev/tsup": "workspace:*",
     "@types/humanize-duration": "^3.27.1",
     "@types/jest": "^29.5.3",
-    "@types/node": "16",
+    "@types/node": "^18",
     "jest": "^29.6.2",
     "rimraf": "^3.0.2",
     "ts-jest": "^29.1.1",

packages/core/src/v3/apiClient/index.ts

@@ -1,15 +1,14 @@
 import { context, propagation } from "@opentelemetry/api";
 import { zodfetch } from "../../zodfetch";
-import {
-  BatchTriggerTaskRequestBody,
-  BatchTriggerTaskResponse,
-  GetBatchResponseBody,
-  TriggerTaskRequestBody,
-  TriggerTaskResponse,
-} from "../schemas/api";
 import { taskContextManager } from "../tasks/taskContextManager";
 import { SafeAsyncLocalStorage } from "../utils/safeAsyncLocalStorage";
 import { getEnvVar } from "../utils/getEnv";
+import {
+  TriggerTaskRequestBody,
+  TriggerTaskResponse,
+  BatchTriggerTaskRequestBody,
+  BatchTriggerTaskResponse,
+} from "../schemas";
 
 export type TriggerOptions = {
   spanParentAsLink?: boolean;
@@ -19,10 +18,14 @@ export type TriggerOptions = {
  * Trigger.dev v3 API client
  */
 export class ApiClient {
+  private readonly baseUrl: string;
+
   constructor(
-    private readonly baseUrl: string,
+    baseUrl: string,
     private readonly accessToken: string
-  ) {}
+  ) {
+    this.baseUrl = baseUrl.replace(/\/$/, "");
+  }
 
   triggerTask(taskId: string, body: TriggerTaskRequestBody, options?: TriggerOptions) {
     return zodfetch(TriggerTaskResponse, `${this.baseUrl}/api/v1/tasks/${taskId}/trigger`, {

packages/core/src/v3/index.ts

@@ -3,9 +3,11 @@ import { BatchTriggerTaskRequestBody, TriggerTaskRequestBody } from "./schemas";
 export * from "./schemas";
 export * from "./apiClient";
 export * from "./zodMessageHandler";
+export * from "./zodNamespace";
 export * from "./errors";
 export * from "./runtime-api";
 export * from "./logger-api";
+export * from "./types";
 export { SemanticInternalAttributes } from "./semanticInternalAttributes";
 export { iconStringForSeverity } from "./icons";
 export {
@@ -28,6 +30,7 @@ export function parseBatchTriggerTaskRequestBody(body: unknown) {
 export { taskContextManager, TaskContextSpanProcessor } from "./tasks/taskContextManager";
 export type { RuntimeManager } from "./runtime/manager";
 export { DevRuntimeManager } from "./runtime/devRuntimeManager";
+export { ProdRuntimeManager } from "./runtime/prodRuntimeManager";
 export { TriggerTracer } from "./tracer";
 
 export type { TaskLogger } from "./logger/taskLogger";

packages/core/src/v3/runtime/devRuntimeManager.ts

@@ -82,9 +82,16 @@ export class DevRuntimeManager implements RuntimeManager {
   resumeTask(completion: TaskRunExecutionResult, execution: TaskRunExecution): void {
     const wait = this._taskWaits.get(execution.run.id);
 
-    if (wait) {
-      wait.resolve(completion);
-      this._taskWaits.delete(execution.run.id);
+    if (!wait) {
+      return;
     }
+
+    if (completion.ok) {
+      wait.resolve(completion);
+    } else {
+      wait.reject(completion);
+    }
+
+    this._taskWaits.delete(execution.run.id);
   }
 }

packages/core/src/v3/runtime/prodRuntimeManager.ts

@@ -0,0 +1,114 @@
+import {
+  BatchTaskRunExecutionResult,
+  TaskMetadataWithFilePath,
+  TaskRunContext,
+  TaskRunExecution,
+  TaskRunExecutionResult,
+  childToWorkerMessages,
+} from "../schemas";
+import { ZodMessageSender } from "../zodMessageHandler";
+import { RuntimeManager } from "./manager";
+
+export class ProdRuntimeManager implements RuntimeManager {
+  _taskWaits: Map<
+    string,
+    { resolve: (value: TaskRunExecutionResult) => void; reject: (err?: any) => void }
+  > = new Map();
+
+  _batchWaits: Map<
+    string,
+    { resolve: (value: BatchTaskRunExecutionResult) => void; reject: (err?: any) => void }
+  > = new Map();
+
+  _tasks: Map<string, TaskMetadataWithFilePath> = new Map();
+
+  constructor(private sender: ZodMessageSender<typeof childToWorkerMessages>) {}
+
+  disable(): void {
+    // do nothing
+  }
+
+  registerTasks(tasks: TaskMetadataWithFilePath[]): void {
+    for (const task of tasks) {
+      this._tasks.set(task.id, task);
+    }
+  }
+
+  getTaskMetadata(id: string): TaskMetadataWithFilePath | undefined {
+    return this._tasks.get(id);
+  }
+
+  async waitForDuration(ms: number): Promise<void> {
+    if (ms > 30_000) {
+      // TODO: sender with ack support
+      await this.sender.send("WAIT_FOR_DURATION", { ms });
+      // TODO: resolve after resume signal instead
+    }
+
+    return new Promise((resolve) => {
+      setTimeout(resolve, ms);
+    });
+  }
+
+  async waitUntil(date: Date): Promise<void> {
+    return this.waitForDuration(date.getTime() - Date.now());
+  }
+
+  async waitForTask(params: { id: string; ctx: TaskRunContext }): Promise<TaskRunExecutionResult> {
+    const promise = new Promise<TaskRunExecutionResult>((resolve, reject) => {
+      this._taskWaits.set(params.id, { resolve, reject });
+    });
+
+    await this.sender.send("WAIT_FOR_TASK", {
+      id: params.id,
+    });
+
+    return await promise;
+  }
+
+  async waitForBatch(params: {
+    id: string;
+    runs: string[];
+    ctx: TaskRunContext;
+  }): Promise<BatchTaskRunExecutionResult> {
+    if (!params.runs.length) {
+      return Promise.resolve({ id: params.id, items: [] });
+    }
+
+    const promise = Promise.all(
+      params.runs.map((runId) => {
+        return new Promise<TaskRunExecutionResult>((resolve, reject) => {
+          this._taskWaits.set(runId, { resolve, reject });
+        });
+      })
+    );
+
+    await this.sender.send("WAIT_FOR_BATCH", {
+      id: params.id,
+      runs: params.runs,
+    });
+
+    const results = await promise;
+
+    return {
+      id: params.id,
+      items: results,
+    };
+  }
+
+  resumeTask(completion: TaskRunExecutionResult, execution: TaskRunExecution): void {
+    const wait = this._taskWaits.get(execution.run.id);
+
+    if (!wait) {
+      return;
+    }
+
+    if (completion.ok) {
+      wait.resolve(completion);
+    } else {
+      wait.reject(completion);
+    }
+
+    this._taskWaits.delete(execution.run.id);
+  }
+}

packages/core/src/v3/schemas/api.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { BackgroundWorkerMetadata } from "./resources";
+import { BackgroundWorkerMetadata, ImageDetailsMetadata } from "./resources";
 import { QueueOptions } from "./messages";
 
 export const WhoAmIResponseSchema = z.object({
@@ -88,3 +88,16 @@ export const GetEnvironmentVariablesResponseBody = z.object({
 export type GetEnvironmentVariablesResponseBody = z.infer<
   typeof GetEnvironmentVariablesResponseBody
 >;
+
+export const CreateImageDetailsRequestBody = z.object({
+  metadata: ImageDetailsMetadata,
+});
+
+export type CreateImageDetailsRequestBody = z.infer<typeof CreateImageDetailsRequestBody>;
+
+export const CreateImageDetailsResponse = z.object({
+  id: z.string(),
+  contentHash: z.string(),
+});
+
+export type CreateImageDetailsResponse = z.infer<typeof CreateImageDetailsResponse>;

packages/core/src/v3/schemas/index.ts

@@ -3,6 +3,7 @@ export * from "./api";
 export * from "./resources";
 export * from "./common";
 export * from "./messages";
+export * from "./schemas";
 export * from "./style";
 export * from "./fetch";
 export * from "./eventFilter";

packages/core/src/v3/schemas/messages.ts

@@ -9,11 +9,35 @@ export const TaskRunExecutionPayload = z.object({
 
 export type TaskRunExecutionPayload = z.infer<typeof TaskRunExecutionPayload>;
 
+export const ProdTaskRunExecution = TaskRunExecution.extend({
+  worker: z.object({
+    id: z.string(),
+    contentHash: z.string(),
+    version: z.string(),
+  }),
+});
+
+export type ProdTaskRunExecution = z.infer<typeof ProdTaskRunExecution>;
+
+export const ProdTaskRunExecutionPayload = z.object({
+  execution: ProdTaskRunExecution,
+  traceContext: z.record(z.unknown()),
+  environment: z.record(z.string()).optional(),
+});
+
+export type ProdTaskRunExecutionPayload = z.infer<typeof ProdTaskRunExecutionPayload>;
+
 export const BackgroundWorkerServerMessages = z.discriminatedUnion("type", [
   z.object({
     type: z.literal("EXECUTE_RUNS"),
     payloads: z.array(TaskRunExecutionPayload),
   }),
+  z.object({
+    type: z.literal("SCHEDULE_ATTEMPT"),
+    id: z.string(),
+    image: z.string(),
+    envId: z.string(),
+  }),
 ]);
 
 export type BackgroundWorkerServerMessages = z.infer<typeof BackgroundWorkerServerMessages>;
@@ -178,4 +202,17 @@ export const childToWorkerMessages = {
     id: z.string(),
   }),
   READY_TO_DISPOSE: z.undefined(),
+  WAIT_FOR_DURATION: z.object({
+    version: z.literal("v1").default("v1"),
+    ms: z.number(),
+  }),
+  WAIT_FOR_TASK: z.object({
+    version: z.literal("v1").default("v1"),
+    id: z.string(),
+  }),
+  WAIT_FOR_BATCH: z.object({
+    version: z.literal("v1").default("v1"),
+    id: z.string(),
+    runs: z.string().array(),
+  }),
 };

packages/core/src/v3/schemas/resources.ts

@@ -19,3 +19,10 @@ export const BackgroundWorkerMetadata = z.object({
 });
 
 export type BackgroundWorkerMetadata = z.infer<typeof BackgroundWorkerMetadata>;
+
+export const ImageDetailsMetadata = z.object({
+  contentHash: z.string(),
+  imageTag: z.string(),
+});
+
+export type ImageDetailsMetadata = z.infer<typeof ImageDetailsMetadata>;

packages/core/src/v3/schemas/schemas.ts

@@ -0,0 +1,117 @@
+import { z } from "zod";
+import { RequireKeys } from "../types";
+import { TaskRunExecution, TaskRunExecutionResult } from "./common";
+import { ProdTaskRunExecution } from "./messages";
+import { TaskResource } from "./resources";
+
+export const Config = z.object({
+  project: z.string(),
+  triggerDirectories: z.string().array().optional(),
+  triggerUrl: z.string().optional(),
+  projectDir: z.string().optional(),
+});
+
+export type Config = z.infer<typeof Config>;
+export type ResolvedConfig = RequireKeys<
+  Config,
+  "triggerDirectories" | "triggerUrl" | "projectDir"
+>;
+
+export const Machine = z.object({
+  cpu: z.string().default("1").optional(),
+  memory: z.string().default("500Mi").optional(),
+});
+
+export type Machine = z.infer<typeof Machine>;
+
+export const ProviderToPlatformMessages = {
+  LOG: z.object({
+    version: z.literal("v1").default("v1"),
+    data: z.string(),
+  }),
+};
+
+export const PlatformToProviderMessages = {
+  HEALTH: z.object({
+    // TODO: callback: (ack: { status: "ok" }) => void
+    version: z.literal("v1").default("v1"),
+  }),
+  INDEX: z.object({
+    version: z.literal("v1").default("v1"),
+    imageTag: z.string(),
+    contentHash: z.string(),
+    envId: z.string(),
+  }),
+  INVOKE: z.object({
+    version: z.literal("v1").default("v1"),
+    name: z.string(),
+    machine: Machine,
+  }),
+  RESTORE: z.object({
+    version: z.literal("v1").default("v1"),
+    id: z.string(),
+    attemptId: z.string(),
+    type: z.enum(["DOCKER", "KUBERNETES"]),
+    location: z.string(),
+    reason: z.string().optional(),
+  }),
+  DELETE: z.object({
+    // TODO: callback: (ack: { message: string }) => void
+    version: z.literal("v1").default("v1"),
+    name: z.string(),
+  }),
+  GET: z.object({
+    version: z.literal("v1").default("v1"),
+    name: z.string(),
+  }),
+};
+
+export const CoordinatorToPlatformMessages = {
+  LOG: z.object({
+    version: z.literal("v1").default("v1"),
+    metadata: z.any(),
+    text: z.string(),
+  }),
+  CREATE_WORKER: z.object({
+    version: z.literal("v1").default("v1"),
+    projectRef: z.string(),
+    envId: z.string(),
+    metadata: z.object({
+      cliPackageVersion: z.string(),
+      contentHash: z.string(),
+      packageVersion: z.string(),
+      tasks: TaskResource.array(),
+    }),
+  }),
+  // TODO: callback: (ack: { success: false } | { success: true; payload: ProdTaskRunExecutionPayload }) => void
+  READY_FOR_EXECUTION: z.object({
+    version: z.literal("v1").default("v1"),
+    attemptId: z.string(),
+  }),
+  TASK_RUN_COMPLETED: z.object({
+    version: z.literal("v1").default("v1"),
+    execution: ProdTaskRunExecution,
+    completion: TaskRunExecutionResult,
+  }),
+  TASK_HEARTBEAT: z.object({
+    version: z.literal("v1").default("v1"),
+    attemptFriendlyId: z.string(),
+  }),
+  CHECKPOINT_CREATED: z.object({
+    version: z.literal("v1").default("v1"),
+    attemptId: z.string(),
+    docker: z.boolean(),
+    location: z.string(),
+    reason: z.string().optional(),
+  }),
+};
+
+export const PlatformToCoordinatorMessages = {
+  RESUME: z.object({
+    version: z.literal("v1").default("v1"),
+    attemptId: z.string(),
+    image: z.string(),
+    completions: TaskRunExecutionResult.array(),
+    executions: TaskRunExecution.array(),
+  }),
+};

packages/core/src/v3/types/index.ts

@@ -0,0 +1,2 @@
+export * from "./socketIo";
+export * from "./utils";

packages/core/src/v3/types/socketIo.ts

@@ -0,0 +1,129 @@
+import {
+  TaskResource,
+  ProdTaskRunExecutionPayload,
+  TaskRunExecutionResult,
+  TaskRunExecution,
+  ProdTaskRunExecution,
+} from "../schemas";
+
+export type VersionedMessage<TMessage> = { version: "v1" } & TMessage;
+
+// provider <--> platform
+export interface ProviderClientToServerEvents {
+  LOG: (message: VersionedMessage<{ data: string }>) => void;
+}
+
+export interface ProviderServerToClientEvents {
+  HEALTH: (message: VersionedMessage<{}>, callback: (ack: { status: "ok" }) => void) => void;
+  INDEX: (
+    message: VersionedMessage<{ imageTag: string; contentHash: string; envId: string }>
+  ) => void;
+  RESTORE: (
+    message: VersionedMessage<{
+      id: string;
+      attemptId: string;
+      type: "DOCKER" | "KUBERNETES";
+      location: string;
+      reason?: string;
+    }>
+  ) => void;
+  DELETE: (
+    message: VersionedMessage<{ name: string }>,
+    callback: (ack: { message: string }) => void
+  ) => void;
+  GET: (message: VersionedMessage<{ name: string }>) => void;
+}
+
+// coordinator <--> prod worker
+export interface ProdWorkerToCoordinatorEvents {
+  LOG: (message: VersionedMessage<{ text: string }>, callback: () => {}) => void;
+  INDEX_TASKS: (
+    message: VersionedMessage<{
+      tasks: TaskResource[];
+      packageVersion: string;
+    }>,
+    callback: (params: { success: boolean }) => {}
+  ) => void;
+  READY_FOR_EXECUTION: (message: VersionedMessage<{ attemptId: string }>) => void;
+  TASK_HEARTBEAT: (message: VersionedMessage<{ attemptFriendlyId: string }>) => void;
+  WAIT_FOR_BATCH: (message: VersionedMessage<{ id: string; runs: string[] }>) => void;
+  WAIT_FOR_DURATION: (
+    message: VersionedMessage<{ ms: number }>,
+    callback: (ack: { success: boolean }) => void
+  ) => void;
+  WAIT_FOR_TASK: (message: VersionedMessage<{ id: string }>) => void;
+}
+
+export interface CoordinatorToProdWorkerEvents {
+  RESUME: (
+    message: VersionedMessage<{
+      attemptId: string;
+      image: string;
+      completions: TaskRunExecutionResult[];
+      executions: TaskRunExecution[];
+    }>
+  ) => void;
+  EXECUTE_TASK_RUN: (
+    message: VersionedMessage<{ payload: ProdTaskRunExecutionPayload }>,
+    callback: (ack: { completion: TaskRunExecutionResult }) => void
+  ) => void;
+}
+
+export interface ProdWorkerSocketData {
+  cliPackageVersion: string;
+  contentHash: string;
+  projectRef: string;
+  envId: string;
+  attemptId: string;
+  podName: string;
+}
+
+// coordinator <--> platform
+export interface CoordinatorToPlatformEvents {
+  LOG: (message: VersionedMessage<{ metadata: any; text: string }>) => void;
+  CREATE_WORKER: (
+    message: VersionedMessage<{
+      projectRef: string;
+      envId: string;
+      metadata: {
+        cliPackageVersion: string;
+        contentHash: string;
+        packageVersion: string;
+        tasks: TaskResource[];
+      };
+    }>,
+    callback: (ack: { success: boolean }) => void
+  ) => void;
+  READY_FOR_EXECUTION: (
+    message: VersionedMessage<{ attemptId: string }>,
+    callback: (
+      ack: { success: false } | { success: true; payload: ProdTaskRunExecutionPayload }
+    ) => void
+  ) => void;
+  TASK_RUN_COMPLETED: (
+    message: VersionedMessage<{
+      execution: ProdTaskRunExecution;
+      completion: TaskRunExecutionResult;
+    }>
+  ) => void;
+  TASK_HEARTBEAT: (message: VersionedMessage<{ attemptFriendlyId: string }>) => void;
+  CHECKPOINT_CREATED: (
+    message: VersionedMessage<{
+      attemptId: string;
+      docker: boolean;
+      location: string;
+      reason?: string;
+    }>
+  ) => void;
+}
+
+export interface PlatformToCoordinatorEvents {
+  RESUME: (
+    message: VersionedMessage<{
+      attemptId: string;
+      image: string;
+      completions: TaskRunExecutionResult[];
+      executions: TaskRunExecution[];
+    }>
+  ) => void;
+}

packages/core/src/v3/zodMessageHandler.ts

@@ -5,7 +5,7 @@ export interface ZodMessageCatalogSchema {
 }
 
 export type ZodMessageHandlers<TCatalogSchema extends ZodMessageCatalogSchema> = Partial<{
-  [K in keyof TCatalogSchema]: (payload: z.infer<TCatalogSchema[K]>) => Promise<void>;
+  [K in keyof TCatalogSchema]: (payload: z.infer<TCatalogSchema[K]>) => Promise<any>;
 }>;
 
 export type ZodMessageHandlerOptions<TMessageCatalog extends ZodMessageCatalogSchema> = {
@@ -31,6 +31,10 @@ const messageSchema = z.object({
   payload: z.unknown(),
 });
 
+interface EventEmitterLike {
+  on(eventName: string | symbol, listener: (...args: any[]) => void): this;
+}
+
 export class ZodMessageHandler<TMessageCatalog extends ZodMessageCatalogSchema> {
   #schema: TMessageCatalog;
   #handlers: ZodMessageHandlers<TMessageCatalog> | undefined;
@@ -50,10 +54,13 @@ export class ZodMessageHandler<TMessageCatalog extends ZodMessageCatalogSchema>
     const handler = this.#handlers[parsedMessage.type];
 
     if (!handler) {
-      throw new Error(`Unknown message type: ${String(parsedMessage.type)}`);
+      console.error(`No handler for message type: ${String(parsedMessage.type)}`);
+      return;
     }
 
-    await handler(parsedMessage.payload);
+    const ack = await handler(parsedMessage.payload);
+
+    return ack;
   }
 
   public parseMessage(message: unknown): MessageFromCatalog<TMessageCatalog> {
@@ -80,6 +87,35 @@ export class ZodMessageHandler<TMessageCatalog extends ZodMessageCatalogSchema>
       payload: parsedPayload.data,
     };
   }
+
+  public registerHandlers(emitter: EventEmitterLike, logger?: (...args: any[]) => void) {
+    const log = logger ?? console.log;
+
+    if (!this.#handlers) {
+      log("No handlers provided");
+      return;
+    }
+
+    for (const eventName of Object.keys(this.#schema)) {
+      emitter.on(eventName, async (message: any, callback?: any): Promise<void> => {
+        log(`handling ${eventName}`, message);
+
+        let ack;
+
+        if ("payload" in message) {
+          ack = await this.handleMessage({ type: eventName, ...message });
+        } else {
+          // Handle messages not sent by ZodMessageSender
+          const { version, ...payload } = message;
+          ack = await this.handleMessage({ type: eventName, version, payload });
+        }
+
+        if (callback && typeof callback === "function") {
+          callback(ack);
+        }
+      });
+    }
+  }
 }
 
 type ZodMessageSenderCallback<TMessageCatalog extends ZodMessageCatalogSchema> = (message: {
@@ -121,3 +157,7 @@ export class ZodMessageSender<TMessageCatalog extends ZodMessageCatalogSchema> {
     await this.#sender({ type, payload, version: "v1" });
   }
 }
+
+export type MessageCatalogToSocketIoEvents<TCatalog extends ZodMessageCatalogSchema> = {
+  [K in keyof TCatalog]: (message: z.infer<TCatalog[K]>) => void;
+};

packages/core/src/v3/zodNamespace.ts

@@ -0,0 +1,158 @@
+import { DisconnectReason, Namespace, Server, Socket } from "socket.io";
+import {
+  ZodMessageCatalogSchema,
+  ZodMessageHandlerOptions,
+  MessageCatalogToSocketIoEvents,
+  ZodMessageHandler,
+  ZodMessageSender,
+} from "./zodMessageHandler";
+
+interface ExtendedError extends Error {
+  data?: any;
+}
+
+export type ZodSocket<
+  TClientMessages extends ZodMessageCatalogSchema,
+  TServerMessages extends ZodMessageCatalogSchema,
+> = Socket<
+  MessageCatalogToSocketIoEvents<TClientMessages>,
+  MessageCatalogToSocketIoEvents<TServerMessages>
+>;
+
+interface ZodNamespaceOptions<
+  TClientMessages extends ZodMessageCatalogSchema,
+  TServerMessages extends ZodMessageCatalogSchema,
+> {
+  io: Server;
+  name: string;
+  clientMessages: TClientMessages;
+  serverMessages: TServerMessages;
+  messageHandler?: ZodMessageHandlerOptions<TClientMessages>["messages"];
+  authToken?: string;
+  preAuth?: (
+    socket: ZodSocket<TClientMessages, TServerMessages>,
+    next: (err?: ExtendedError) => void
+  ) => void;
+  postAuth?: (
+    socket: ZodSocket<TClientMessages, TServerMessages>,
+    next: (err?: ExtendedError) => void
+  ) => void;
+  onConnection?: (
+    socket: ZodSocket<TClientMessages, TServerMessages>,
+    handler: ZodMessageHandler<TClientMessages>,
+    sender: ZodMessageSender<TServerMessages>,
+    logger: (...args: any[]) => void
+  ) => Promise<void>;
+  onDisconnect?: (
+    socket: ZodSocket<TClientMessages, TServerMessages>,
+    reason: DisconnectReason,
+    description: any,
+    logger: (...args: any[]) => void
+  ) => Promise<void>;
+  onError?: (
+    socket: ZodSocket<TClientMessages, TServerMessages>,
+    err: Error,
+    logger: (...args: any[]) => void
+  ) => Promise<void>;
+}
+
+export class ZodNamespace<
+  TClientMessages extends ZodMessageCatalogSchema,
+  TServerMessages extends ZodMessageCatalogSchema,
+> {
+  #handler: ZodMessageHandler<TClientMessages>;
+  sender: ZodMessageSender<TServerMessages>;
+
+  io: Server;
+  namespace: Namespace<
+    MessageCatalogToSocketIoEvents<TClientMessages>,
+    MessageCatalogToSocketIoEvents<TServerMessages>
+  >;
+
+  constructor(opts: ZodNamespaceOptions<TClientMessages, TServerMessages>) {
+    this.#handler = new ZodMessageHandler({
+      schema: opts.clientMessages,
+      messages: opts.messageHandler,
+    });
+
+    this.io = opts.io;
+
+    this.namespace = this.io.of(opts.name);
+
+    this.sender = new ZodMessageSender({
+      schema: opts.serverMessages,
+      sender: async (message) => {
+        return new Promise((resolve, reject) => {
+          try {
+            // @ts-expect-error
+            this.namespace.emit(message.type, message.payload);
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
+      },
+    });
+
+    if (opts.preAuth) {
+      this.namespace.use(opts.preAuth);
+    }
+
+    if (opts.authToken) {
+      this.namespace.use((socket, next) => {
+        const logger = createLogger(`[${opts.name}][${socket.id}][auth]`);
+
+        const { auth } = socket.handshake;
+
+        if (!("token" in auth)) {
+          logger("no token");
+          return socket.disconnect(true);
+        }
+
+        if (auth.token !== opts.authToken) {
+          logger("invalid token");
+          return socket.disconnect(true);
+        }
+
+        logger("success");
+
+        next();
+      });
+    }
+
+    if (opts.postAuth) {
+      this.namespace.use(opts.postAuth);
+    }
+
+    this.namespace.on("connection", async (socket) => {
+      const logger = createLogger(`[${opts.name}][${socket.id}]`);
+      logger("connection");
+
+      this.#handler.registerHandlers(socket, logger);
+
+      socket.on("disconnect", async (reason, description) => {
+        logger("disconnect", { reason, description });
+
+        if (opts.onDisconnect) {
+          await opts.onDisconnect(socket, reason, description, logger);
+        }
+      });
+
+      socket.on("error", async (error) => {
+        logger("error", error);
+
+        if (opts.onError) {
+          await opts.onError(socket, error, logger);
+        }
+      });
+
+      if (opts.onConnection) {
+        await opts.onConnection(socket, this.#handler, this.sender, logger);
+      }
+    });
+  }
+}
+
+function createLogger(prefix: string) {
+  return (...args: any[]) => console.log(prefix, ...args);
+}

packages/database/prisma/migrations/20240213154458_add_image_details/migration.sql

@@ -0,0 +1,30 @@
+-- CreateTable
+CREATE TABLE "ImageDetails" (
+    "id" TEXT NOT NULL,
+    "friendlyId" TEXT NOT NULL,
+    "contentHash" TEXT NOT NULL,
+    "tag" TEXT NOT NULL,
+    "backgroundWorkerId" TEXT,
+    "projectId" TEXT NOT NULL,
+    "runtimeEnvironmentId" TEXT NOT NULL,
+    "metadata" JSONB NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "ImageDetails_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ImageDetails_friendlyId_key" ON "ImageDetails"("friendlyId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ImageDetails_projectId_runtimeEnvironmentId_tag_key" ON "ImageDetails"("projectId", "runtimeEnvironmentId", "tag");
+
+-- AddForeignKey
+ALTER TABLE "ImageDetails" ADD CONSTRAINT "ImageDetails_backgroundWorkerId_fkey" FOREIGN KEY ("backgroundWorkerId") REFERENCES "BackgroundWorker"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ImageDetails" ADD CONSTRAINT "ImageDetails_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ImageDetails" ADD CONSTRAINT "ImageDetails_runtimeEnvironmentId_fkey" FOREIGN KEY ("runtimeEnvironmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/database/prisma/migrations/20240224132039_change_image_details_unique_constraint/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[projectId,runtimeEnvironmentId,contentHash]` on the table `ImageDetails` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "ImageDetails_projectId_runtimeEnvironmentId_tag_key";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ImageDetails_projectId_runtimeEnvironmentId_contentHash_key" ON "ImageDetails"("projectId", "runtimeEnvironmentId", "contentHash");

packages/database/prisma/migrations/20240228114913_add_checkpoint_model/migration.sql

@@ -0,0 +1,30 @@
+-- CreateEnum
+CREATE TYPE "CheckpointType" AS ENUM ('DOCKER', 'KUBERNETES');
+
+-- CreateTable
+CREATE TABLE "Checkpoint" (
+    "id" TEXT NOT NULL,
+    "friendlyId" TEXT NOT NULL,
+    "type" "CheckpointType" NOT NULL,
+    "location" TEXT NOT NULL,
+    "reason" TEXT,
+    "attemptId" TEXT NOT NULL,
+    "projectId" TEXT NOT NULL,
+    "runtimeEnvironmentId" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "Checkpoint_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Checkpoint_friendlyId_key" ON "Checkpoint"("friendlyId");
+
+-- AddForeignKey
+ALTER TABLE "Checkpoint" ADD CONSTRAINT "Checkpoint_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "TaskRunAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Checkpoint" ADD CONSTRAINT "Checkpoint_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Checkpoint" ADD CONSTRAINT "Checkpoint_runtimeEnvironmentId_fkey" FOREIGN KEY ("runtimeEnvironmentId") REFERENCES "RuntimeEnvironment"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/database/prisma/schema.prisma

@@ -389,6 +389,8 @@ model RuntimeEnvironment {
   taskQueues                TaskQueue[]
   batchTaskRuns             BatchTaskRun[]
   environmentVariableValues EnvironmentVariableValue[]
+  imageDetails              ImageDetails[]
+  checkpoints               Checkpoint[]
 
   @@unique([projectId, slug, orgMemberId])
   @@unique([projectId, shortcode])
@@ -432,6 +434,8 @@ model Project {
   taskTags              TaskTag[]
   taskQueues            TaskQueue[]
   environmentVariables  EnvironmentVariable[]
+  imageDetails          ImageDetails[]
+  checkpoints           Checkpoint[]
 }
 
 enum ProjectVersion {
@@ -1513,9 +1517,10 @@ model BackgroundWorker {
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
-  tasks      BackgroundWorkerTask[]
-  attempts   TaskRunAttempt[]
-  lockedRuns TaskRun[]
+  tasks        BackgroundWorkerTask[]
+  attempts     TaskRunAttempt[]
+  lockedRuns   TaskRun[]
+  imageDetails ImageDetails[]
 
   @@unique([projectId, runtimeEnvironmentId, version])
 }
@@ -1670,6 +1675,8 @@ model TaskRunAttempt {
   taskRunDependency      TaskRunDependency? @relation("dependentAttempt")
   batchTaskRunDependency BatchTaskRun?
 
+  checkpoints Checkpoint[]
+
   @@unique([taskRunId, number])
 }
 
@@ -1884,3 +1891,55 @@ model EnvironmentVariableValue {
 
   @@unique([variableId, environmentId])
 }
+
+model ImageDetails {
+  id String @id @default(cuid())
+
+  friendlyId String @unique
+
+  contentHash String
+  tag         String
+
+  backgroundWorker   BackgroundWorker? @relation(fields: [backgroundWorkerId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  backgroundWorkerId String?
+
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  projectId String
+
+  runtimeEnvironment   RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  runtimeEnvironmentId String
+
+  metadata Json
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  @@unique([projectId, runtimeEnvironmentId, contentHash])
+}
+
+model Checkpoint {
+  id String @id @default(cuid())
+
+  friendlyId String @unique
+
+  type     CheckpointType
+  location String
+  reason   String?
+
+  attempt   TaskRunAttempt @relation(fields: [attemptId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  attemptId String
+
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  projectId String
+
+  runtimeEnvironment   RuntimeEnvironment @relation(fields: [runtimeEnvironmentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  runtimeEnvironmentId String
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+}
+
+enum CheckpointType {
+  DOCKER
+  KUBERNETES
+}

packages/emails/package.json

@@ -29,7 +29,7 @@
   },
   "devDependencies": {
     "@trigger.dev/tsconfig": "workspace:*",
-    "@types/node": "16",
+    "@types/node": "^18",
     "@types/react": "18.2.17",
     "typescript": "^4.9.4"
   },

packages/otlp-importer/README.md

@@ -1,17 +1,25 @@
+# OTLP Importer
+
 ## Getting started
 
 Install dependencies:
 
-```sh
+```sh Mac
 brew install protobuf
 ```
 
-# Submodules
+```sh Linux
+apt install -y protobuf-compiler
+```
+
+Alternatively, follow the [manual install instructions](https://github.com/protocolbuffers/protobuf?tab=readme-ov-file#protobuf-compiler-installation) for the protobuf compiler.
+
+## Submodules
 
 **Submodule is always pointing to certain revision number. So updating the submodule repo will not have impact on your code.
 Knowing this if you want to change the submodule to point to a different version (when for example proto has changed) here is how to do it:**
 
-## Updating submodule to point to certain revision number
+### Updating submodule to point to certain revision number
 
 1. Make sure you are in the same folder as this instruction
 

packages/otlp-importer/package.json

@@ -40,7 +40,7 @@
     "@trigger.dev/tsconfig": "workspace:*",
     "@trigger.dev/tsup": "workspace:*",
     "@types/jest": "^29.5.3",
-    "@types/node": "16",
+    "@types/node": "^18",
     "jest": "^29.6.2",
     "rimraf": "^3.0.2",
     "ts-jest": "^29.1.1",

references/v3-catalog/package.json

@@ -3,6 +3,7 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
+    "build:trigger": "trigger.dev build",
     "dev:trigger": "trigger.dev dev",
     "start": "ts-node -r tsconfig-paths/register -r dotenv/config src/index.ts",
     "start:concurrency": "ts-node -r tsconfig-paths/register -r dotenv/config src/concurrencyUsage.ts",

references/v3-catalog/src/index.ts

@@ -1,7 +1,7 @@
-import { parentTask, simpleParentTask } from "./trigger/simple";
+import { parentTask } from "./trigger/simple";
 
 export async function main() {
-  const handle = await simpleParentTask.trigger({
+  const handle = await parentTask.trigger({
     payload: { message: "This is a message from the trigger-dev CLI" },
   });